C# Hadoop学习笔记（一）—环境安装

一、安装环境

1，前期准备：官网下载“NuGet Package Manager”，按自己已有的VS环境下载对应版本；

2，利用NuGet下载Hadoop For .NET SDK,地址“http://hadoopsdk.codeplex.com/”

3，安装。

4，通过HDInsight，安装Windows Azure，目前是预览版本。

5，参照网址“http://blogs.msdn.com/b/data_otaku/archive/2013/08/14/hadoop-for-net-developers.aspx” 系统学习API

 

二、测试DEMO

 

using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Threading.Tasks;using Microsoft.Hadoop;using Microsoft.Hadoop.MapReduce;using Microsoft.Hadoop.WebClient.WebHCatClient;using System.Diagnostics;using System.IO;using System.IO.MemoryMappedFiles;namespace HadoopConsol{    class Program    {        static void Main(string[] args)        {            Stopwatch sw = new Stopwatch();            long hadoopTime=0;            long normalTime=0;            sw.Start();            //start hadoop            Console.WriteLine("     Hadoop Process Strating ....");            #region Hadoop time            #region hadoopconnet            Console.WriteLine("     Hadoop Connect Strating ....");            //establish job configuration            HadoopJobConfiguration myConfig = new HadoopJobConfiguration();            myConfig.InputPath = "/demo/simple/in";            myConfig.OutputFolder = "/demo/simple/out";            //connect to cluster            Uri myUri = new Uri("http://localhost");            string userName = "hadoop";            string passWord = null;            IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord);            hadoopTime += sw.ElapsedMilliseconds;            Console.WriteLine("     Hadoop Connect End.");            Console.WriteLine("     Hadoop Connect time:" + sw.ElapsedMilliseconds);            #endregion            #region hadoopmapreduce            sw.Reset();            sw.Start();            Console.WriteLine("     Hadoop MapReduce Strating ....");            //execute mapreduce job            MapReduceResult jobResult =                myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig);            hadoopTime += sw.ElapsedMilliseconds;            Console.WriteLine("     Hadoop MapReduce End.");            Console.WriteLine("     Hadoop MapReduce Time:"+sw.ElapsedMilliseconds);            #endregion            #region Hadoop End            sw.Reset();            sw.Start();            Console.WriteLine("     Hadoop Endprocess Strating ....");            //write job result to console            int exitCode = jobResult.Info.ExitCode;            string exitStatus = "Failure";            if (exitCode == 0) exitStatus = "Success";            exitStatus = exitCode + " (" + exitStatus + ")";            Console.WriteLine();            Console.Write("Exit Code = " + exitStatus);            Console.WriteLine("     Hadoop Endprocess End.");            hadoopTime += sw.ElapsedMilliseconds;            Console.WriteLine("     Hadoop Exit Time:" + sw.ElapsedMilliseconds);            Console.WriteLine("     Hadoop Process All Time:" + hadoopTime);            #endregion            #endregion            #region Normal time            //start Normal            Console.WriteLine("     Normal Process Strating ....");            sw.Reset();            sw.Start();            //normal process            #region Normal Process            int myevenCount = 0;            int myeventSum = 0;            int myoddCount = 0;            int myoddSum = 0;            StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt");            while (fs.Peek() >= 0)            {                string strTemp = fs.ReadLine();                if (Int32.Parse(strTemp) % 2 == 0)                {                    myevenCount++;                    myeventSum += Int32.Parse(strTemp);                }                else                {                    myoddCount++;                    myoddSum += Int32.Parse(strTemp);                }            }               //MemoryMappedFile m = MemoryMappedFile.            Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum);            Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum);            #endregion            Console.WriteLine("     Normal Process End.");            normalTime += sw.ElapsedMilliseconds;            Console.WriteLine("     Normal Exit Time:" + sw.ElapsedMilliseconds);            Console.WriteLine("     Normal Process All Time:" + normalTime);            #endregion            sw.Stop();            Console.Read();        }    }    public class MySimpleMapper : MapperBase    {        public override void Map(string inputLine, MapperContext context)        {            //interpret the incoming line as an integer value            int value = int.Parse(inputLine);            //determine whether value is even or odd            string key = (value % 2 == 0) ? "even" : "odd";            //output key assignment with value            context.EmitKeyValue(key, value.ToString());        }    }    public class MySimpleReducer : ReducerCombinerBase    {        public override void Reduce(            string key, IEnumerable<string> values, ReducerCombinerContext context            )        {            //initialize counters            int myCount = 0;            int mySum = 0;            //count and sum incoming values            foreach (string value in values)            {                mySum += int.Parse(value);                myCount++;            }            //output results            context.EmitKeyValue(key, myCount + "\t" + mySum);        }    }}

三、测试结果