hadoop下c++程序-天气实例

来源：互联网发布：windows光盘怎么安装编辑：程序博客网时间：2024/04/30 14:03

很希望能在hadoop上做c++程序，自己对c++还是有点情节的，根据《hadoop权威指南中文第二版》Hadoop的Pipes进行了试验，并测试成功

#include <algorithm>  #include <limits.h>  #include <stdint.h>  #include <string>    #include "Pipes.hh"  #include "TemplateFactory.hh"  #include "StringUtils.hh"    class MaxTemperatureMapper : public HadoopPipes::Mapper {  public:    MaxTemperatureMapper(HadoopPipes::TaskContext& context) {    }    void map(HadoopPipes::MapContext& context) {      std::string line = context.getInputValue();      std::string year = line.substr(15, 4);      std::string airTemperature = line.substr(87, 5);      std::string q = line.substr(92, 1);      if (airTemperature != "+9999" &&          (q == "0" || q == "1" || q == "4" || q == "5" || q == "9")) {        context.emit(year, airTemperature);      }    }  };    class MapTemperatureReducer : public HadoopPipes::Reducer {  public:    MapTemperatureReducer(HadoopPipes::TaskContext& context) {    }    void reduce(HadoopPipes::ReduceContext& context) {      int maxValue = INT_MIN;      while (context.nextValue()) {        maxValue = std::max(maxValue, HadoopUtils::toInt(context.getInputValue()));      }      context.emit(context.getInputKey(), HadoopUtils::toString(maxValue));    }  };    int main(int argc, char *argv[]) {    return HadoopPipes::runTask(HadoopPipes::TemplateFactory<MaxTemperatureMapper,                                 MapTemperatureReducer>());  }

注意：和书上不一样的地方：limit.h头文件

Makefile文件（自己进行了修改）：

.SUFFIXES:.h .c .cpp .oCC=g++CPPFLAGS = -m64 RM = rmSRCS = max_temperature.cppPROGRAM = max_temperatureINC_PATH = -I$(HADOOP_DEV_HOME)/includeLIB_PATH = -L$(HADOOP_DEV_HOME)/lib/nativeLIBS = -lhadooppipes -lcrypto -lhadooputils -lpthread$(PROGRAM):$(SRCS)$(CC) $(CPPFLAGS) $(INC_PATH) $< -Wall $(LIB_PATH) $(LIBS)  -g -O2 -o $@.PHONY:cleanclean:$(RM) $(PROGRAM)

源数据文件：

0067011990999991950051507004+68750+023550FM-12+038299999V0203301N00671220001CN9999999N9+00001+99999999999
0043011990999991950051512004+68750+023550FM-12+038299999V0203201N00671220001CN9999999N9+00221+99999999999
0043011990999991950051518004+68750+023550FM-12+038299999V0203201N00261220001CN9999999N9-00111+99999999999
0043012650999991949032412004+62300+010750FM-12+048599999V0202701N00461220001CN0500001N9+01111+99999999999
0043012650999991949032418004+62300+010750FM-12+048599999V0202701N00461220001CN0500001N9+00781+99999999999

上传到HDFS：hdfs dfs -put sample.txt

make后生成了可执行文件上传到HDFS： hdfs dfs -put max_temperature /bin

执行方法： hadoop pipes -D hadoop.pipes.java.recordreader=true -D hadoop.pipes.java.recordwriter=true -input /user/root/sample.txt -output /output -program /bin/max_temperature
数据输出结果：

1 0