任務7實訓：MapReduce編程

任務七實訓：MapReduce編程一、實驗目的1.1掌握MapReduce的編程方法二、實驗內容2.1在idea中新建一個Maven工程2.2在pom文件中添加依賴添加依賴<dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>2.9.2</version></dependency>2.3新建一個class命名為WordCount，importorg.apache.hadoop.conf.Configuration;

importorg.apache.hadoop.fs.Path;

importorg.apache.hadoop.io.IntWritable;

importorg.apache.hadoop.io.Text;

importorg.apache.hadoop.mapreduce.Job;

importorg.apache.hadoop.mapreduce.Mapper;

importorg.apache.hadoop.mapreduce.Reducer;

importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;

importorg.apache.hadoop.mapreduce.lib.input.TextInputFormat;

importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

importorg.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

importjava.io.IOException;

importjava.util.StringTokenizer;publicclassWordCount{

publicstaticclassMyMapperextendsMapper<Object,Text,Text,IntWritable>{

privateTextword=newText();

privatefinalstaticIntWritableone=newIntWritable(1);

@Override

protectedvoidmap(Objectkey,Textvalue,Contextcontext)throwsIOException,InterruptedException{

StringTokenizeritr=newStringTokenizer(value.toString());

while(itr.hasMoreTokens()){

word.set(itr.nextToken());

context.write(word,one);

}

}

}

publicstaticclassMyReducerextendsReducer<Text,IntWritable,Text,IntWritable>{

privateIntWritableresult=newIntWritable();

@Override

protectedvoidreduce(Textkey,Iterable<IntWritable>values,Contextcontext)throwsIOException,InterruptedException{

intsum=0;

for(IntWritableval:values){

sum+=val.get();

}

result.set(sum);

context.write(key,result);

}

}

publicstaticvoidmain(String[]args)throwsIOException,InterruptedException,ClassNotFoundException{

Configurationconf=newConfiguration();

Jobjob=Job.getInstance(conf);

job.setJarByClass(WordCount.class);

job.setJobName("WordCount");

job.setInputFormatClass(TextInputFormat.class);

job.setOutputFormatClass(TextOutputFormat.class);

FileInputFormat.addInputPath(job,newPath(args[0]));

FileOutputFormat.setOutputPath(job,newPath(args[1]));

job.setMapperClass(MyMapper.class);

job.setReducerClass(MyReducer.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(IntWritable.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

job.waitForCompletion(true);

}

}2.4.配置hadoop環(huán)境變量Windows本地解壓Hadoop壓縮包，然后像配置JDK環(huán)境變量一樣在系統(tǒng)環(huán)境變量里配置HADOOP_HOME和path：%HADOOP_HOME\bin%環(huán)境變量。注意：hadoop安裝目錄盡量不要包含空格或者中文字符。2.5.下載hadoop對應版本補丁MapReduce程序在Linux、mac系統(tǒng)中可以直接運行，但無法直接在Windows下運行，因為MapReduce代碼在Windows下運行存在bug，需要添加hadoop補丁才可以。Hadoop補丁下載地址：/cdarlint/winutilsHadoop補丁文件為hadoop.dll和winutils.exe。2.6補丁添加hadoop安裝目錄hadoop補丁文件下載之后，解壓，然后將相應版本中的hadoop.dll和winutils.exe補丁文件，拷貝到{HADOOP_HOME}/bin目錄下（可能需要重啟IDEA）2.6補丁添加windowsc盤目錄如果1.3步驟操作完之后，還無法運行MapReduce程序，則需要將hadoop.dll和winutils.exe補丁文件，拷貝