hadoop第一个例子WordCount

hadoop查看自己空间 http://127.0.0.1:50070/dfshealth.jsp

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.GenericOptionsParser;

public class WordCount {

public static class TokenizerMapper

extends Mapper<Object, Text, Text, IntWritable>{



private final static IntWritable one = new IntWritable(1);

private Text word = new Text();



public void map(Object key, Text value, Context context

) throws IOException, InterruptedException {

StringTokenizer itr = new StringTokenizer(value.toString());

while (itr.hasMoreTokens()) {

word.set(itr.nextToken());

context.write(word, one);

}

}

}



public static class IntSumReducer

extends Reducer<Text,IntWritable,Text,IntWritable> {

private IntWritable result = new IntWritable();

public void reduce(Text key, Iterable<IntWritable> values,

Context context

) throws IOException, InterruptedException {

int sum = 0;

for (IntWritable val : values) {

sum += val.get();

}

result.set(sum);

context.write(key, result);

}

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

if (otherArgs.length != 2) {

System.err.println("Usage: wordcount <in> <out>");

System.exit(2);

}

Job job = new Job(conf, "word count");

job.setJarByClass(WordCount.class);

job.setMapperClass(TokenizerMapper.class);

job.setCombinerClass(IntSumReducer.class);

job.setReducerClass(IntSumReducer.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

FileInputFormat.addInputPath(job, new Path(otherArgs[0]));

FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

System.exit(job.waitForCompletion(true) ? 0 : 1);

}

}

3.配置Map/Reduce Locations

在Window-->Show View中打开Map/Reduce Locations,在Map/Reduce Locations中新建一个Hadoop Location。在这个View中，右键-->New Hadoop Location。在弹出的对话框中你需要配置Location name，如Hadoop，还有Map/Reduce Master和DFS Master。这里面的Host、Port分别为你在mapred-site.xml、core-site.xml中配置的地址及端口。如：

4.新建项目。

File-->New-->Other-->Map/Reduce Project,项目名可以随便取，如WordCount。

复制 hadoop安装目录/src/example/org/apache/hadoop/examples/WordCount.java到刚才新建的项目WordCount下,删除WordCount.java首行package

5.在本地新建word.txt，内容为：

java c++ python cjava c++ javascript helloworld hadoopmapreduce java hadoop hbase

6.通过hadoop的命令在HDFS上创建/tmp/workcount目录，命令如下：

bin/hadoop fs -mkdir /tmp/wordcount

通过copyFromLocal命令把本地的word.txt复制到HDFS上，命令如下：

bin/hadoop fs -copyFromLocal /home/wangxing/Development/eclipseWorkspace/word.txt/tmp/wordcount/word.txt

7.运行项目

(1).在新建的项目Hadoop，点击WordCount.java，右键-->Run As-->Run Configurations

(2).在弹出的Run Configurations对话框中，点Java Application，右键-->New，这时会新建一个application名为WordCount

(3).配置运行参数，点Arguments，在Program arguments中输入你要传给程序的输入文件夹和你要求程序将计算结果保存的文件夹，如：

hdfs://localhost:9000/tmp/wordcount/word.txt hdfs://localhost:9000/tmp/wordcount/out

(4)点击Run，运行程序

过段时间将运行完成，等运行结束后，查看例子的输出结果，使用命令：

bin/hadoop fs -ls /tmp/wordcount/out

发现有两个文件夹和一个文件，使用命令查看part-r-00000里的运行结果:

bin/hadoop fs -cat /tmp/wordcount/out/part-r-00000

巴特西

hadoop第一个例子WordCount

最新文章

热门文章