1.需求

2.思路

3.代码实现

3.1MyWeather 类代码:

这个类主要是用来定义hadoop的配置,在执行计算程序时所需加载的一些类。

package com.hadoop.mr.weather;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TestMapReduceLazyOutput.TestMapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class MyWeather { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf =new Configuration(true);
Job job = Job.getInstance(conf); job.setJarByClass(MyWeather.class); //----------conf----------------------- //---begin Map :
//输入格式化类
// job.setInputFormatClass(ooxx.class);
//设置mapper类
job.setMapperClass(TMapper.class); job.setMapOutputKeyClass(TQ.class);
job.setMapOutputValueClass(IntWritable.class); //设置partitioner类
job.setPartitionerClass(TPartitioner.class); //设置排序比较器类
job.setSortComparatorClass(TSortComparator.class); //设置combiner类
// job.setCombinerClass(TCombiner.class); //----end Map //----begin Reduce:
//设置组比较器的类
job.setGroupingComparatorClass(TGroupingComparator.class);
//设置reducer类
job.setReducerClass(TReducer.class); //-----end Reduce: //设置输入数据的路径
Path input = new Path("/data/tq/input");
FileInputFormat.addInputPath(job, input); //设置输出数据的路径
Path output=new Path("/data/tq/output");
if(output.getFileSystem(conf).exists(output)){
//如果目录存在递归删除
output.getFileSystem(conf).delete(output,true);
}
FileOutputFormat.setOutputPath(job, output); //设置reduceTask的数量 和 partitions数量对应
job.setNumReduceTasks(2);
//-------------------------------------
job.waitForCompletion(true);
}
}

3.2Tmapper类代码

该类继承Mapper类他的主要作用是对输入的文件做一些预处理工作。

package com.hadoop.mr.weather;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date; import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.StringUtils;
//TextInputFormat.class --key类型是 longWritable 偏移量 --value是Text类型 public class TMapper extends Mapper<LongWritable, Text, TQ, IntWritable>{ //创建map的 k v 对象
TQ mkey=new TQ(); // map --->key
IntWritable mval=new IntWritable(); //map --->value //重写map方法
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, TQ, IntWritable>.Context context)
throws IOException, InterruptedException {
/**
1949-10-01 14:21:02 34c
1949-10-01 19:21:02 38c
1949-10-02 14:01:02 36c
1950-01-01 11:21:02 32c
1950-10-01 12:21:02 37c
**/ try {
String[] strs = StringUtils.split(value.toString(),'\t');//对文本将制表符切分 SimpleDateFormat sdf= new SimpleDateFormat("yyyy-MM-dd");
Date date = sdf.parse(strs[0]);
Calendar cal= Calendar.getInstance();
cal.setTime(date); mkey.setYear(cal.get(Calendar.YEAR));
mkey.setMonth(cal.get(Calendar.MONTH)+1); //第一个月默认从0开始所以加1
mkey.setDay(cal.get(Calendar.DAY_OF_MONTH)); int wd = Integer.parseInt(strs[1].substring(0, strs[1].length()-1));//获取温度字符串并强转为int类型
mkey.setWd(wd); mval.set(wd); context.write(mkey, mval);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} }
}

3.3TQ类代码

该类实现WritableComparable接口他的作用是给生成相关的属性并重写 写入,读取,比较的方法,

package com.hadoop.mr.weather;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException; import org.apache.hadoop.io.WritableComparable; public class TQ implements WritableComparable<TQ> { //定义属性
private int year;
private int month;
private int day;
private int wd; //温度属性 public int getYear() {
return year;
} public void setYear(int year) {
this.year = year;
} public int getMonth() {
return month;
} public void setMonth(int month) {
this.month = month;
} public int getDay() {
return day;
} public void setDay(int day) {
this.day = day;
} public int getWd() {
return wd;
} public void setWd(int wd) {
this.wd = wd;
} @Override
public void write(DataOutput out) throws IOException {
out.writeInt(year);
out.writeInt(month);
out.writeInt(day);
out.writeInt(wd); } @Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
this.year=in.readInt();
this.month=in.readInt();
this.day=in.readInt();
this.wd=in.readInt();
} @Override
public int compareTo(TQ that) {
//compare方法返回值说明the value 0 if x == y; a value less than 0 if x < y; and a value greater than 0 if x > y
// 日期正序 ,使用这年和那年比较 -.-
int c1=Integer.compare(this.year, that.getYear());
// 如果年份相同比较天
if(c1==0){
int c2=Integer.compare(this.month, that.getMonth());
//如果是同一天返回0
if(c2==0){
return Integer.compare(this.day, that.getDay());
}
return c2;
} return 0;
} }

3.4Tpartitioner类代码

该类的作用,是定义输出文件的分布规则,避免产生数据倾斜

package com.hadoop.mr.weather;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Partitioner; public class TPartitioner extends Partitioner<TQ, IntWritable> { //约定成俗规则:避免数据倾斜,将少的数据都放在一个reduce任务组里,将数据量大的单独放一个任务组里。
@Override
public int getPartition(TQ key, IntWritable value, int numPartitions) { return key.hashCode() % numPartitions;
} }

3.5TSortComparator类代码:

该类的作用是定义一个排序比较器

package com.hadoop.mr.weather;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator; public class TSortComparator extends WritableComparator{ public TSortComparator() {
super(TQ.class,true);
} @Override
public int compare(WritableComparable a, WritableComparable b) { TQ t1=(TQ) a;
TQ t2=(TQ) b; int c1 = Integer.compare(t1.getYear(), t2.getYear());
if(c1==0){ int c2= Integer.compare(t1.getMonth(), t2.getMonth());
if(c2==0){
return -Integer.compare(t1.getWd(), t2.getWd());// -号表示返回温度的倒序排列
} }
return super.compare(a, b);
}
}

3.6TGroupingComparator类代码:

该类的作用是根据年月两个维度做分组

package com.hadoop.mr.weather;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator; public class TGroupingComparator extends WritableComparator { public TGroupingComparator() {
super(TQ.class,true);
} @Override
public int compare(WritableComparable a, WritableComparable b) { TQ t1=(TQ) a;
TQ t2=(TQ) b; int c1 = Integer.compare(t1.getYear(), t2.getYear());
if(c1==0){
return Integer.compare(t1.getMonth(), t2.getMonth()); //返回月份的比较结果来分组
}
return c1;
}
}

3.7TReducer 类代码

该类的作用是定义数据的输出格式和内容

package com.hadoop.mr.weather;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer; public class TReducer extends Reducer<TQ, IntWritable, Text, IntWritable>{ Text rkey=new Text();
IntWritable rval=new IntWritable(); /* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.Reducer#reduce(KEYIN, java.lang.Iterable, org.apache.hadoop.mapreduce.Reducer.Context)
*/
@Override
protected void reduce(TQ key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
//相同的key为一组。。。。
//1970 01 01 88 88
//1970 01 11 78 78
//1970 01 21 68 68
//1970 01 01 58 58 int flag=0; //迭代的次数
int day=0; for (IntWritable v : values) {
if(flag==0){
//将reduce的key格式化成1970-01-01:88
rkey.set(key.getYear()+"-"+key.getMonth()+"-"+key.getDay()+":"+key.getWd());
//将reduce的value设置为温度
rval.set(key.getWd());
flag++;
day=key.getDay();
context.write(rkey, rval);
}
//如果迭代次数不为0且当前的天不等于迭代得到的天就将新的天气数据赋值给reduce的 kv
if(flag!=0 && day!=key.getDay()){
//将reduce的key格式化成1970-01-01:88
rkey.set(key.getYear()+"-"+key.getMonth()+"-"+key.getDay()+":"+key.getWd());
//将reduce的value设置为温度
rval.set(key.getWd());
context.write(rkey, rval);
break;
}
} }
}

4.执行程序

4.1将包导出为jar包 上传至服务器

aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAM4AAACGCAYAAACVFLXFAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAADIVSURBVHhe7Z0HuBXltbDPOfrkz01u7s0NN/FRE7GBFWPNNYBgEqMmGoMaGxaqCAoKiiISQaVXwYMgYlCxKxZsFEXFFguWiErvXYpYY+r61/pm1uw1a9a0vfcp6OF5XmfP12bNzHrPNzO7WFHZ5DdQsc8JULHviRGojqhq+tsQXK6J66/L81K5TzIVe2P8Sex7PO7jr/G1Yq/jPPz1Sp+KPbE9Q/V7/gr5RT4aHxtQuUcrk4qfHLNDUPXjZCp3b5lIxW4tErHGlFT+uHkiFbv/vCSsMSVWn1hxOOmJb7w4e/0ymQRpGsRJxxpTYiVzbRIvjpWsftIT3zhxuDzAkEWyo4vzY5wVErCSvZxYsoT4Cc5KPlYyyySvCeS2ApISnmkQx5BF0iBOSZiySHYEcTjZGS1NfRQnjYpSxdFipNEgTi5MWST1WRxOcsYShtFtmUjCx5TnRYvQstOwXLToNBQZAi06Kjr4+OstfYLygEH5aD9QcQO0bBelRS1giqIxZJFYyV5OTFkk9VKcfVEEg6omJxWoh+JUX9UWug+/J9tyREz5sLugus95uH43rp9bWHJ55uVUXJ5fWA7F5dVyeScuLzCWd+CyXY0tnTh7GKLkxExmgSVDHqwxJVYy1yaZxQlJU0/FoeSXs4rkF50GQpueY6Jcprh0lMdlCi4PGJmTEYrhNj1K49j2/SMzDEPy0LI+iGP1yYOVzBI5IxWDNabEFKei6UlQMk1+m8w+KE8ClrxEUG+IQzOGFoYhSY7tMgJaXTjccYwPr7uyzsRQR6sLh3nLgCGOY/DyLp7BmWnVaVCEYzoODEirtzi282CUb4S7DAwQ4hQ741j3X/K+TNeVG3MbxuVjCNnWwJIhD5Zs9VacUL0hTtKMQ+KQHLzewidY78jg/Qvd19A9kLjHadlxsKNFhySMe5sYWnYYGCKU7EhavUWrjoMSxck144jkiyQu1nFZpK5cpGw/FW4bgyVDqaA4v8PkT8OQRWLJIhESWKSJoy/dSILqK9sGQmhIHJphdPnXXhwGhck144jksxKXyyJ15SJl+6lw2xisxM+DfanW5GRM7hQsWSSWLAJLjDTSxKEbfi0G840XB6EHEA3ieFgy5KHmxEnBEiONNHHyzTj0mi7d8Ka69wToM/5RmDrjjXpNvwmPQttrJppSEGni0FM7WlY0Die9hUy+oIwTD+tk21BdRnR/Tdr2g9dFYsmQh5h7HOvSTGMLkRVLjDTSxMk343jidB40Fe546nV4/b1F8PHHH8Nnn30Gf/3rX+Grr76qF1Asn3/+OWzfvh3e+mApPDH3Hehb/XBECkKKY81UbsbBZYM4NSVOk9/jrJJAPRUn74xzYo9xcNsTr8Li5avhyy+/hH/9619QX/9RbCTStm3b4PG5b5szjyWOFKhhxilgyVAqO6w4eWecbsPuhedenw9ffPFFvZaG//373/928sybvwT6jp8WCMOkidMw4xSwEj8P9j0OiWHd14TARJYYcoTQ7fV4wSVgDKH2KJJ4k5Vo2Xk4zjjnuCUhpSG0OPRA4LbHX4FVa9bD3//+d5eUNQZK+eX8p+DL9x6363Pwj3/8A7Zu3epib9l+UIhWHQdDmx4oTjuURdUR9IkFWlY2ps/OYSLWYyr3bJ2I1SeAHn5Y5YKQpFqsLH8YLHnNxNVoESxZJLp9SATE2oYk1N6fhZQ43UfcGxKHXxNtet0IrbpgUglxbn3sZXfpQ7ONlaTl4l9//Qy2T+sNnzzR36zPA8VK92IUuxYjTRz6uA8tv/biZEEnveIbJY6cccKMMMQZDpMefdGJ889//tMlZE3x1fLXYPPY38Lnz48z6/NAsVLMFDsJIiVp3XEInHrpSPea6yQ049CyYg/6hLZKpHpC2cRIQye9IlUci2iiWigRLFkkur0eT4uiCbXPNuM4Qfx1Eid8qTYcJj78grvsocsfSsga4W9fwfbH+8OmQT/Dy7Un7TY54Es1il1KkUUcmnFoWR/FkbNJrYhjzBhaFoluaxJNVAslgiWLRLfX42lRNKH26TNOqjhYdvNDz9e4OJ+/fg9svO4w2HbXhfCPTzeZbfLA4lDsUooddcbRwjBW27JSTnFoLMJMXI0WwZJFotuHRECsbUhU+7QZx2NEgHephkkl6qsfmANbtmxxDwcoIcvNZ28+gDPN/8GmIc3hyyUvmm3yQrFSzBQ7C0Gi7GgzjiWLxOqTRq7+lgCCzOKwNDuKOHq8FigHzTi01LS8cKQ346A4QVmnETDuvmdg8+bN8Le//c0lZNn46xfw6StTYOPAI2HjtQfBpy9NCtV/tXUNfPHBM/DXTcvg7zm3TbFSzBQ7C8PsSE/VZJJrqvY61iyXpI2n6yNYEgjSxAnKIuJEElUTTlxTFolur8fTomh0ezUeyUAzDouhCcTBexuCxBlzzyz46KOP3HsjlJDl4Kuta2H7EwNgwzX7w4a+TWH7jCFBHYnyyfM3w7ZH+sL22SPgy9XvhPpmgWKlmCl2KQXJsiM9VdOJTpAwjFUvSRtL1lskSUESyHqrTVAWESctca02gsr9TnFkLc9PWEySo7r32RFhGGvGGXXXjLKKQyJsua0tbLh6H9j4xwNh+9ODXflXn3+MwoyHzdUnweaxv4FPXpoMX322NdI/CywOxa4/ONqq0xBocxnuo1uPfgCVvolKS/oNhKo9f+kgiSRx5RozGTMkcFBnjClxl5NI1vIscF+iqjGKmoAWR2P12WHF6T7yvkAMjSXOiDufgk2bNpXls2mfr3wLPrrx17Dhysaw8bpD4RO8VKPyL9a8B5snnenKN09pB5+veifSNw8UK8VMsbMwTFgcRXv/HgeX9GMhUg6djOUQx6qXWGNasMiM1SYLUhrCSnyJJYvE6rODiBOmReeROOOc5ZYWbXqNRXFGBeskztDbn4CNGze6z6lRQhbL52s/gE1jfg3rr/gxbBx0FHz27hOu/LNFc2HTyNaufNtj/eDLjz1JS4FipZgpdi1HmjjuqZovjpVMTFIdIyVJlYXeyVfv5sukTqKmxJGxlEqw/1Zi7gjidB95fyCGxhJn8J+mw4YNG9xn1Sghi+KTLbD59vaw/vJdYX3ffWD7K7e78s/WLYCNI1AaLN9yd1dstzXatwgoVoqZYtdypF2q0Q+H0NL9XBUn0E/whEtkciVg3YdYSRWHldwS3k7W8jS4X4ARUyns0OLkmXHocu2GyY/C+vXr3cf2KSGL4dNFL8H6qxrDOhRk852d4ItPtsMX27fAtieuh3U9d4ENw5rD5+uWmH2LgWKlmCl2FobJc48TJFAJ4sTdyHMiJWIkt4S3k7U8De4XoBK/VGifvhEzDolz3S3TYN26de47OJSQxfDxa/fAul67wPp+TeHjNx50ZZ99tAo+uuUMWHfZD2FT9cmRPqVAsVLMFDsLw+S5xzGTySepjkkSh4irC8pFUifRcKmWUp4b9VWHFp1HQ/WVZ7qlRZteN6E4Y/x1T5xrJzxYsjjbP3jGCbJhQDP4ZMELruyzTcvho5vbwLoejWBD/4Ng+3szIv2KhcWh2LUcee5xZBJyMvF6pgQ1ZNDINhJXZ41pIKXJFFcMIWloX424JJYgErOPnaj1TBzG/34QCeHNOCxGeOmJQzMOl4+EfuPvh7Vr18Knn37qErIYPt20GqU5BC/X9oCP5z3kl62BTXd0gHXdv4/y/AA2TToLPt2yIdK3GChWipli13LkuceJS8RcCeonjBbGiSESysQaTxAkeMbyNLhf0N+KSWDJIjHbc3InEUlgxGoXh9U/Pyf5ItMboHiP4z45QFLQI+fw0rtU84QhWnYaCVePuxfWrFkDn3zyiUvIYtn28h2w7pL/go/u7uqV4Xjb3nzIzThUTvc6W2aPifQrBoqVYqbYW7YfDC07DPGWSOuOQ+HUS0d5ZQbVV5/n2lU2/lXwm9iSwu9i0//7JwlMxr1xmfZD8wm4GHyq9jwuQlw5QXXWmCGKjCsrppxWomusRLbaxWH1z4f/Hk7eTw746yTOlTfeBatXr3bf56eELJrN62HTlHaw7ordYdtb013Z9g0rsewCWHvx9xz0kODjRa9G++aEYqWYKfa84nQf6vVxCVuSONTfI5SgOZKUpYkTJK6Oy60xQxQRUx5ixbGTNRktRxJW/3xExYn7rBphiXPF6Dth1apV7othlJAlsWYxbKg+BTaM/hVsX7/ClW1b8hpsGPwzWNv1O7CubxPYOu+xaL+cUKwUM8XuXYLxj8DTr4MOxUs1vBwNysLQb1pTOy/xyyNO8fKI8Xi7kri6oJ81piB3PPkoqziEFiQOq28+Sp9xeo64HVauXOm+GEYJWSzb+PXqhbBxwmnw0QOXw8dbPnJlW56bCGsv+jZsGHgEbF34aqgfE/TPAMVKMVPsYTHSxaEfhM8sjkxWg0RxshAIELOtuLqg3BgzDiPxS+VrJU7eGeeSwbfC8uXL3cf06TsuDCVnEpu3bIX1GzfBmvUb3C/kLF21NqjbvvID2HxPd9j69lNeWxRnQ9fvwicPXxO0YbYia9ZvhDfnL4TX3v0QVq/bELt9jo1ipZgpdi2GJ87oSDmTa8aRyWogxSlKHhYgbltxdUG5MWYcRuKXSlnF0XIkYfXPR+kzTteBE2HZsmXuY/oSTlLN+g0bYdGyVfD03Dfg5vtnwK3TZsP0OX+Gdz5cEmq3afVS2Lrifdi6fiVsvftiWHtLe9i6cW2oDbFy9Vp46oU3oPrep6DnsCnwx+p7YeaLbzohdVsZH8VMsZMM9CujLEarTsOcOLJMUs4Zp2qfEuVhAeK2FVcXlBtjxmEkfqnEipOGlcxWuzis/kXhPx4nGfLNOCPgwutuhqVLl7pPG0tkkjJvzl8Ao+54DLoNmuT+2k94YAa8NG8+LFu52mzvWPgGrHz+Xli3fj2sRelWrF4HC5euhPkLl8L7i5fBgiUr4O0PFsN7C5bC0yjMBdfcBGdfdSOMuXN6ZCwZH8VMsTshSBJfFBaH1zXlnHGq9jneETtGGixA3Lbi6oJyQ5A4jMQPxikSfkgh2WHFyTfjjIBO/athyZIl7tPGEpmkBJWNu/sJOL3XSDilxzD4Ay7HTJ3uZotnXnnLzUBP4mtm+ty3YPrz82Dms3Pg4aefhXHTXoKb7p8JI6c8DFePvRuuGj0V7ntqLqxZtz60nWmzXoFTL8NYLxseKidkfBQzxR5Ik1Gc8s44njhFyyOSMLE+ttwQJI7aEsdMUIUlQxrWOOWC3pupvjLt+zj0eriDvjrd/o9jYfHixe7TxhKZpMx7C5fAAzPmOmGuGHk7zjy34uXSJOg0YEKEzgNudsvzB06FLkOnwqWDb4Frq+9xM8m9T74AL735HixftSayjZkvvgGnoZRteo6I1Mn4KGaKnX8wnmlF+9lzDIozzKS6z7kozlBMJkx2Sngk9P8/zZr0iJZFI8WysMaUWGMmYY2RCUOKisYknNFWYvWzElNjiZGENUY5IXG6j0j7Pg699n6wg8Q5v+8YWLRokfu0sUQmaRyr166F5197B159az68+rYPvfZ5GeWY994CWIuXaRuM/pq1OPtcc9Pd8Hucba66cWqkXsZHMVPsecWh/38QLWtDnDSsMSVWnySsMTKjBahJcdKoC3Eyzzi+OOf2GQULFy50nzaWyCS1ePfDRXAjzh6Tp80y6+NYjvdDJIguX7V6Ddx0z5NwcvehMPjWafD6ux9E2sj4KGaKnfaBf7WHaHUh7SeKI8ok9H+so2XFPicE4pjyZMBKXomZbHmgMRIo6/YsCax2EquPlZh5qQtx8s44Z185whQnjUkPznBJ3gkvyUbd8ShMfmgW3PPEczAD73WeeuE15HXHtFkvwW3TZsPN9z0Fw257GPqOu8v1Xbx0hRtnzbp18OZfPoSRtz/i7p9GTnkElq5YGdmehmKm2GkfpDxp4vBva2txipFHJ67GTLY80BgJlH17WgKrTRpWYmq0GGlYY5QEfzLaXy9mxjnz8qGwYMEC92ljiZWskhffeNclebs/3uQEIn6HnH3lGDiL6T3a3atwPXFOnxuh//h7nSyr16yFx5/7sxPqWix7aOaL7umbtT1CxkcxU+wsDssjxZHlDP/fHOhH6rU4eeXRiasxEysPNEYCadvT9Ra6zzdWnLwzzuk9B8OHH37oPm0skUkax4qVq1CAD+Dpua/D7Y8+40QaNOlBvNR6CHnQvR522zS4FWcYkoJmormvvwtvv78Ali5fCQsWL8MZ6nV4GmemhUuWm9uQyPgoZoqd5WAscSTuqaMvTtW+JziKlcdKRImZWHmgMRJI256uT0L3/WaI45N9xsG/vp3pr/UwOPXSgaY4Wfnz2/NhHs4eupw+vbxGlWmW4yWZVZ4Fipli935ssSBG6y709QkWJ1xH0PtctKQfcGRxLHmyCFS5F/YrATPxJMY2JWnJr+sjyFj2xP0tB2ai5qQuxMk24xTEOaX79UWL88zL86DzgAnQc/if4L6nXnCzx0tv/AXemr/AzSZ/+WARzjDvuJlm6mPPwt2Pz4E33n3fHCsvFDPFni4O49XTjENLTxyW5ze+LHz5lmWJiMSzqNobZ7UEdKJHwOROQoug++v6CCpeU4S8mImak1Rp/Dcu6bVsq9sn1UnSZ5wxKA5epjlxvEuZky8eULQ4r8x7D87rOw5OuXQ4nNpzJLTtM9a9d3PpsD/BVWOmQq8RU6Bj//GunO51qB3VPzr7ZTcjWWNmhWKm2PU+tuoyCvdzbKSc4d+do48qBT8hHLMM/uddMctAoCKp2BuTtQRIYkuoQB6rLoQ9rm5nSodISQOsxMxLarLXgDjJM05UnN92vdZ8qmYlK80WtzwwA4ZMnubuaRbirPLcn992Zb1G3o43/mPh95eNiHDmlWPgwutvgQET7ofbHp7thLPEobL3Fy6OlDMyPoqZYtf7mCYO/+5cIE4CTpAELBnyYCdtdkoWZx8ch4iMHW5nSUOEhGGsxMxLarLX+owjL9U8cU7o0s98A5QTdNnKVTDzpTeh7033QNdBt7oPY8559W2Y996H7g1QakPL+QuWwGvvfAAv4M3/bLyEm/XSPLektq++/b67fKNH0PT4mce2uHP6HPdIm97X0XUyPoqZYtf7mHvGScCSRWLJkIdowubjay+OVeeo9RknKs6vO/c1P3LDCXr1uHvg/H7j4fQrxkCn6yaiDG+FErgU1iMk0sKly2HGi2/ijPQAdBs82YmmRSFkfBQzxa73MXXG8T/LR58mt2TJg3ePFI8liySasPloEAdfSzF0n6Q6SfYZhy7XvJvlX3XsY37IkxO018g7oG3fajjjyrF4yTUW2l87AboPnQIDb50Gdz7+HDz67Kvw+POvwRt/WQDvfrg4wl9wJpr33kKYhcJNf+41eOSZV2HSQ7Ng1J3T4YZbHnKi0JjtkN6jp2K7ebB2vf2RHxkfxUyx631MnXH8T4+TOFVNTkrEkiUE3Q8l4O6HErBkykfDpVqorW6fVCfJPuMUxPlF+6vM7+NwgtIHMd96fxFeQj0PN0yaBr3H3AUXD/0TdL5hEpzX7+aAC/zkjzLRLc8Xbdv2He/KLxo0GXqNmgoj73zcfaJ66YrVEUEkMj6KmWLX+5jnHseSJTdNUcAYLFkktgx5aBAn1Fa3T6qTFGYc+upA4esDjCcOlRfEaX1Bb/dtyqQvjX2ErMNZYPmqtSjRYngCk/zOx1+AIbc9Av1ufgB6DL8D2vW/xZfE5gKk8/W3whVj7oYbbn0YJjwwG+6f8TLMm78IVq3Fy7JN9neAJDI+ipli1/uY5x6nqsnvSkAJZGDdF4UwZMpDgzj4WrbV7ZPqJIUZJ7s4rc6/HF7+8xsuGeUPYsgkrS/I2Chmil3vY74ZxxKiwE5NT0nh5ERMWSSGDHn42opT02iRSBb67eiWF45yaIEK4njrx2Bdy/b9YcDYye6LYpSU8ieY6hscF8VKMbdoT+/j8D56tOpCv1g6LlLO8G9ry+NYPP5X15Pwv9ZesS+J4hOUKRk4kf31tPuktAcR9NWJRPxxon3pTV6sJzmULBIpV4B9oOoXljj0S565xOk0BFqc2wtmv/By8Fed5alvUGwUI8VKMbfoNNTfvwJp4vBva8vjWDyGKBqWxCDLA4QkUsWJzCQKf5xo32+gOPaM40GXMPJH1504JFG7AdD8nJ5w/ZiJ7sOU1o15fYAeUVOMFGtzjFnuG1O7M04KfCluQcme8elbBH9m4v6xY/AMFoffrrzi0M5ZB6MekX3G8YgTx8nTfhD8/Ly+8H9n9ajXHI0xNqffhXazZnj/iNqdcVJgSawymimanGQmf6IMhJ/4Vt9iaBAHE6KYGcfRKT98yVd3hPePqHczTlw5Xqq5p29N498PspJckqdtEuUVx9rhekY5ZxxLjDTsZK5NwvtH1KsZJw5/1qEnb/q9H4kUw5IjqS4PDeJgQjTMODlmHPW1jNw0TWG/Nna5gx53ozz7ebNOHFKOJCwhstIgDiZEw4yTY8axZMiDKYSAxJHyNKFzxfV0/jxxgpnHv3SLWwYf8zE+7mMJkZV6IQ4ns0zoWPwpm17LftxXlyVB7Skh+C+qxe973gjHdh3jkotofdGYAC6rXUaVSHRM2r+avMfRxzyJuHZyDJInbum1p8fa9pKECn12DpFvsMaVO/ZGQZKkEw8f0h53h8q84PJTOCB1Iw7/RbU4tvMQl1TMqZffFCDLa4+xJWKNOc7tp7X/RKn3OPqYJxHXTo6RhO6ncTNRgjxWmYTFMOXJIg6/kSrfG7ICzUKeHa/tGUfDl3TWZV3tYF1+5cEaM5n6NOPo8vz4b7Iy6g1WE5SASRQHSRVHClOqOETmA1MmcXg8SoikGUdTt9IQlgx5sMZMptQZh9DHPY64dln7p+Fd1lH+xAhklQmkOERecSx2WHGKmXGsutrBkiEP1pjJlDrjEPq4xxHXLmv/NFicWIEspDxNaYnC+Gh5WBwut0TRfC3vcTQsTt3JY8mQB2vMZOrTPY4uz0vV/h5xAlllYXx5fIEiouh1QxSiLJdqfFAyHZgaEKfhHieZ+nSPk4bup2Fx4gSysAVCxEwkHyokPVzgmcnhfwyoYcapFTwBjr7gBjjy7L5w6Gm9HPSaysKSWFhjFjj6goE41jVi3Gugy8Apri44BznRxzyJuHZyjCR0P40WR8uj1wlTGianOHpGcrjgOLFFgpeTqv29N8IYOlhUpg+aPJiyTpeTAJP+2AGOOqcfHHba5XDwSd2ChDm63SBXT8ug/mSqvxyOxHUqb0EC5UElahJSUg3F0+zkKfAcrIPJnc9ycVPMqf27jA61oZhcGS5pnfbr4Etnwgo37tluf+8YfImrr1DHjpHHPStJY+g6Or+MrqttLLGkXKnyyBmHCGacEsWxgpXUhDjtrhmPSTgIblsF+G91KBGpvpCk+G/V/dCmtZdQlGSBEBkJJbFBKKkRLpfyEgcc3wH2/vkYmA2rYOIZx0CT1mc54eP6BxQhDh0fqrfEkcc8D1nG0W3qgzg7HeDJI5EC8ftEse8V7ajiaChZ7htzFSbeVTBxJcDSZcth2W3nwl5Hn+Lk8aTpBqfctZa0AVg2GU48mBL1bDfzsBBZiSSyIpTUCJd7cXhyPzf4bNjjiONhl6bXwSxYDtUn7gt7HH68izeuf0AWcXB/af+ItBnHOqZZyDqObpeGNUYuaIwEdjqgjcs3iRWH/kp55b4oFgmkL9OIiDQ1IA6h25cqzqnd+kPjw7tB9TKAp8eOhSWYjpc1PdL9FSdpmrbuAxNWrIJxN80CWDIWWjfaFX7U9CiXWJRoDIlEiUbLOGi2oDE1VE5y0CxH0OxBInB9U9xW4yO6YRwAs/scBf+9277w7e/1gKcx2rHNK+C/d93HycSXmYzcHr324vDuX2h8irfZ7y525dSepdkN/zgQ9PrsXsNrTxwjZ3TbJHTf3NAYCViiyP5cRr+voOUJZiF9nxORxjgIpWIFWqo4T94+An7Y9DwYtwTF6VIBXZ5GP246zsnh/rL3mYtT0Xho7irGQvOK5jBuKU4+t53nLpmI/Ye/jZdxj8EkbLri3j4u4Zq0roY5OEnNwRnCrQ9+07U5CWeHxocPh2e9Ocz9W3EvzXpneXR+DC+VCv/mXH+KP7uIfxhPi29188QZi3Hxv5erC+Pg9t3lpf+P4trr57/H2fQyuBVnrmf7+zGsfgzOQGECcXD7y+Al6OWLQzNygzgelZhroXVaiv46Fi2Ogz58KuVxnWtYHJJErlOApYpzUvsrQ+JUkCCYkhc32h2+v2sbuAklCcqdOP7rpRPdZRv9Ze71PMAsnAl+OXE5Xktd56T7IQq3DGcqEozWL8MspsvA7+96lbvEGn/SwdCoMXEG3EwzSb9j3Mw3ae79Tq4f4azXqDcODM9Dt+81wgPcHMZyjBWVSBeMkkLtgrPPD+D7J07GhF8FN596JO7PANzGKphw6lHQaE/cxp5nuNlqVm9ve+MxTBqX5Ni7eRsnTJw4tTrjKHS7NKwxckFjJFCFl2okD2PNQJKwOL48KA5TNnGsjUtKFUeX03U/zTi7SHH8BF0yrpUvy7iCLCwOJu0M/GtP9xffbdQLk3QOirYr7NxyHJbOhq6Y6F1nLoVxrXvADJwdWn7vOJylaB0FcGJG/3mCoSwo03GT8IYr+DcLurhtWuJ4l2o7/7//QHmOw8vN5biN/4FvXRKan4J/Sye2wXhPdDPmjEv+x0m1xxEnBOLQ7Nm47f2wFPen+65474Qz7iMTrq8TcXSbtCTNghYhL1UHnIbCnBpQtR8um6JMPrTO0PpOIfyfx1LfJ9phxKEDwEsSh2acsDgFSegqaMnY5qEyT5wKaI5ZTHLREmZ0c8lLyT1uCSUzJrV/WTeW1rvgvZOcrfzXXsL/wN2j/CfOcN9uPR5VwGq8VHRyNKd7rqdTxfHWedvx2+AYeZzv4jZ32e9nbmY54NftYbdmrdzMRdu86FvfcTMlHZ/aFEfX0flldF0xyPOfe4myBN8ZQkgO+X2jnfY/zSEFIrQ48vtEFW7whAPA9W6K86c8CsYLqFCWCLaVB1IezKRtB21pxw/w/lrQsmWXMfDkFJxx9jtfiCMug4KkxbKQOH5SL5mJgnjJSslJiXkMXtvR7zTjYNjOFwzXPQELY9O2OKm7z57jLsd27jrTbaMFJi21dVIGMXgJP6OrV8fijGv5HTfOzt9qVRAnZhskgxSHZ5z9uz3pHkGPP/lgX96CODQjx4lTDqzzZpXR+Qv+GFOZO59+DlEZ1dHrGHgcgpI4bknb2Xk/L19oyeW83WKgMQrQo+vCMiIOUWjswRsP7bS/Xg5x+DUTaufwhGFaoDindhvgnljRU7WZPXZ3T6yIX9L9yswewfr3e+GtPt7XtMa/0rT+n/+7O3TDPOdEp7JGezaDRidPwcSjS6FGLmG9RPQu077baDeP4ya6NvyP7m8oSen+g+Lgf0tmzsJ2s1wS7/yt/4AWZDf9c3H0gpl4N0KXizSb0P2Yu1QjkbCtd9lY+EfbIFH4Um1WL+9SjGYbfu/GEidpxikHVr5YZXQOg3Iq88+pyyFRngSNQxIkQbJIuDzL+GnwvkjqnTi6nTd+VJz7Rl8JTY49x12mEPQXmKHEkevchiBJ6FaCZhKSaPdmrd2NNrEL9tPQWPyYl6HEpSdddG9BSyqj+5wCXj/vBt+Dts3jURs9pmvnHjpgW1yn93ii2/BicZdoWM6PoalvvZ5xqIzqqQzXCV6XdXHwWEnovOHyLOOnIbcTjOv+4+8sE2nkT7XBDnNbqqcg08B2STvGr812jqg47fqNh4N/d4mThzjg+I4Ber0prpMYJBHNLEvhGXeJRRJRXTMch6DXGk5ShtYZXqc3XkPQ42MBbbvpsWe7JZfJcdwY3B77e4+l1TZEPxKGoDqSmO553OUhzqLH4H6RpPQ+V70Rh+pKgMdKIpI3frmLh5YlILcTjOv+I3ZY7nTQqIbFYbiMce0cUXEm9WsPR7a9Fg49vbeDXsdxGNaTGH1f9S9/+nizCYlE5UdhG4LaaQ497QrzDVOC3yANvTnpv0Ep6w87/QoHl1t1vB6M5bcL13nIWNx7Ue6f94SQZh8SLOl9nHJgnTerrHAOowLFlWt4rCRkbrlt+uUuHlqWgNxOMK77j9hhudNBI3/jekfpdTkfDljbpv5SGhan+6gH3DIL9DDh6PZDMAl7u1mEhKG//jQbkTBUH0eLC/2PtiRBH4ORqHr6eEzwERmjPK1elzNHtxvkBOKZx13i4ZLWkz6rVg6sc2aVufPt5wjnjc4fXa7hsZJIzC9aloDcTjCu+4/YYbnTQSMq3x+XB1ASI/Sa1w/EHefyOGgc2hlBEBiNza8FobaGONVXnBkRJAkSh2YmeSlH61Te4iJsQxj9WiQkbkCqOFLCAlLQpPoWXXAcg6Pbozht++ElK97r4KUgQdIcevrlOCN3cG3csSyVppj4Cvk4N6nMPdotURz5KDkOuiqRBHXu56usMUuEDoyUhpAHzTWichKFJalBcXQ7d2ANcfLMOAQJQpdt8tLOSUP1tSCOlkKKEQgSU6+FYVgcEoXkoSWtU3n3Ufe7Nu480XEtBV8CiSWJVcbiaHn43O6w4mhRiNCU5wgnbioHnhbGapMHNR4lefUVZ0QTntdVecuLboyUJSLHQGRy1wVmjCnwjEzHj/8IMWYiJBJOUo1OWg29c5+ENabEGjMP9j4lEz5mxpg7qjjdRz1YSG5OGF5X5d9EcXhGpuMXToL6J47Vp5zY+5RM+JgZYzbMOAZyjKx9krhobDa6YJySoNwYM4Xyzzh47GOo3P/0RCxZJDopNRUH/qEEKEZrn5IJHzMjJkscKQ93jCRzEirRzTZ5UONRMtfmjGO2yYOUI4kyihM341hJkg4e9wQsWSSWLBKdlBpbiKxQjCS+tV/xyGNmxrSjilOrM47g6A7+Q4Y/9IaDT7nEwQ8cGHpjlvEeRPTHfsOwv5DEoozixM046UkSxZJFYskisWSR6KTU2EJkhWKsJXFYmoYZJwxJQ8J4H64s/Jsz5BzYiz4V0Lyn92WzASP8L5tNhzNQnroQJ889jq6PYguRFUsWiU5KjS1EVjBndlhxDqSyUgiPR0lcFzOOJ85omPzydDjpyBPgR/sdBY16v4CGzIUe9EHRPc/0v2z2AvRq1srJRLOOE6f9UBxbicHUgDh57nF0fRRbiKxYsuSh4oDTSwDzp0RxQrnNZBHH7JiESnSvjOuKITweJXFdzDh0iXbACR3dZ8HCX1p7xv8kNH1FwPuENX0Ojj6hUFfilG/G4adif4jFkkWiRciLLURWMH/U/mZBHgM3hsbNJmgkT0GSQuB+Y5XAseUxuLGM8mAc5iAsk6j2lMTVl5/hhAhJUUZ4bDk+Xabt1W4aLEVV0r609t3/3R12O6S1uwdyb7bibCXHLxUrPgnPyO6Y7Y9/cZPwz2HlQZho1J7WRXnpWJdQebDGRDhfrDoBv9Eah5TEItiOhAQpWZwUeJxgvLRxMojTffSDiYlTKlZikgSNB7zqvleT9KU192WzvZpBk1+0LbswjBWfhGdk8/haYLsGcWyC7UhIkB1RnLqacZr84mr3Ix38j7+0JsVxXzbDeyASzevrX3IVTTg2wopPEppxMkDSMO4chMSxkrkewJdjVl1AfRZHlyukNG483YbHYTKIUxczjieO9+U5ejAg4W+cElRP7ai919eSIQ/h2AgrPkloxskACVN1MN6vsDiMq7cSsp6QKg7dh1He2dIQliyS0PFgSJAdUZy6mHHoXoVmEZLCe/xcgGYYhur53sbra8mQh3BshBWfpJgZJxCHyvzz6q3bCVmbVB50RgirTRw6/zRW7ksi+Um4iq/5PY5MMou0PlxmvflpQfXhBwKWDHkIx0bI+Cz4+AQipJAkjk7avFjJLLH6ZCFrf51/Giv3JZH8JNx/+HpWwwdWr+vyFHSgqeNkECfPjNMSky8JK1nD9fa42YmOXyoyPovqy//gllYiWVQdfGaAS0j/EojqKkrFT/A4dCxZydpf508EnX8aq4+rKFUcXa6oCXHoLyrJ45Z4WULX9LFLTKLuox+KX16By1FqGapPGT91aYxf6tLaD7VkcaQUFrqNTmpThjyI8Sw4wYshS3+dPxF0/mmsPmal7uQveTq3pvQkdjrwdIcWx7sMCF9KuPWDFepAW3+BGflXN6DruESscSQ8Tlx5GrpfTSP3rVKJYELJJaA+El2vqWp2VqQP49UlU3EQbSMBK2aJfEDAcYm6UI7J3I7L3SxtzErdyV9yojOyLolyi5OE+RenRIJxMmzLItSP1msYnbyh7Vuk9Vf1GikOl/F6FnHSMGOOg+MSZZF8lYjcC5HWxqzUnfwlBxAJhPvF0CCOwPqLmgr1yw4nLWO1SSLaP5mqZmdjOz/J/TJel3XFIrdlEndsXXmCODLn/NcBRrsQZmVCp1AQGalzcYx2IahNAnHjyG0kEZxY6+RmgvrlI5z4dpskwv2TSRNHtrXgtnFYfUKI4xrpg3XeeaA88vNM5baVhwGGAw73H13JA/vlsYMiSXVMqeLoRNSkJbOst5BJYhIzjtxGEmniyCSxUfEo0hI/jfT+VkwFalqcNOLG4XJ+AKLPHxM9X7rey1NJZnFcY1mnyxOoc3FSCCeJAbWhtmI7cduyqG1xrDZJpPe3YiqQJg6/joPaJGH1icPqw+J45yLbOQy38fJU0iAOEk0UBbWhtmI7cduyqGtx6AY9ibT+dkwFZKLqfcqS+JzsxRI3TlBOx4D2g/bPOH9SLCbczstTSSZxIo+f/XIqo7qgXwwFcbyPP3gBUX8KksblMfztUOAkDS39HUmCd9Drb7QJEqBI/HHkdmK3pXD70CBOIjLZiyFunKCcjgHtB+2fcf5kvSTcTovjS5BMYWPhwcIb0OUMB1Z1kL8T/tLbsejSO/h0ErMmVnb0QbXquJ63zwmv28u6VChBa4jKQ/A+wignkuqyIo9LMVhjSijGJNLa63qNPm/BuSNSzx/msMtN73VAbYhTeSBu2MAOFOvEDtKBl+vFIMfkk5lUx/UyJtmOiSvXVNBBN05ouQglD21LbC9LYqUhj0sxWGNKpAQWaW1lvYV1Tgh3/jL84as34kSCo+DlDtDO4gGp+uk5Dmun80Bj8ms+mUl1XB/E48PtmLhyjT6R5SaUPN8wcSg/ZL0FnwcZkzsv4twmUWfiVBygYFEkQZD+zsoD5O94sfC4hDxwcXVxB5bbMXHlGn0iyw0do2D9ayqOVc5/WPV4Gium4NxkOH90HxiRxxZFE5ZGCxJXHtQnCYPoQPWO6/q8yIOTdOD0gc0qRhp6fySUAFa5RCeMRdDeEOfrBgvDWG0kfE53OuScAD7HWeCHKCyPE8gWRROWRgsSVx4Qd4nmwzvGBEngY+1MHmgb/Jq3kVTH9Rwf18ch+1no/XH7JE6sVS+RbeMI2n/NxdHSZBFHCiOhc5Pl/NWZOJyAsfgnOe5Ey/qiwG3oJOd1q47rrfgsrJglaX2seolsG0fQ3hCnokT0tvJijVk0KIqk8tC2djsBC7YTtmWyCKcJjjFhi6IJS6MFiSsP8BMzDutky50NBVwM9UyctHqNbm8RtP86i4P5oKXJKo6URovDuRZHaF/4OMukzwSKkIac0jy8xNQEQShkoMUS2vlDzg2obIYHWyHrrTZp8aXVh2JJQfeNQ/fRMQQCEc0w0Q7B/fjpuY7KQ88LXtcLKDaJ1aYkwsIlQsdSYR3zBnEMKg7GAyNJiS2tPhRLClZ/C6uPjMNB0hBp4ujE1ci2NUGNbw/3Pyt0HBXWMa8TcSInWBEkgo+V3GlIWSRWW01EHIFuS2Om1edB9yfkSQzgE03oOn1MVUKSOCF5rKS1ymqK2txWGjIWn1oTh6hrcYisiZlWbwlBcJ+0+jxY44SkkLA4jK7nY8pJgInB0mh5dDkTJFQdUCcxCGEYUxy6eTcFiUMJEkcp4ugEsJIpC1kTM61eI9tbfXR9HvRYhD4eASyMJtIWx/ATQ0qRlYrDSsMaUyIT16pPQ/YvGSUNUWvihKWpG3GspCSsthp56aVnk7Qxrfo8yLEYfTwcLAkSPF0SZY6gPdUVkrLqsPOD1zLxZJkst2TIgx633ARxlgMlDVHn4piSWAQnXJz4vIiDkZaYGi1O3LiEO3Ep9XL7aZjxRI4HwnIgLE6sQCIWFw+KI+WRdRrX15AhD9a45YT3L5ZDc/BTOn4K69x+PcXhg3m+W6Ylpqy32gTjygOHBCcupV6Pn4Tetrd9fTwQIYa3HWwrljoWSgpXf7gvDS5dO0xsXo9bahGKhbenlzoeucxCZF81Wo4ksoqz04FnwM4HnRkLfW/GvQlISz+xeeoKJXsC3L5YgsBJCAsniEfloRdA5U/bh5fGwZaEDoiJGFsQbNePwax3MYqDb2Fus4BLLBUrr1Od24478bhNSrjD2jncOnNYyvJwv88R7aHq8PaFdarH1+XAjWcsOQ6zHKnCdUYfY4LL+XzItvK8cH0IPD9cH2rDueXObfh8ODKJQx/GVOJIMdLQImjktaNFIcnEzkiCncWDRbJIgoPiJZqFeWB8vHpvfHmAicJ2vTZmvYtRSGJhbFcSF2ewHT4G9JrKMGm81ygAQcmfBiUqgeI4eF3W1RQ6BlVO+8MysBASKQmfB0sc2SZA5k8s4ePuoEs1963MBL6J4oTrvfHlCSAK2/XamPUyviKJxsnl/nZ4G/Sayg7rUJAmizicqCyNlqemkTFosI5mQJJHwvsaQPsvjkPQxh2n5HNt/RFL7UPiJOHuW2pYnDRqUxy73hu/MJZHsN2M9cXixSBOWqgOt8PHgF7TtrU4VkJa1JU4TMz2SRxGCyQFkcdBlkfPpzyWiJKGiW1PWLJIvBt+lMYXJ5LUShIL3ScvNS1OGjx+YSyPYLtGHZNUl51wHAFYFySNW6fE6hCiLOLo8nIjtlF5JMaNyLqdjugQkscSKHwc8JiJMnmPWDiWBQmsMoefd9Y5Sb1U46diUgC+hAoSW4mi4XZxyMsyi5LECeQJH7g88PiRgxdTXn7CcQRgnYRlqTq8oyOQh5MwjpTEDV7XMLxtHQOJI7EkYkm84yCEwjISh5Hn1TumMZdlQc5Z4lwA/x+5FCoGjbgWrQAAAABJRU5ErkJggg==" alt="" />
 

4.2创建hdfs文件输入路径

hdfs dfs -mkdir -p /data/tq/input

4.3上传测试文件到创建的hdfs目录下

[root@node01 ~]# cat tq.txt
-- :: 34c
-- :: 38c
-- :: 36c
-- :: 32c
-- :: 37c
-- :: 23c
-- :: 41c
-- :: 27c
-- :: 45c
-- :: 46c
-- :: 47c [root@node01 ~]# hdfs dfs -put tq.txt /data/tq/input
 

4.4服务端执行程序

[root@node01 ~]# hadoop jar Myweather.jar com.hadoop.mr.weather.MyWeather
-- ::, INFO client.ConfiguredRMFailoverProxyProvider: Failing over to rm2
-- ::, WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
-- ::, INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /user/root/.staging/job_1546092355023_0004
-- ::, INFO input.FileInputFormat: Total input files to process :
-- ::, INFO mapreduce.JobSubmitter: number of splits:
-- ::, INFO Configuration.deprecation: yarn.resourcemanager.zk-address is deprecated. Instead, use hadoop.zk.address
-- ::, INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled
-- ::, INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1546092355023_0004
-- ::, INFO mapreduce.JobSubmitter: Executing with tokens: []
-- ::, INFO conf.Configuration: resource-types.xml not found
-- ::, INFO resource.ResourceUtils: Unable to find 'resource-types.xml'.
-- ::, INFO impl.YarnClientImpl: Submitted application application_1546092355023_0004
-- ::, INFO mapreduce.Job: The url to track the job: http://node04:8088/proxy/application_1546092355023_0004/
-- ::, INFO mapreduce.Job: Running job: job_1546092355023_0004
-- ::, INFO mapreduce.Job: Job job_1546092355023_0004 running in uber mode : false
-- ::, INFO mapreduce.Job: map % reduce %
-- ::, INFO mapreduce.Job: map % reduce %
-- ::, INFO mapreduce.Job: map % reduce %
-- ::, INFO mapreduce.Job: map % reduce %
-- ::, INFO mapreduce.Job: Job job_1546092355023_0004 completed successfully
-- ::, INFO mapreduce.Job: Counters:
File System Counters
FILE: Number of bytes read=
FILE: Number of bytes written=
FILE: Number of read operations=
FILE: Number of large read operations=
FILE: Number of write operations=
HDFS: Number of bytes read=
HDFS: Number of bytes written=
HDFS: Number of read operations=
HDFS: Number of large read operations=
HDFS: Number of write operations=
Job Counters
Launched map tasks=
Launched reduce tasks=
Rack-local map tasks=
Total time spent by all maps in occupied slots (ms)=
Total time spent by all reduces in occupied slots (ms)=
Total time spent by all map tasks (ms)=
Total time spent by all reduce tasks (ms)=
Total vcore-milliseconds taken by all map tasks=
Total vcore-milliseconds taken by all reduce tasks=
Total megabyte-milliseconds taken by all map tasks=
Total megabyte-milliseconds taken by all reduce tasks=
Map-Reduce Framework
Map input records=
Map output records=
Map output bytes=
Map output materialized bytes=
Input split bytes=
Combine input records=
Combine output records=
Reduce input groups=
Reduce shuffle bytes=
Reduce input records=
Reduce output records=
Spilled Records=
Shuffled Maps =
Failed Shuffles=
Merged Map outputs=
GC time elapsed (ms)=
CPU time spent (ms)=
Physical memory (bytes) snapshot=
Virtual memory (bytes) snapshot=
Total committed heap usage (bytes)=
Peak Map Physical memory (bytes)=
Peak Map Virtual memory (bytes)=
Peak Reduce Physical memory (bytes)=
Peak Reduce Virtual memory (bytes)=
Shuffle Errors
BAD_ID=
CONNECTION=
IO_ERROR=
WRONG_LENGTH=
WRONG_MAP=
WRONG_REDUCE=
File Input Format Counters
Bytes Read=
File Output Format Counters
Bytes Written=
 

4.5将hdfs上生成的输出文件 拉取到本地

[root@node01 ~]# hdfs dfs -get  /data/tq/output/* ./test
 

4.6查看输出文件

[root@node01 test]# ls
part-r- part-r- _SUCCESS
[root@node01 test]# cat part-r-
[root@node01 test]# cat part-r-
--:
--:
--:
--:
--:
--:
--:
--:
--:
 
0分区是空的  1分区有程序定义的k v输出。这就发生了数据倾斜,可能上面的Tpartitioner类的代码对数据分布规则定义的不恰当导致的。

5.Combiner说明

由于数据量比较少,这边没有对combiner类做扩展

每一个map都可能会产生大量的本地输出Combiner的作用就是对map端的输出先做一次合并,以减少在map和reduce节点之间的数据传输量,以提高网络IO性能,是MapReduce的一种优化手段之一,其具体的作用如下所述。

(1)Combiner最基本是实现本地key的聚合,对map输出的key排序,value进行迭代。如下所示:

  map: (K1, V1) → list(K2, V2)
  combine: (K2, list(V2)) → list(K2, V2)
  reduce: (K2, list(V2)) → list(K3, V3)

(2)Combiner还有本地reduce功能(其本质上就是一个reduce),例如Hadoop自带的wordcount的例子和找出value的最大值的程序,combiner和reduce完全一致,如下所示:

  map: (K1, V1) → list(K2, V2)
  combine: (K2, list(V2)) → list(K3, V3)
  reduce: (K3, list(V3)) → list(K4, V4)

如果在wordcount中不用combiner,那么所有的结果都是reduce完成,效率会相对低下。使用combiner之后,先完成的map会在本地聚合,提升速度。对于hadoop自带的wordcount的例子,value就是一个叠加的数字,所以map一结束就可以进行reduce的value叠加,而不必要等到所有的map结束再去进行reduce的value叠加。

最新文章

  1. bash 源码分析
  2. Nice Validator(Form验证)及Juery zTree控件
  3. java 获取当月第一天和最后一天 获取前一个月第一天和最后一天
  4. jQuery_04之第三方、自定义
  5. 15款免费的响应式 WordPress 主题下载
  6. Hello 2016
  7. Spring MVC中使用Interceptor拦截器
  8. 2年后的Delphi XE6
  9. 在其他页面调用 Discuz 7.2 BBS 论坛会员登录信息
  10. PO、BO、VO、DTO、POJO、DAO的区别
  11. typeof和GetType的区别
  12. C# DllImport的用法
  13. JavaScript 中的日期和时间
  14. 剑指Offer——银行考试
  15. C#使用GUID
  16. 命令提示符出现-bash-4.1$如何解决
  17. C# .NET newtonsoft.json 多版本冲突解决
  18. 每天一条linux命令
  19. MySQL 8.0 —— 数据字典
  20. C++ 运算符的重载

热门文章

  1. oracle语法
  2. 【BZOJ4889】[Tjoi2017]不勤劳的图书管理员 分块+树状数组
  3. 添加网站标题logo
  4. 【树形DP】【P1364】医院放置
  5. 创建JavaScript的哈希表Hashtable
  6. Linux系统通过AWS命令行上传文件至S3
  7. 浏览器报错:unexpected end of input 解决方法
  8. 51Nod 1092 回文字符串 | 最长公共子序列变形
  9. 删除windows上特定目录下以*.rar后缀名的python脚本
  10. hibernate单列的多值查询