hadoop项目之求出每年二月的最高气温（Combiner优化）

一、项目结构

一、java实现随机生成日期和气温

package com.shujia.weather;

import java.io.BufferedWriter;

import java.io.FileWriter;

import java.io.IOException;

import java.text.DateFormat;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Date;

public class RandomWeather {

    public static void main(String[] args) throws ParseException, IOException {

        //创建日期格式

        DateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

        long start = sdf.parse("2000-01-01 00:00:00").getTime();

        long end = sdf.parse("2022-12-31 00:00:00").getTime();

        long difference=end - start;

        BufferedWriter bw = new BufferedWriter(new FileWriter("F:\\software\\IdeaProjects\\bigdata19-project\\biddata19-mapreduce\\src\\data\\weather.txt"));

        for (int i=0;i<10000;i++){

            //随机生成时间

            Date date = new Date(start + (long) (Math.random() * difference));

            //随机生成一个温度

            int temperature = -20+(int) (Math.random()*60);

            //打印

//            System.out.println(date+"\t"+temperature);

            bw.write(sdf.format(date)+"\t"+temperature);//将结果写入文件

            bw.newLine();

            bw.flush();

        }

        bw.close();

    }

}

二、将这个weather.txt文件上传到虚拟机后再上传到hadoop

1、通过xftp上传文件

2、通过命令上传到hadoop

hadoop fs -put weather.txt /路径

三、项目实现

package com.shujia.weather;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

class WeatherMapper extends Mapper<LongWritable,Text,Text,LongWritable>{

    /*

    2022-06-12 02:40:26	21

    2002-01-03 03:49:27	-13

    2001-04-21 19:19:22	-16

    2005-01-18 01:52:15	10

    求出每年二月份的最高气温

     */

    @Override

    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, LongWritable>.Context context) throws IOException, InterruptedException {

        String line = value.toString();

        String[] str = line.split("\t");

        String temperature = str[1];

        String[] strings = str[0].split("-");

        String Month = strings[1];

        if ("02".equals(Month)){

            context.write(new Text(strings[0]+"-"+Month),new LongWritable(Long.parseLong(temperature)));

        }

    }

}

class WeatherReducer extends Reducer<Text,LongWritable,Text,LongWritable>{

    @Override

    protected void reduce(Text key, Iterable<LongWritable> values, Reducer<Text, LongWritable, Text, LongWritable>.Context context) throws IOException, InterruptedException {

        long max=0L;

        for (LongWritable value : values) {

            long l = value.get();

            if (l>max){

                max=l;

            }

        }

        context.write(key,new LongWritable(max));

    }

}

public class WeatherDemo {

    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);

        job.setCombinerClass(WeatherReducer.class);//Combiner优化

        job.setJarByClass(WeatherDemo.class);

        job.setMapperClass(WeatherMapper.class);

        job.setReducerClass(WeatherReducer.class);

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(LongWritable.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(LongWritable.class);

        FileInputFormat.setInputPaths(job,new Path(args[0]));

        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        job.waitForCompletion(true);

    }

}

优化前

优化后

减少了reduce 从map拉取数据的过程，提高计算效率。

hadoop 的计算特点：将计算任务向数据靠拢，而不是将数据向计算靠拢。

注意：将reduce端的聚合操作，放到map 进行执行。适合求和，计数，等一些等幂操作。不适合求平均值，次幂等类似操作

巴特西

hadoop项目之求出每年二月的最高气温（Combiner优化）

hadoop项目之求出每年二月的最高气温（Combiner优化）

一、项目结构

一、java实现随机生成日期和气温

二、将这个weather.txt文件上传到虚拟机后再上传到hadoop

三、项目实现

最新文章

热门文章