一、spark-streaming-kafka-0-8_2.11-2.0.2.jar

1、pom.xml



  1. <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11 -->
    2. 

  2. <dependency>
    3. 

  3. <groupId>org.apache.spark</groupId>
    4. 

  4. <artifactId>spark-core_2.11</artifactId>
    5. 

  5. <version>2.0.2</version>
    6. 

  6. <scope>runtime</scope>
    7. 

  7. </dependency>
    8. 

  8. <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming_2.11 -->
    9. 

  9. <dependency>
    10. 

  10. <groupId>org.apache.spark</groupId>
    11. 

  11. <artifactId>spark-streaming_2.11</artifactId>
    12. 

  12. <version>2.0.2</version>
    13. 

  13. <scope>runtime</scope>
    14. 

  14. </dependency>
    15. 

  15. <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-8_2.11 -->
    16. 

  16. <dependency>
    17. 

  17. <groupId>org.apache.spark</groupId>
    18. 

  18. <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
    19. 

  19. <version>2.0.2</version>
    20. 

  20. <scope>runtime</scope>
    21. 

  21. </dependency>


2、Kafka Consumer类




  1. package com.spark.main;
    2. 

  2. import java.util.Arrays;
    3. 

  3. import java.util.HashMap;
    4. 

  4. import java.util.HashSet;
    5. 

  5. import java.util.Map;
    6. 

  6. import java.util.Set;
    7. 

  7. import org.apache.spark.SparkConf;
    8. 

  8. import org.apache.spark.api.java.JavaRDD;
    9. 

  9. import org.apache.spark.api.java.function.Function;
    10. 

  10. import org.apache.spark.api.java.function.VoidFunction;
    11. 

  11. import org.apache.spark.streaming.Durations;
    12. 

  12. import org.apache.spark.streaming.api.java.JavaDStream;
    13. 

  13. import org.apache.spark.streaming.api.java.JavaPairInputDStream;
    14. 

  14. import org.apache.spark.streaming.api.java.JavaStreamingContext;
    15. 

  15. import org.apache.spark.streaming.kafka.KafkaUtils;
    16. 

  16. import kafka.serializer.StringDecoder;
    17. 

  17. import scala.Tuple2;
    18. 

  18. public class KafkaConsumer{
    19. 

  19. public static void main(String[] args) throws InterruptedException{
    20. 

  20. /**
    21. 

  21. * SparkConf sparkConf = new SparkConf().setAppName("KafkaConsumer").setMaster("local[2]");
    22. 

  22. * setMaster("local[2]")，至少要指定两个线程，一条用于用于接收消息，一条线程用于处理消息
    23. 

  23. * Durations.seconds(2)每两秒读取一次kafka
    24. 

  24. */
    25. 

  25. SparkConf sparkConf = new SparkConf().setAppName("KafkaConsumer").setMaster("local[2]");
    26. 

  26. JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, Durations.milliseconds(500));
    27. 

  27. jssc.checkpoint("hdfs://192.168.168.200:9000/checkpoint/KafkaConsumer");
    28. 

  28. /**
    29. 

  29. * 配置连接kafka的相关参数
    30. 

  30. */
    31. 

  31. Set<String> topicsSet = new HashSet<String>(Arrays.asList("TestTopic"));
    32. 

  32. Map<String, String> kafkaParams = new HashMap<String, String>();
    33. 

  33. kafkaParams.put("metadata.broker.list", "192.168.168.200:9092");
    34. 

  34. kafkaParams.put("auto.offset.reset", "smallest");//smallest：从最初开始；largest ：从最新开始
    35. 

  35. kafkaParams.put("fetch.message.max.bytes", "524288");
    36. 

  36. JavaPairInputDStream<String, String> messages = KafkaUtils.createDirectStream(jssc, String.class, String.class,
    37. 

  37. StringDecoder.class, StringDecoder.class, kafkaParams, topicsSet);
    38. 

  38. /**
    39. 

  39. * _2()获取第二个对象的值
    40. 

  40. */
    41. 

  41. JavaDStream<String> lines = messages.map(new Function<Tuple2<String, String>, String>() {
    42. 

  42. public String call(Tuple2<String, String> tuple2) {
    43. 

  43. return tuple2._2();
    44. 

  44. }
    45. 

  45. });
    46. 

  46. lines.foreachRDD(new VoidFunction<JavaRDD<String>>() {
    47. 

  47. public void call(JavaRDD<String> rdd) throws Exception {
    48. 

  48. rdd.foreach(new VoidFunction<String>() {
    49. 

  49. public void call(String s) throws Exception {
    50. 

  50. System.out.println(s);
    51. 

  51. }
    52. 

  52. });
    53. 

  53. }
    54. 

  54. });
    55. 

  55. // Start the computation
    56. 

  56. jssc.start();
    57. 

  57. jssc.awaitTermination();
    58. 

  58. }
    59. 

  59. }


二、spark-streaming-kafka-0-10_2.11-2.0.2.jar


1、pom.xml




  1. <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11 -->
    2. 

  2. <dependency>
    3. 

  3. <groupId>org.apache.spark</groupId>
    4. 

  4. <artifactId>spark-core_2.11</artifactId>
    5. 

  5. <version>2.0.2</version>
    6. 

  6. <scope>runtime</scope>
    7. 

  7. </dependency>
    8. 

  8. <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming_2.11 -->
    9. 

  9. <dependency>
    10. 

  10. <groupId>org.apache.spark</groupId>
    11. 

  11. <artifactId>spark-streaming_2.11</artifactId>
    12. 

  12. <version>2.0.2</version>
    13. 

  13. <scope>runtime</scope>
    14. 

  14. </dependency>
    15. 

  15. <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-10_2.11 -->
    16. 

  16. <dependency>
    17. 

  17. <groupId>org.apache.spark</groupId>
    18. 

  18. <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
    19. 

  19. <version>2.0.2</version>
    20. 

  20. <scope>runtime</scope>
    21. 

  21. </dependency>


2、Kafka Consumer类




  1. package com.spark.main;
    2. 

  2. import java.util.Arrays;
    3. 

  3. import java.util.HashMap;
    4. 

  4. import java.util.HashSet;
    5. 

  5. import java.util.Map;
    6. 

  6. import java.util.Set;
    7. 

  7. import org.apache.kafka.clients.consumer.ConsumerRecord;
    8. 

  8. import org.apache.kafka.common.serialization.StringDeserializer;
    9. 

  9. import org.apache.spark.SparkConf;
    10. 

  10. import org.apache.spark.api.java.JavaRDD;
    11. 

  11. import org.apache.spark.api.java.function.Function;
    12. 

  12. import org.apache.spark.api.java.function.VoidFunction;
    13. 

  13. import org.apache.spark.streaming.Durations;
    14. 

  14. import org.apache.spark.streaming.api.java.JavaDStream;
    15. 

  15. import org.apache.spark.streaming.api.java.JavaInputDStream;
    16. 

  16. import org.apache.spark.streaming.api.java.JavaPairInputDStream;
    17. 

  17. import org.apache.spark.streaming.api.java.JavaStreamingContext;
    18. 

  18. import org.apache.spark.streaming.kafka010.ConsumerStrategies;
    19. 

  19. import org.apache.spark.streaming.kafka010.KafkaUtils;
    20. 

  20. import org.apache.spark.streaming.kafka010.LocationStrategies;
    21. 

  21. import kafka.serializer.StringDecoder;
    22. 

  22. import scala.Tuple2;
    23. 

  23. public class Kafka10Consumer{
    24. 

  24. public static void main(String[] args) throws InterruptedException{
    25. 

  25. /**
    26. 

  26. * SparkConf sparkConf = new SparkConf().setAppName("KafkaConsumer").setMaster("local[2]");
    27. 

  27. * setMaster("local[2]")，至少要指定两个线程，一条用于用于接收消息，一条线程用于处理消息
    28. 

  28. * Durations.seconds(2)每两秒读取一次kafka
    29. 

  29. */
    30. 

  30. SparkConf sparkConf = new SparkConf().setAppName("Kafka10Consumer").setMaster("local[2]");
    31. 

  31. JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, Durations.milliseconds(500));
    32. 

  32. jssc.checkpoint("hdfs://192.168.168.200:9000/checkpoint/Kafka10Consumer");
    33. 

  33. /**
    34. 

  34. * 配置连接kafka的相关参数
    35. 

  35. */
    36. 

  36. Set<String> topicsSet = new HashSet<String>(Arrays.asList("TestTopic"));
    37. 

  37. Map<String, Object> kafkaParams = new HashMap<String, Object>();
    38. 

  38. kafkaParams.put("bootstrap.servers", "192.168.168.200:9092");
    39. 

  39. kafkaParams.put("key.deserializer", StringDeserializer.class);
    40. 

  40. kafkaParams.put("value.deserializer", StringDeserializer.class);
    41. 

  41. kafkaParams.put("group.id", "Kafka10Consumer");
    42. 

  42. kafkaParams.put("auto.offset.reset", "earliest");//earliest : 从最早开始；latest ：从最新开始
    43. 

  43. kafkaParams.put("enable.auto.commit", false);
    44. 

  44. //通过KafkaUtils.createDirectStream(...)获得kafka数据，kafka相关参数由kafkaParams指定
    45. 

  45. JavaInputDStream<ConsumerRecord<Object,Object>> messages = KafkaUtils.createDirectStream(
    46. 

  46. jssc,
    47. 

  47. LocationStrategies.PreferConsistent(),
    48. 

  48. ConsumerStrategies.Subscribe(topicsSet, kafkaParams)
    49. 

  49. );
    50. 

  50. /**
    51. 

  51. * _2()获取第二个对象的值
    52. 

  52. */
    53. 

  53. JavaDStream<String> lines = messages.map(new Function<ConsumerRecord<Object,Object>, String>() {
    54. 

  54. @Override
    55. 

  55. public String call(ConsumerRecord<Object, Object> consumerRecord) throws Exception {
    56. 

  56. // TODO Auto-generated method stub
    57. 

  57. return consumerRecord.value().toString();
    58. 

  58. }
    59. 

  59. });
    60. 

  60. lines.foreachRDD(new VoidFunction<JavaRDD<String>>() {
    61. 

  61. public void call(JavaRDD<String> rdd) throws Exception {
    62. 

  62. rdd.foreach(new VoidFunction<String>() {
    63. 

  63. public void call(String s) throws Exception {
    64. 

  64. System.out.println(s);
    65. 

  65. }
    66. 

  66. });
    67. 

  67. }
    68. 

  68. });
    69. 

  69. // Start the computation
    70. 

  70. jssc.start();
    71. 

  71. jssc.awaitTermination();
    72. 

  72. }
    73. 

  73. }


												


																

最新文章
	

    
						
  1. MATLAB 物体识别算法说明：vision.ForegroundDetector， vision.BlobAnalysis
		
    2. 
						
  2. groovy-输入输出
		
    3. 
						
  3. tcp的三次握手及四次挥手（连接与中断流程）
		
    4. 
						
  4. C# OpenFileDialog和PictrueBox
		
    5. 
						
  5. PHP面向对象(OOP):.static和const关键字的使用（self::）
		
    6. 
						
  6. 关于js对象引用的小例子
		
    7. 
						
  7. .h 与 .hpp 文件
		
    8. 
						
  8. poj2635 同余定理 + 素数筛法
		
    9. 
						
  9. python安装第三方库
		
    10. 
						
  10. 1013. Battle Over Cities (25)
		
    11. 
						
  11. Azure系列2.1.11 —— CloudBlobContainer
		
    12. 
						
  12. Jsp基本语法 第二章
		
    13. 
						
  13. 小程序视图层（xx.xml)和逻辑层(xx.js)
		
    14. 
						
  14. Android图片加载框架最全解析（八），带你全面了解Glide 4的用法
		
    15. 
						
  15. linux 2.6.xx自动加载kvm模块
		
    16. 
						
  16. Django 1.10文档中文版Part1
		
    17. 
						
  17. 解析IE, FireFox, Opera 浏览器支持Alpha透明的方法
		
    18. 
						
  18. 为什么说git比svn好
		
    19. 
						
  19. VI编辑器、ipython、jupyter及进程
		
    20. 
						
  20. rmdir  命令
		
    21. 
		

	


热门文章
	

    
									
  1. c++下基于windows socket的多线程服务器(基于TCP协议)
			
    2. 
						
  2. python --flask框架的搭建--(flask,python2,python3都可以运行)
			
    3. 
						
  3. ubuntu discuz 该函数需要 PHP 支持 XML。请联系空间商，确定开启了此项功能
			
    4. 
						
  4. mAP的计算
			
    5. 
						
  5. todolist待办事项
			
    6. 
						
  6. unity 实现调用Windows窗口/对话框交互
			
    7. 
						
  7. CodeForces - 616F：Expensive Strings （后缀自动机）
			
    8. 
						
  8. lesson2-cnn-fastai
			
    9. 
						
  9. ajax遍历数组对象
			
    10. 
						
  10. Blender 3D 打印插件Print Toolbox
			
    11.