package com.darrenchan.spark

import org.apache.spark.{SparkConf, SparkContext}

object SparkCoreApp2 {

  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf().setMaster("local[2]").setAppName("WordCountApp")

    val sc = new SparkContext(sparkConf)

    //业务逻辑

    val counts = sc.textFile("D:\\hello.txt").

      flatMap(_.split(" ")).

      map((_, 1)).

      reduceByKey(_+_)

    println(counts.collect().mkString("\n"))

    sc.stop()

  }

}

package com.darrenchan.spark.javaapi;

import org.apache.spark.SparkConf;

import org.apache.spark.SparkContext;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.sql.SparkSession;

import scala.Tuple2;

import java.util.Arrays;

public class WordCountApp2 {

    public static void main(String[] args) {

        SparkConf sparkConf = new SparkConf().setMaster("local[2]").setAppName("WordCountApp");

        JavaSparkContext sc = new JavaSparkContext(sparkConf);

        //业务逻辑

        JavaPairRDD<String, Integer> counts =

                sc.textFile("D:\\hello.txt").

                        flatMap(line -> Arrays.asList(line.split(" ")).iterator()).

                        mapToPair(word -> new Tuple2<>(word, 1)).

                        reduceByKey((a, b) -> a + b);

        System.out.println(counts.collect());

        sc.stop();

    }

}

from pyspark import SparkConf, SparkContext

def main():

    # 创建SparkConf，设置Spark相关的参数信息

    conf = SparkConf().setMaster("local[2]").setAppName("spark_app")

    # 创建SparkContext

    sc = SparkContext(conf=conf)

    # 业务逻辑开发

    counts = sc.textFile("D:\\hello.txt").\

        flatMap(lambda line: line.split(" ")).\

        map(lambda word: (word, 1)).\

        reduceByKey(lambda a, b: a + b)

    print(counts.collect())

    sc.stop()

if __name__ == '__main__':

    main()