import org.apache.spark.HashPartitioner

import java.io.PrintWriter

import java.io.File

val links = sc.parallelize(List(("A",List("B","C")),("B",List("A","C")),("C",List("A","B","D")),("D",List("C")))).partitionBy(new HashPartitioner(100)).persist()

var ranks=links.mapValues(v=>1.0)

for (i <- 0 until 10) {

val contributions=links.join(ranks).flatMap {

case (pageId,(links,rank)) => links.map(dest=>(dest,rank/links.size))

}

ranks=contributions.reduceByKey((x,y)=>x+y).mapValues(v=>0.15+0.85*v)

}

ranks.sortByKey().collect()

var input = sc.textFile("hdfs://localhost:9000/wordcount/srcdata/article.data")

val writer = new PrintWriter(new File("/home/user/bigdata/spark_output.txt"))

writer.println(input.flatMap(x=>x.split(" ")).countByValue())

writer.close()

input.flatMap(x=>x.split(" ")).countByValue()