1. object OnlineTheTop3ItemForEachCategory2DB {
2. def main(args: Array[String]){
3. val conf = new SparkConf() //创建SparkConf对象
4. //设置应用程序的名称，在程序运行的监控界面可以看到名称
5. conf.setAppName("OnlineTheTop3ItemForEachCategory2DB")
6. conf.setMaster("spark://Master:7077") //此时，程序在Spark集群
7. //设置batchDuration时间间隔来控制Job生成的频率并且创建Spark Streaming执行的入口
8. val ssc = new StreamingContext(conf, Seconds(5))
9. ssc.checkpoint("/root/Documents/SparkApps/checkpoint")
10. val soketDStream = ssc.socketTextStream("Master", 9999)
11. /// 业务处理逻辑 .....
12. ssc.start()
13. ssc.awaitTermination()
14. }
15. }

1. private[streaming] def createNewSparkContext(conf: SparkConf): SparkContext = {
2. new SparkContext(conf)
3. }

1. val socketDstram = ssc.socketTextStream("Master", 9999)

3）可看到代码最后面调用socketStream。

4）实际上生成SocketInputDStream。

SocketInputDStream继承ReceiverInputDStream。

DStream的getOrCompute：

主要是生成RDD，再将生成的RDD放在HashMap中。具体生成RDD过程以后剖析。

1. /**
2. * Start the execution of the streams.
3. *
4. * @throws IllegalStateException if the StreamingContext is already stopped.
5. */
6. def start(): Unit = synchronized {
7. state match {
8. case INITIALIZED =>
9. startSite.set(DStream.getCreationSite())
10. StreamingContext.ACTIVATION_LOCK.synchronized {
11. StreamingContext.assertNoOtherContextIsActive()
12. try {
13. validate()
14. // Start the streaming scheduler in a new thread, so that thread local properties
15. // like call sites and job groups can be reset without affecting those of the
16. // current thread.
17. //线程本地存储，线程有自己的私有属性，设置这些线程的时候不会影响其他线程，
18. ThreadUtils.runInNewThread("streaming-start") {
19. sparkContext.setCallSite(startSite.get)
20. sparkContext.clearJobGroup()
21. sparkContext.setLocalProperty(SparkContext.SPARK_JOB_INTERRUPT_ON_CANCEL, "false")
22. //启动JobScheduler
23. scheduler.start()
24. }
25. state = StreamingContextState.ACTIVE
26. } catch {
27. case NonFatal(e) =>
28. logError("Error starting the context, marking it as stopped", e)
29. scheduler.stop(false)
30. state = StreamingContextState.STOPPED
31. throw e
32. }
33. StreamingContext.setActiveContext(this)
34. }
35. shutdownHookRef = ShutdownHookManager.addShutdownHook(
36. StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown)
37. // Registering Streaming Metrics at the start of the StreamingContext
38. assert(env.metricsSystem != null)
39. env.metricsSystem.registerSource(streamingSource)
40. uiTab.foreach(_.attach())
41. logInfo("StreamingContext started")
42. case ACTIVE =>
43. logWarning("StreamingContext has already been started")
44. case STOPPED =>
45. throw new IllegalStateException("StreamingContext has already been stopped")
46. }
47. }

1. def start(): Unit = synchronized {
2. if (eventLoop != null) return // scheduler has already been started
3. logDebug("Starting JobScheduler")
4. eventLoop = new EventLoop[JobSchedulerEvent]("JobScheduler") {
5. override protected def onReceive(event: JobSchedulerEvent): Unit = processEvent(event)
6. override protected def onError(e: Throwable): Unit = reportError("Error in job scheduler", e)
7. }
8. // 启动消息循环处理线程。用于处理JobScheduler的各种事件。
9. eventLoop.start()
10. // attach rate controllers of input streams to receive batch completion updates
11. for {
12. inputDStream <- ssc.graph.getInputStreams
13. // rateController可以控制输入速度
14. rateController <- inputDStream.rateController
15. } ssc.addStreamingListener(rateController)
16. // 启动监听器。用于更新Spark UI中StreamTab的内容。
17. listenerBus.start(ssc.sparkContext)
18. receiverTracker = new ReceiverTracker(ssc)
19. // 生成InputInfoTracker。用于管理所有的输入的流，以及他们输入的数据统计。这些信息将通过 StreamingListener监听。
20. inputInfoTracker = new InputInfoTracker(ssc)
21. // 启动ReceiverTracker。用于处理数据接收、数据缓存、Block生成。
22. receiverTracker.start()
23. // 启动JobGenerator。用于DStreamGraph初始化、DStream与RDD的转换、生成Job、提交执行等工作。
24. jobGenerator.start()
25. logInfo("Started JobScheduler")
26. }

1. private def processEvent(event: JobSchedulerEvent) {
2. try {
3. event match {
4. case JobStarted(job, startTime) => handleJobStart(job, startTime)
5. case JobCompleted(job, completedTime) => handleJobCompletion(job, completedTime)
6. case ErrorReported(m, e) => handleError(m, e)
7. }
8. } catch {
9. case e: Throwable =>
10. reportError("Error in job scheduler", e)
11. }
12. }

1. private val eventQueue: BlockingQueue[E] = new LinkedBlockingDeque[E]()

1. private val eventThread = new Thread(name) {
2. setDaemon(true)
3. override def run(): Unit = {
4. try {
5. while (!stopped.get) {
6. val event = eventQueue.take()
7. try {
8. onReceive(event)
9. } catch {
10. case NonFatal(e) => {
11. try {
12. onError(e)
13. } catch {
14. case NonFatal(e) => logError("Unexpected error in " + name, e)
15. }
16. }
17. }
18. }
19. } catch {
20. case ie: InterruptedException => // exit even if eventQueue is not empty
21. case NonFatal(e) => logError("Unexpected error in " + name, e)
22. }
23. }
24. }

 def start(): Unit = synchronized {

   if (isTrackerStarted) {

     throw new SparkException("ReceiverTracker already started")

   }

   if (!receiverInputStreams.isEmpty) {

     endpoint = ssc.env.rpcEnv.setupEndpoint(

       "ReceiverTracker", new ReceiverTrackerEndpoint(ssc.env.rpcEnv))

     if (!skipReceiverLaunch) launchReceivers()

     logInfo("ReceiverTracker started")

     trackerState = Started

   }

 }

/**

 * Get the receivers from the ReceiverInputDStreams, distributes them to the

 * worker nodes as a parallel collection, and runs them.

 */

private def launchReceivers(): Unit = {

  val receivers = receiverInputStreams.map(nis => {

    val rcvr = nis.getReceiver()

    rcvr.setReceiverId(nis.id)

    rcvr

  })

  runDummySparkJob()

  logInfo("Starting " + receivers.length + " receivers")

  endpoint.send(StartAllReceivers(receivers))

}
 

/**

 * Run the dummy Spark job to ensure that all slaves have registered. This avoids all the

 * receivers to be scheduled on the same node.

 *

 * TODO Should poll the executor number and wait for executors according to

 * "spark.scheduler.minRegisteredResourcesRatio" and

 * "spark.scheduler.maxRegisteredResourcesWaitingTime" rather than running a dummy job.

 */

private def runDummySparkJob(): Unit = {

  if (!ssc.sparkContext.isLocal) {

    ssc.sparkContext.makeRDD( to , ).map(x => (x, )).reduceByKey(_ + _, ).collect()

  }

  assert(getExecutors.nonEmpty)

}

override def receive: PartialFunction[Any, Unit] = {

  // Local messages

  case StartAllReceivers(receivers) =>

    val scheduledLocations = schedulingPolicy.scheduleReceivers(receivers, getExecutors)

    for (receiver <- receivers) {

      val executors = scheduledLocations(receiver.streamId)

      updateReceiverScheduledExecutors(receiver.streamId, executors)

      receiverPreferredLocations(receiver.streamId) = receiver.preferredLocation

      startReceiver(receiver, executors)

    }
 

/**

 * Start a receiver along with its scheduled executors

 */

private def startReceiver(

    receiver: Receiver[_],

    scheduledLocations: Seq[TaskLocation]): Unit = {

  def shouldStartReceiver: Boolean = {

    // ........... 此处省略1万字 (无关代码) , 呵呵哒 .........

  // Function to start the receiver on the worker node

  val startReceiverFunc: Iterator[Receiver[_]] => Unit =

    (iterator: Iterator[Receiver[_]]) => {

      if (!iterator.hasNext) {

        throw new SparkException(

          "Could not start receiver as object not found.")

      }

      if (TaskContext.get().attemptNumber() == ) {

        val receiver = iterator.next()

        assert(iterator.hasNext == false)

        //实例化Receiver监控者

        val supervisor = new ReceiverSupervisorImpl(

          receiver, SparkEnv.get, serializableHadoopConf.value, checkpointDirOption)

        supervisor.start()

        supervisor.awaitTermination()

      } else {

        // It's restarted by TaskScheduler, but we want to reschedule it again. So exit it.

      }

    }

  // Create the RDD using the scheduledLocations to run the receiver in a Spark job

  val receiverRDD: RDD[Receiver[_]] =

    if (scheduledLocations.isEmpty) {

      ssc.sc.makeRDD(Seq(receiver), )

    } else {

      val preferredLocations = scheduledLocations.map(_.toString).distinct

      ssc.sc.makeRDD(Seq(receiver -> preferredLocations))

    }

  receiverRDD.setName(s"Receiver $receiverId")

  ssc.sparkContext.setJobDescription(s"Streaming job running receiver $receiverId")

  ssc.sparkContext.setCallSite(Option(ssc.getStartSite()).getOrElse(Utils.getCallSite()))

  val future = ssc.sparkContext.submitJob[Receiver[_], Unit, Unit](

    receiverRDD,

   startReceiverFunc, //提交Job时候传入startReceiverFunc这个方法，因为startReceiverFunc该方法是在Executor上执行的

  Seq(), (_, _) => Unit, ())

  // 一直重启 receiver job直到 ReceiverTracker is stopped

  future.onComplete {

    case Success(_) =>

      if (!shouldStartReceiver) {

        onReceiverJobFinish(receiverId)

      } else {

        logInfo(s"Restarting Receiver $receiverId")

        self.send(RestartReceiver(receiver))

      }

    case Failure(e) =>

      if (!shouldStartReceiver) {

        onReceiverJobFinish(receiverId)

      } else {

        logError("Receiver has been stopped. Try to restart it.", e)

        logInfo(s"Restarting Receiver $receiverId")

        self.send(RestartReceiver(receiver))

      }

  }(submitJobThreadPool)

  logInfo(s"Receiver ${receiver.streamId} started")

}
 

/** Start the supervisor */

def start() {

  onStart()

  startReceiver()

}

override protected def onStart() {

  registeredBlockGenerators.foreach { _.start() }

}

/** Start receiver */

def startReceiver(): Unit = synchronized {

  try {

    if (onReceiverStart()) {

      logInfo("Starting receiver")

      receiverState = Started

      receiver.onStart()

      logInfo("Called receiver onStart")

    } else {

      // The driver refused us

      stop("Registered unsuccessfully because Driver refused to start receiver " + streamId, None)

    }

  } catch {

    case NonFatal(t) =>

      stop("Error starting receiver " + streamId, Some(t))

  }

}

override protected def onReceiverStart(): Boolean = {

  val msg = RegisterReceiver(

    streamId, receiver.getClass.getSimpleName, host, executorId, endpoint)

  trackerEndpoint.askWithRetry[Boolean](msg)

}

override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {

  // Remote messages

  case RegisterReceiver(streamId, typ, host, executorId, receiverEndpoint) =>

    val successful =

      registerReceiver(streamId, typ, host, executorId, receiverEndpoint, context.senderAddress)

    context.reply(successful)
 

/** Register a receiver */

private def registerReceiver(

    streamId: Int,

    typ: String,

    host: String,

    executorId: String,

    receiverEndpoint: RpcEndpointRef,

    senderAddress: RpcAddress

  ): Boolean = {

  if (!receiverInputStreamIds.contains(streamId)) {

    throw new SparkException("Register received for unexpected id " + streamId)

  }

    // ........... 此处省略1万字 (无关代码) , 呵呵哒 .........

  if (!isAcceptable) {

    // Refuse it since it's scheduled to a wrong executor

    false

  } else {

    val name = s"${typ}-${streamId}"

    val receiverTrackingInfo = ReceiverTrackingInfo(

      streamId,

      ReceiverState.ACTIVE,

      scheduledLocations = None,

      runningExecutor = Some(ExecutorCacheTaskLocation(host, executorId)),

      name = Some(name),

      endpoint = Some(receiverEndpoint))

    receiverTrackingInfos.put(streamId, receiverTrackingInfo)

    listenerBus.post(StreamingListenerReceiverStarted(receiverTrackingInfo.toReceiverInfo))

    logInfo("Registered receiver for stream " + streamId + " from " + senderAddress)

    true

  }

}

private[streaming]

class SocketReceiver[T: ClassTag](

    host: String,

    port: Int,

    bytesToObjects: InputStream => Iterator[T],

    storageLevel: StorageLevel

  ) extends Receiver[T](storageLevel) with Logging {

  def onStart() {

    // Start the thread that receives data over a connection

    new Thread("Socket Receiver") {

      setDaemon(true)

      override def run() { receive() }

    }.start()

  }

  /** Create a socket connection and receive data until receiver is stopped */

  def receive() {

    var socket: Socket = null

    try {

      logInfo("Connecting to " + host + ":" + port)

      socket = new Socket(host, port)

      logInfo("Connected to " + host + ":" + port)

      val iterator = bytesToObjects(socket.getInputStream())

      while(!isStopped && iterator.hasNext) {

        store(iterator.next)

      }

      if (!isStopped()) {

        restart("Socket data stream had no more data")

      } else {

        logInfo("Stopped receiving")

      }

    } catch {

        // ........... 此处省略1万字 (无关代码) , 呵呵哒 .........

  }

}

//根据创建StreamContext时传入的batchInterval，定时发送GenerateJobs消息

private val timer = new RecurringTimer(clock, ssc.graph.batchDuration.milliseconds,

  longTime => eventLoop.post(GenerateJobs(new Time(longTime))), "JobGenerator")

JobGenerator的start()方法：

/** Start generation of jobs */

def start(): Unit = synchronized {

  if (eventLoop != null) return // generator has already been started

  // Call checkpointWriter here to initialize it before eventLoop uses it to avoid a deadlock.

  // See SPARK-10125

  checkpointWriter

  eventLoop = new EventLoop[JobGeneratorEvent]("JobGenerator") {

    override protected def onReceive(event: JobGeneratorEvent): Unit = processEvent(event)

    override protected def onError(e: Throwable): Unit = {

      jobScheduler.reportError("Error in job generator", e)

    }

  }

  // 启动消息循环处理线程

  eventLoop.start()

  if (ssc.isCheckpointPresent) {

    restart()

  } else {

    // 开启定时生成Job的定时器

    startFirstTime()

  }

}

1. /** Processes all events */
2. private def processEvent(event: JobGeneratorEvent) {
3. logDebug("Got event " + event)
4. event match {
5. case GenerateJobs(time) => generateJobs(time)
6. case ClearMetadata(time) => clearMetadata(time)
7. case DoCheckpoint(time, clearCheckpointDataLater) =>
8. doCheckpoint(time, clearCheckpointDataLater)
9. case ClearCheckpointData(time) => clearCheckpointData(time)
10. }
11. }

/** Generate jobs and perform checkpoint for the given `time`.  */

private def generateJobs(time: Time) {

  // Set the SparkEnv in this thread, so that job generation code can access the environment

  // Example: BlockRDDs are created in this thread, and it needs to access BlockManager

  // Update: This is probably redundant after threadlocal stuff in SparkEnv has been removed.

  SparkEnv.set(ssc.env)

  Try {

    // 根据特定的时间获取具体的数据

    jobScheduler.receiverTracker.allocateBlocksToBatch(time) // allocate received blocks to batch

    //调用DStreamGraph的generateJobs生成Job

    graph.generateJobs(time) // generate jobs using allocated block

  } match {

    case Success(jobs) =>

      val streamIdToInputInfos = jobScheduler.inputInfoTracker.getInfo(time)

      jobScheduler.submitJobSet(JobSet(time, jobs, streamIdToInputInfos))

    case Failure(e) =>

      jobScheduler.reportError("Error generating jobs for time " + time, e)

  }

  eventLoop.post(DoCheckpoint(time, clearCheckpointDataLater = false))

}

// 输出流：具体Action的输出操作

private val outputStreams = new ArrayBuffer[DStream[_]]()

def generateJobs(time: Time): Seq[Job] = {

  logDebug("Generating jobs for time " + time)

  val jobs = this.synchronized {

    outputStreams.flatMap { outputStream =>

      val jobOption = outputStream.generateJob(time)

      jobOption.foreach(_.setCallSite(outputStream.creationSite))

      jobOption

    }

  }

  logDebug("Generated " + jobs.length + " jobs for time " + time)

  jobs

}

1. /**
2. * Generate a SparkStreaming job for the given time. This is an internal method that
3. * should not be called directly. This default implementation creates a job
4. * that materializes the corresponding RDD. Subclasses of DStream may override this
5. * to generate their own jobs.
6. */
7. private[streaming] def generateJob(time: Time): Option[Job] = {
8. getOrCompute(time) match {
9. case Some(rdd) => {
10. val jobFunc = () => {
11. val emptyFunc = { (iterator: Iterator[T]) => {} }
12. context.sparkContext.runJob(rdd, emptyFunc)
13. }
14. Some(new Job(time, jobFunc))
15. }
16. case None => None
17. }
18. }

def submitJobSet(jobSet: JobSet) {

  if (jobSet.jobs.isEmpty) {

    logInfo("No jobs added for time " + jobSet.time)

  } else {

    listenerBus.post(StreamingListenerBatchSubmitted(jobSet.toBatchInfo))

    jobSets.put(jobSet.time, jobSet)

    jobSet.jobs.foreach(job => jobExecutor.execute(new JobHandler(job)))

    logInfo("Added jobs for time " + jobSet.time)

  }

}

private class JobHandler(job: Job) extends Runnable with Logging {

    import JobScheduler._

    def run() {

      try {

         //  *********** 此处省略无关代码 *******************

        // We need to assign `eventLoop` to a temp variable. Otherwise, because

        // `JobScheduler.stop(false)` may set `eventLoop` to null when this method is running, then

        // it's possible that when `post` is called, `eventLoop` happens to null.

        var _eventLoop = eventLoop

        if (_eventLoop != null) {

          _eventLoop.post(JobStarted(job, clock.getTimeMillis()))

          // Disable checks for existing output directories in jobs launched by the streaming

          // scheduler, since we may need to write output to an existing directory during checkpoint

          // recovery; see SPARK-4835 for more details.

          PairRDDFunctions.disableOutputSpecValidation.withValue(true) {

            job.run()

          }

          _eventLoop = eventLoop

          if (_eventLoop != null) {

            _eventLoop.post(JobCompleted(job, clock.getTimeMillis()))

          }

        } else {

          // JobScheduler has been stopped.

        }

      } finally {

        ssc.sc.setLocalProperty(JobScheduler.BATCH_TIME_PROPERTY_KEY, null)

        ssc.sc.setLocalProperty(JobScheduler.OUTPUT_OP_ID_PROPERTY_KEY, null)

      }

    }

  }

}

1. private[streaming]
2. class Job(val time: Time, func: () => _) {
3. private var _id: String = _
4. private var _outputOpId: Int = _
5. private var isSet = false
6. private var _result: Try[_] = null
7. private var _callSite: CallSite = null
8. private var _startTime: Option[Long] = None
9. private var _endTime: Option[Long] = None
10. def run() {
11. _result = Try(func())
12. }