title: Spark Streaming Context Hangs on Start
body: <p>All,</p>
  
  <p>I am trying to use Kinesis with Spark Streaming on Spark 1.6.0 via Databricks and my <code>ssc.start()</code> command is hanging.</p>
  
  <p>I am using the following function to make my Spark Streaming Context:</p>
  
  <pre><code>def creatingFunc(sc: SparkContext): StreamingContext = 
  {
          // Create a StreamingContext
          val ssc = new StreamingContext(sc, Seconds(batchIntervalSeconds))
  
          // Creata a Kinesis stream
          val kinesisStream = KinesisUtils.createStream(ssc,
              kinesisAppName, kinesisStreamName,
              kinesisEndpointUrl, RegionUtils.getRegionByEndpoint(kinesisEndpointUrl).getName,
              InitialPositionInStream.LATEST, Seconds(kinesisCheckpointIntervalSeconds),
              StorageLevel.MEMORY_AND_DISK_SER_2, config.awsAccessKeyId, config.awsSecretKey)
  
          kinesisStream.print()
  
          ssc.remember(Minutes(1))
          ssc.checkpoint(checkpointDir)
          ssc
  }
  </code></pre>
  
  <p>However when I run the following to start the streaming context:</p>
  
  <pre><code>// Stop any existing StreamingContext 
  val stopActiveContext = true
  if (stopActiveContext) {    
    StreamingContext.getActive.foreach { _.stop(stopSparkContext = false) }
  } 
  
  // Get or create a streaming context.
  val ssc = StreamingContext.getActiveOrCreate(() =&gt; main.creatingFunc(sc))
  
  // This starts the streaming context in the background. 
  ssc.start()
  </code></pre>
  
  <p>The last bit, <code>ssc.start()</code>, hangs indefinitely without issuing any log messages. I am running this on a freshly spun up cluster with no other notebooks attached so there aren't any other streaming contexts running.</p>
  
  <p>Any thoughts?</p>
  
  <p>Additionally, here are the libraries I am using (from my build.sbt file):</p>
  
  <pre><code>"org.apache.spark" % "spark-core_2.10" % "1.6.0"
  "org.apache.spark" % "spark-sql_2.10" % "1.6.0"
  "org.apache.spark" % "spark-streaming-kinesis-asl_2.10" % "1.6.0"
  "org.apache.spark" % "spark-streaming_2.10" % "1.6.0"
  </code></pre>
  
  <p><strong>Edit 1:</strong>
  After running the code again recently, I got the following error:</p>
  
  <pre><code>java.rmi.RemoteException: java.util.concurrent.TimeoutException: Timed out retrying send to http://10.210.224.74:7070: 2 hours; nested exception is: 
      java.util.concurrent.TimeoutException: Timed out retrying send to http://10.210.224.74:7070: 2 hours
      at com.databricks.backend.daemon.data.client.DbfsClient.send0(DbfsClient.scala:71)
      at com.databricks.backend.daemon.data.client.DbfsClient.sendIdempotent(DbfsClient.scala:40)
      at com.databricks.backend.daemon.data.client.DatabricksFileSystem.listStatus(DatabricksFileSystem.scala:189)
      at org.apache.spark.streaming.util.FileBasedWriteAheadLog.initializeOrRecover(FileBasedWriteAheadLog.scala:228)
      at org.apache.spark.streaming.util.FileBasedWriteAheadLog.&lt;init&gt;(FileBasedWriteAheadLog.scala:72)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$$anonfun$2.apply(WriteAheadLogUtils.scala:141)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$$anonfun$2.apply(WriteAheadLogUtils.scala:141)
      at scala.Option.getOrElse(Option.scala:120)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$.createLog(WriteAheadLogUtils.scala:140)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$.createLogForDriver(WriteAheadLogUtils.scala:98)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker$$anonfun$createWriteAheadLog$1.apply(ReceivedBlockTracker.scala:254)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker$$anonfun$createWriteAheadLog$1.apply(ReceivedBlockTracker.scala:252)
      at scala.Option.map(Option.scala:145)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker.createWriteAheadLog(ReceivedBlockTracker.scala:252)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker.&lt;init&gt;(ReceivedBlockTracker.scala:75)
      at org.apache.spark.streaming.scheduler.ReceiverTracker.&lt;init&gt;(ReceiverTracker.scala:106)
      at org.apache.spark.streaming.scheduler.JobScheduler.start(JobScheduler.scala:80)
      at org.apache.spark.streaming.StreamingContext$$anonfun$liftedTree1$1$1.apply$mcV$sp(StreamingContext.scala:610)
      at org.apache.spark.streaming.StreamingContext$$anonfun$liftedTree1$1$1.apply(StreamingContext.scala:606)
      at org.apache.spark.streaming.StreamingContext$$anonfun$liftedTree1$1$1.apply(StreamingContext.scala:606)
      at ... run in separate thread using org.apache.spark.util.ThreadUtils ... ()
      at org.apache.spark.streaming.StreamingContext.liftedTree1$1(StreamingContext.scala:606)
      at org.apache.spark.streaming.StreamingContext.start(StreamingContext.scala:600)
  Caused by: java.util.concurrent.TimeoutException: Timed out retrying send to http://10.210.224.74:7070: 2 hours
      at com.databricks.rpc.ReliableJettyClient.retryOnNetworkError(ReliableJettyClient.scala:138)
      at com.databricks.rpc.ReliableJettyClient.sendIdempotent(ReliableJettyClient.scala:46)
      at com.databricks.backend.daemon.data.client.DbfsClient.doSend(DbfsClient.scala:83)
      at com.databricks.backend.daemon.data.client.DbfsClient.send0(DbfsClient.scala:60)
      at com.databricks.backend.daemon.data.client.DbfsClient.sendIdempotent(DbfsClient.scala:40)
      at com.databricks.backend.daemon.data.client.DatabricksFileSystem.listStatus(DatabricksFileSystem.scala:189)
      at org.apache.spark.streaming.util.FileBasedWriteAheadLog.initializeOrRecover(FileBasedWriteAheadLog.scala:228)
      at org.apache.spark.streaming.util.FileBasedWriteAheadLog.&lt;init&gt;(FileBasedWriteAheadLog.scala:72)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$$anonfun$2.apply(WriteAheadLogUtils.scala:141)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$$anonfun$2.apply(WriteAheadLogUtils.scala:141)
      at scala.Option.getOrElse(Option.scala:120)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$.createLog(WriteAheadLogUtils.scala:140)
      at org.apache.spark.streaming.util.WriteAheadLogUtils$.createLogForDriver(WriteAheadLogUtils.scala:98)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker$$anonfun$createWriteAheadLog$1.apply(ReceivedBlockTracker.scala:254)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker$$anonfun$createWriteAheadLog$1.apply(ReceivedBlockTracker.scala:252)
      at scala.Option.map(Option.scala:145)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker.createWriteAheadLog(ReceivedBlockTracker.scala:252)
      at org.apache.spark.streaming.scheduler.ReceivedBlockTracker.&lt;init&gt;(ReceivedBlockTracker.scala:75)
      at org.apache.spark.streaming.scheduler.ReceiverTracker.&lt;init&gt;(ReceiverTracker.scala:106)
      at org.apache.spark.streaming.scheduler.JobScheduler.start(JobScheduler.scala:80)
      at org.apache.spark.streaming.StreamingContext$$anonfun$liftedTree1$1$1.apply$mcV$sp(StreamingContext.scala:610)
      at org.apache.spark.streaming.StreamingContext$$anonfun$liftedTree1$1$1.apply(StreamingContext.scala:606)
      at org.apache.spark.streaming.StreamingContext$$anonfun$liftedTree1$1$1.apply(StreamingContext.scala:606)
      at org.apache.spark.util.ThreadUtils$$anon$1.run(ThreadUtils.scala:122)
  </code></pre>