title: Export Dataset<Row> to CSV
body: <p>I'm trying to generate a CSV file based on some Spark SQL results.</p>
  
  <p>I tried to fill all null values but in vain. Seems like something linked to the way I'm configuring it.</p>
  
  <p>Here is the code I'm running.</p>
  
  <pre><code>SparkSession spark = SparkSession.builder().appName("Workshop").master("local[*]").getOrCreate();
  SQLContext sqlContext= new SQLContext(spark);
  
  Dataset&lt;Row&gt; customers = spark.read().option("header", "true").csv(pathToCustomers);
  Dataset&lt;Row&gt; unsubscribed = spark.read().option("header", "true").csv(pathToUnsubscribed);
  Dataset&lt;Row&gt; cleaned = spark.read().option("header", "true").csv(pathToCleaned);
  
  sqlContext.registerDataFrameAsTable(customers, "customers");
  sqlContext.registerDataFrameAsTable(unsubscribed, "unsubscribed");        
  sqlContext.registerDataFrameAsTable(cleaned, "cleaned");        
  
  //Run the query then the split
  Dataset&lt;Row&gt; deleteUnsubscribed = sqlContext.sql("select * from customers where Email not in (select Email_Address from unsubscribed)");   
  
  sqlContext.registerDataFrameAsTable(deleteUnsubscribed, "deleteUnsubscribed");        
  
  Dataset&lt;Row&gt; deleteCleaned = sqlContext.sql("select * from deleteUnsubscribed where Email not in (select Email_Address from cleaned)"); 
  deleteCleaned.write().option("sep", ";").option("header", "true").csv("Data/customers.csv");
  </code></pre>
  
  <p>This generates the folowing error </p>
  
  
  
  <pre><code>Exception in thread "main" org.apache.spark.SparkException: Job aborted.
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply$mcV$sp(FileFormatWriter.scala:147)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:121)
  at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:101)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74)
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135)
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132)
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113)
  at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87)
  at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87)
  at org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:492)
  at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:215)
  at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:198)
  at org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:579)
  at com.example.demo.DemoApplication.deleteCleanedAndUnsubscribedFromCustomers(DemoApplication.java:114)
  at com.example.demo.DemoApplication.main(DemoApplication.java:124)
  
  Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 8.0 failed 1 times, most recent failure: Lost task 0.0 in stage 8.0 (TID 8, localhost, executor driver): java.lang.NullPointerException
  at java.lang.ProcessBuilder.start(Unknown Source)
  at org.apache.hadoop.util.Shell.runCommand(Shell.java:404)
  at org.apache.hadoop.util.Shell.run(Shell.java:379)
  at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:589)
  at org.apache.hadoop.util.Shell.execCommand(Shell.java:678)
  at org.apache.hadoop.util.Shell.execCommand(Shell.java:661)
  at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:639)
  at org.apache.hadoop.fs.FilterFileSystem.setPermission(FilterFileSystem.java:468)
  at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:456)
  at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:424)
  at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:905)
  at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:886)
  at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:783)
  at org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.getRecordWriter(TextOutputFormat.java:132)
  at org.apache.spark.sql.execution.datasources.csv.CsvOutputWriter.&lt;init&gt;(CSVRelation.scala:208)
  at org.apache.spark.sql.execution.datasources.csv.CSVOutputWriterFactory.newInstance(CSVRelation.scala:178)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.&lt;init&gt;(FileFormatWriter.scala:234)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:182)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1$$anonfun$3.apply(FileFormatWriter.scala:129)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1$$anonfun$3.apply(FileFormatWriter.scala:128)
  at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  at org.apache.spark.scheduler.Task.run(Task.scala:99)
  at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
  at java.lang.Thread.run(Unknown Source)
  
  Driver stacktrace:
  at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)
  at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)
  at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)
  at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
  at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
  at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)
  at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
  at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
  at scala.Option.foreach(Option.scala:257)
  at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)
  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)
  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)
  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)
  at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
  at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
  at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
  at org.apache.spark.SparkContext.runJob(SparkContext.scala:1951)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply$mcV$sp(FileFormatWriter.scala:127)
  ... 22 more
  
  Caused by: java.lang.NullPointerException
  at java.lang.ProcessBuilder.start(Unknown Source)
  at org.apache.hadoop.util.Shell.runCommand(Shell.java:404)
  at org.apache.hadoop.util.Shell.run(Shell.java:379)
  at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:589)
  at org.apache.hadoop.util.Shell.execCommand(Shell.java:678)
  at org.apache.hadoop.util.Shell.execCommand(Shell.java:661)
  at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:639)
  at org.apache.hadoop.fs.FilterFileSystem.setPermission(FilterFileSystem.java:468)
  at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:456)
  at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:424)
  at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:905)
  at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:886)
  at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:783)
  at org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.getRecordWriter(TextOutputFormat.java:132)
  at org.apache.spark.sql.execution.datasources.csv.CsvOutputWriter.&lt;init&gt;(CSVRelation.scala:208)
  at org.apache.spark.sql.execution.datasources.csv.CSVOutputWriterFactory.newInstance(CSVRelation.scala:178)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.&lt;init&gt;(FileFormatWriter.scala:234)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:182)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1$$anonfun$3.apply(FileFormatWriter.scala:129)
  at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1$$anonfun$3.apply(FileFormatWriter.scala:128)
  at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  at org.apache.spark.scheduler.Task.run(Task.scala:99)
  at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
  at java.lang.Thread.run(Unknown Source0)
  </code></pre>
  
  <p>I tried with a smaller dataset (this dataset contains 57548 row) but same error. </p>
  
  <p>And here is the pom file </p>
  
  <pre class="lang-xml prettyprint-override"><code>&lt;?xml version="1.0" encoding="UTF-8"?&gt;
  &lt;project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"&gt;
  &lt;modelVersion&gt;4.0.0&lt;/modelVersion&gt;
  
  &lt;groupId&gt;com.example&lt;/groupId&gt;
  &lt;artifactId&gt;demo&lt;/artifactId&gt;
  &lt;version&gt;0.0.1-SNAPSHOT&lt;/version&gt;
  &lt;packaging&gt;jar&lt;/packaging&gt;
  
  &lt;name&gt;demo&lt;/name&gt;
  &lt;description&gt;Demo project for Spring Boot&lt;/description&gt;
  
  &lt;parent&gt;
      &lt;groupId&gt;org.springframework.boot&lt;/groupId&gt;
      &lt;artifactId&gt;spring-boot-starter-parent&lt;/artifactId&gt;
      &lt;version&gt;2.1.0.RELEASE&lt;/version&gt;
      &lt;relativePath/&gt; &lt;!-- lookup parent from repository --&gt;
  &lt;/parent&gt;
  
  &lt;properties&gt;
      &lt;project.build.sourceEncoding&gt;UTF-8&lt;/project.build.sourceEncoding&gt;
      &lt;project.reporting.outputEncoding&gt;UTF-8&lt;/project.reporting.outputEncoding&gt;
      &lt;java.version&gt;1.8&lt;/java.version&gt;
  &lt;/properties&gt;
  
  &lt;dependencies&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;org.springframework.boot&lt;/groupId&gt;
          &lt;artifactId&gt;spring-boot-starter&lt;/artifactId&gt;
      &lt;/dependency&gt;
  
      &lt;dependency&gt;
          &lt;groupId&gt;org.springframework.boot&lt;/groupId&gt;
          &lt;artifactId&gt;spring-boot-starter-test&lt;/artifactId&gt;
          &lt;scope&gt;test&lt;/scope&gt;
      &lt;/dependency&gt;
  
      &lt;dependency&gt;
          &lt;groupId&gt;org.apache.spark&lt;/groupId&gt;
          &lt;artifactId&gt;spark-core_2.11&lt;/artifactId&gt;
          &lt;version&gt;2.1.0&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;org.apache.spark&lt;/groupId&gt;
          &lt;artifactId&gt;spark-sql_2.11&lt;/artifactId&gt;
          &lt;version&gt;2.1.0&lt;/version&gt;
              &lt;exclusions&gt;
                 &lt;exclusion&gt;
                   &lt;groupId&gt;org.codehaus.janino&lt;/groupId&gt;
                   &lt;artifactId&gt;janino&lt;/artifactId&gt;
                 &lt;/exclusion&gt;
                 &lt;exclusion&gt;
                   &lt;groupId&gt;org.codehaus.janino&lt;/groupId&gt;
                   &lt;artifactId&gt;commons-compiler&lt;/artifactId&gt;
                 &lt;/exclusion&gt;
               &lt;/exclusions&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
         &lt;groupId&gt;org.codehaus.janino&lt;/groupId&gt;
         &lt;artifactId&gt;commons-compiler&lt;/artifactId&gt;
         &lt;version&gt;3.0.6&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
         &lt;groupId&gt;org.codehaus.janino&lt;/groupId&gt;
         &lt;artifactId&gt;janino&lt;/artifactId&gt;
      &lt;/dependency&gt;
  &lt;/dependencies&gt;
  
  &lt;build&gt;
      &lt;plugins&gt;
          &lt;plugin&gt;
              &lt;groupId&gt;org.springframework.boot&lt;/groupId&gt;
              &lt;artifactId&gt;spring-boot-maven-plugin&lt;/artifactId&gt;
          &lt;/plugin&gt;
      &lt;/plugins&gt;
  &lt;/build&gt;
  </code></pre>
  
  <p></p>
  
  <p>I'm still new to the Spark processing so it could be just a nooby error from a misunderstanding.</p>
  
  <p>Thanks for your help. </p>