title: spark dataframe handle option some none
body: <p>Similar to <a href="https://stackoverflow.com/questions/29103655/apache-spark-dealing-with-option-some-none-in-rdds">Apache Spark: dealing with Option/Some/None in RDDs</a> I have a function which is applied via <code>df.mapPartitions</code></p>
  
  <pre><code>def mapToTopics(iterator: Iterator[RawRecords]): Iterator[TopicContent] = {
      iterator.map(k =&gt; {
        browser.parseString(k.content) &gt;&gt; elementList("doc").map(d =&gt; {
          TopicContent((d &gt;&gt; text("docno")).head, (d &gt;&gt; text("text")).head, k.path)
        })
      })
    }
  </code></pre>
  
  <p>The following is also defined:</p>
  
  <pre><code>@transient lazy val browser = JsoupBrowser()
  case class TopicContent(topic: String, content: String, filepath: String)
  case class RawRecords(path: String, content: String)
  </code></pre>
  
  <p>Above will throw an error (<code>NoSuchElementException</code>) if no xml tags with text exist (which happens for some malformed documents)
  How can I correct and simplify this code to properly handle the options?</p>
  
  <p>When trying to use a util.Try as outlined by the link above and applying a <code>flatMap</code> my code would fail, as instead of <code>Element</code> it was using <code>Char</code></p>
  
  <h1>edit</h1>
  
  <pre><code>try {
                Some(TopicContent((d &gt;&gt; text("docno")).head, (d &gt;&gt; text("text")).head, k.path))
              } catch {
                case noelem: NoSuchElementException =&gt; {
                  println(d.head)
                  None
                }
              }
            })
  val flattended = results.flatten
  </code></pre>
  
  <p>Will unfortunately only return a <code>Option[Nothing]</code></p>
  
  <h2>edit4</h2>
  
  <p><a href="https://gist.github.com/geoHeil/bfb01427b88cf58ea755f912ce539712" rel="nofollow noreferrer">https://gist.github.com/geoHeil/bfb01427b88cf58ea755f912ce539712</a> a minimal sample without spark (and full code below as well)</p>
  
  <pre><code>import net.ruippeixotog.scalascraper.browser.JsoupBrowser
  import net.ruippeixotog.scalascraper.dsl.DSL.Extract._
  import net.ruippeixotog.scalascraper.dsl.DSL._
  import net.ruippeixotog.scalascraper.scraper.ContentExtractors.elementList
  
  @transient lazy val browser = JsoupBrowser()
  val broken =
    """
      |&lt;docno&gt;
      |   LA051089-0001
      | &lt;/docno&gt;
      | &lt;docid&gt;
      |   54901
      | &lt;/docid&gt;
      | &lt;date&gt;
      |  &lt;p&gt; May 10, 1989, Wednesday, Home Edition &lt;/p&gt;
      | &lt;/date&gt;
      | &lt;section&gt;
      |  &lt;p&gt; Metro; Part 2; Page 2; Column 2 &lt;/p&gt;
      | &lt;/section&gt;
      | &lt;graphic&gt;
      |  &lt;p&gt; Photo, Cloudy and Clear A stormy afternoon provides a clear view of Los Angeles' skyline, with the still-emerging Library Tower rising above its companion buildings. KEN LUBAS / Los Angeles Times &lt;/p&gt;
      | &lt;/graphic&gt;
      | &lt;type&gt;
      |  &lt;p&gt; Wild Art &lt;/p&gt;
      | &lt;/type&gt;
    """.stripMargin
  val correct =
    """
      |&lt;DOC&gt;
      |&lt;DOCNO&gt; FR940104-0-00001 &lt;/DOCNO&gt;
      |&lt;PARENT&gt; FR940104-0-00001 &lt;/PARENT&gt;
      |&lt;TEXT&gt;
      |
      |&lt;!-- PJG FTAG 4700 --&gt;
      |
      |&lt;!-- PJG STAG 4700 --&gt;
      |
      |&lt;!-- PJG ITAG l=90 g=1 f=1 --&gt;
      |
      |&lt;!-- PJG /ITAG --&gt;
      |
      |&lt;!-- PJG ITAG l=90 g=1 f=4 --&gt;
      |Federal Register
      |&lt;!-- PJG /ITAG --&gt;
      |
      |&lt;!-- PJG ITAG l=90 g=1 f=1 --&gt;
      |&amp;blank;/&amp;blank;Vol. 59, No. 2&amp;blank;/&amp;blank;Tuesday, January 4, 1994&amp;blank;/&amp;blank;Rules and Regulations
      |
      |&lt;!-- PJG 0012 frnewline --&gt;
      |
      |&lt;!-- PJG /ITAG --&gt;
      |
      |&lt;!-- PJG ITAG l=01 g=1 f=1 --&gt;
      |Vol. 59, No. 2
      |&lt;!-- PJG 0012 frnewline --&gt;
      |
      |&lt;!-- PJG /ITAG --&gt;
      |
      |&lt;!-- PJG ITAG l=02 g=1 f=1 --&gt;
      |Tuesday, January 4, 1994
      |&lt;!-- PJG 0012 frnewline --&gt;
      |
      |&lt;!-- PJG 0012 frnewline --&gt;
      |
      |&lt;!-- PJG /ITAG --&gt;
      |
      |&lt;!-- PJG /STAG --&gt;
      |
      |&lt;!-- PJG /FTAG --&gt;
      |&lt;/TEXT&gt;
      |&lt;/DOC&gt;
    """.stripMargin
  case class RawRecords(path: String, content: String)
  
  case class TopicContent(topic: String, content: String, filepath: String)
  val raw = Seq(RawRecords("first", correct), RawRecords("second", broken))
  val result = mapToTopics(raw.iterator)
  
  // Variant 1
  def mapToTopics(iterator: Iterator[RawRecords]): Iterator[TopicContent] = {
    iterator.flatMap(k =&gt; {
      val documents = browser.parseString(k.content) &gt;&gt; elementList("doc")
      documents.map(d =&gt; {
        val docno = d &gt;&gt; text("docno")
        //        try {
        val textContent = d &gt;&gt; text("text")
        TopicContent(docno, textContent, k.path)
        //        } catch {
        //          case _:NoSuchElementException =&gt; TopicContent(docno, None, k.path)
        //        }
      }) //.filter(_.content !=None)
    })
  }
  
  
  // When broken down even further you see the following will produce Options of strings
  browser.parseString(raw(0).content) &gt;&gt; elementList("doc").map(d =&gt; {
    val docno = d &gt;&gt; text("docno")
    val textContent = d &gt;&gt; text("text")
    (docno.headOption, textContent.headOption)
  })
  
  // while below will now map to characters. What is wrong here?
  val documents = browser.parseString(raw(0).content) &gt;&gt; elementList("doc")
    documents.map(d =&gt; {
    val docno = d &gt;&gt; text("docno")
    val textContent = d &gt;&gt; text("text")
    (docno.headOption, textContent.headOption)
  })
  </code></pre>