title: Spark Couldn't Find Window Function
body: <p>Using the solution provided in <a href="https://stackoverflow.com/a/32407543/5379015">https://stackoverflow.com/a/32407543/5379015</a>
  I tried to recreate the same query but using the programmatic syntax in stead of the <code>Dataframe</code> API as follows:</p>
  
  <pre><code>import org.apache.spark.{SparkContext, SparkConf}
  import org.apache.spark.sql.hive.HiveContext
  import org.apache.spark.sql.expressions.Window
  import org.apache.spark.sql.functions._
  
  object HiveContextTest {
    def main(args: Array[String]) {
      val conf = new SparkConf().setAppName("HiveContextTest")
      val sc = new SparkContext(conf)
      val sqlContext = new HiveContext(sc)
      import sqlContext.implicits._
  
      val df = sc.parallelize(
        ("foo", 1) :: ("foo", 2) :: ("bar", 1) :: ("bar", 2) :: Nil
      ).toDF("k", "v")
  
  
      // using dataframe api works fine
  
      val w = Window.partitionBy($"k").orderBy($"v")
      df.select($"k",$"v", rowNumber().over(w).alias("rn")).show
  
  
      //using programmatic syntax doesn't work
  
      df.registerTempTable("df")
      val w2 = sqlContext.sql("select k,v,rowNumber() over (partition by k order by v) as rn from df")
      w2.show()
  
    }
  }
  </code></pre>
  
  <p>The first <code>df.select($"k",$"v", rowNumber().over(w).alias("rn")).show</code> works fine but the <code>w2.show()</code> results in</p>
  
  <pre><code>Exception in thread "main" org.apache.spark.sql.AnalysisException: Couldn't find window function rowNumber;
  </code></pre>
  
  <p>Does anyone have any ideas as to how I can make this work with the programmatic syntax? Many thanks in advance.</p>