title: How to calculate correlation in spark on columns with nulls?
body: <p>Given a column of dense vectors with NaN entries I would like to calculate correlation between columns. Is there a way to do that without disassembling the vector for value clean up?</p>
  
  <pre><code>    #pyspark
  from pyspark.sql import SparkSession
  from pyspark.ml.linalg import Vectors
  from pyspark.mllib.linalg import Vectors as MlVectors # (
  from pyspark.mllib.stat import Statistics
  
  def get_data():
    spark = SparkSession.builder.getOrCreate()
    df = spark.createDataFrame(
      [
        (Vectors.dense(1., 3., 2.), 0),
        (Vectors.dense(None, 4., 1.), 1),
        (Vectors.dense(3., None, 0.), 2),
        (Vectors.dense(4., 12., None), 3),
        (Vectors.dense(5., 0., 1.), 5),
        (Vectors.dense(6., -1., 0.), 6)], ["features", "foo"])
    return df
  
  
  def correlation(df):
    digestible_data = df.select("features").rdd.map(lambda row: MlVectors.dense(row[0]))
    print(Statistics.corr(digestible_data))
  
  
  if __name__ == '__main__':
      correlation(get_data())
  # OUTPUT:
  # [[  1.  nan  nan]
  #  [ nan   1.  nan]
  #  [ nan  nan   1.]]
  </code></pre>