code: def convert_predict_rdd_to_dataframe(df, prediction_rdd):
      from pyspark.sql import Row
      from pyspark.sql.types import (
          StructType,
          StructField,
          FloatType,
          ArrayType,
      )
      from pyspark.ml.linalg import VectorUDT, Vectors
  
      def combine(pair):
          # list of np array
          if isinstance(pair[1], list):
              row = Row(
                  *(
                      [pair[0][col] for col in pair[0].__fields__]
                      + [[Vectors.dense(elem) for elem in pair[1]]]
                  )
              )
              return row, ArrayType(VectorUDT())
          # scalar
          elif len(pair[1].shape) == 0:
              row = Row(
                  *(
                      [pair[0][col] for col in pair[0].__fields__]
                      + [float(pair[1].item(0))]
                  )
              )
              return row, FloatType()
          # np ndarray
          else:
              dim = len(pair[1].shape)
              if dim == 1:
                  # np 1-D array
                  row = Row(
                      *(
                          [pair[0][col] for col in pair[0].__fields__]
                          + [Vectors.dense(pair[1])]
                      )
                  )
                  return row, VectorUDT()
              else:
                  # multi-dimensional array
                  structType = FloatType()
                  for _ in range(dim):
                      structType = ArrayType(structType)
                  row = Row(
                      *(
                          [pair[0][col] for col in pair[0].__fields__]
                          + [pair[1].tolist()]
                      )
                  )
                  return row, structType
  
      combined_rdd = df.rdd.zip(prediction_rdd).map(combine)
      type = combined_rdd.map(lambda data: data[1]).first()
      result_rdd = combined_rdd.map(lambda data: data[0])
      schema = StructType(
          df.schema.fields + [StructField("prediction", type)]
      )
      result_df = result_rdd.toDF(schema)
      return result_df
  
file_path: analytics-zoo/pyzoo/zoo/orca/learn/utils.py
imports: from zoo.common.utils import get_file_list
  from zoo.orca.data import SparkXShards
  from zoo.orca.data.utils import get_size
  from zoo.util.utils import convert_row_to_numpy
  import numpy
label_desc: Use image node_id 5 for calling a global function with example usage: convert_predict_rdd_to_dataframe(df, prediction_rdd) and returns: result_df, row,  ArrayType, row,  FloatType, row,  VectorUDT, row,  structType