inference_schema/parameter_types/spark_parameter_type.py
def deserialize_input(self, input_data):
        """
        Convert the provided spark-like object into a spark dataframe. Will attempt to enforce column type and array
        shape as specified when constructed.

        :param input_data: The spark-like object to convert.
        :type input_data: list
        :return: The converted spark dataframe.
        :rtype: DataFrame
        """
        if isinstance(input_data, DataFrame):
            return input_data

        if not isinstance(input_data, list):
            raise Exception('Error, unable to convert input of type "{}" to Spark DataFrame'.format(type(input_data)))

        spark_session = SparkSession.builder.getOrCreate()
        sql_context = SQLContext(spark_session.sparkContext)

        if self.apply_sample_schema:
            """
            Because the transform from Spark format to Swagger format is not perfect, (e.g. Spark timestamp can only
            be represented as a formatted string in Swagger, but the string is not directly convertible by Spark back to
            a timestamp type, even when presented with this hint), we first process the incoming payload and fix some of
            these inconsistencies
            """
            for i in range(0, len(input_data)):
                input_data[i] = SparkParameterType._preprocess_json_input(input_data[i], self.sample_input.schema)
            data_frame = sql_context.createDataFrame(data=input_data, schema=self.sample_input.schema)
        else:
            data_frame = sql_context.createDataFrame(data=input_data)

        return data_frame

        Convert the provided spark-like object into a spark dataframe. Will attempt to enforce column type and array
        shape as specified when constructed.

        :param input_data: The spark-like object to convert.
        :type input_data: list
        :return: The converted spark dataframe.
        :rtype: DataFrame
        
Convert the provided spark-like object into a spark dataframe. Will attempt to enforce column type and array
shape as specified when constructed.
Convert the provided spark - like object into a spark dataframe . Will attempt to enforce column type and array shape as specified when constructed .
def deserialize_input(self, input_data):
        if isinstance(input_data, DataFrame):
            return input_data
        if not isinstance(input_data, list):
            raise Exception('Error, unable to convert input of type "{}" to Spark DataFrame'.format(type(input_data)))
        spark_session = SparkSession.builder.getOrCreate()
        sql_context = SQLContext(spark_session.sparkContext)
        if self.apply_sample_schema:
            for i in range(0, len(input_data)):
                input_data[i] = SparkParameterType._preprocess_json_input(input_data[i], self.sample_input.schema)
            data_frame = sql_context.createDataFrame(data=input_data, schema=self.sample_input.schema)
        else:
            data_frame = sql_context.createDataFrame(data=input_data)
        return data_frame
def deserialize_input ( self , input_data ) : if isinstance ( input_data , DataFrame ) : return input_data if not isinstance ( input_data , list ) : raise Exception ( 'Error, unable to convert input of type "{}" to Spark DataFrame' . format ( type ( input_data ) ) ) spark_session = SparkSession . builder . getOrCreate ( ) sql_context = SQLContext ( spark_session . sparkContext ) if self . apply_sample_schema : """
            Because the transform from Spark format to Swagger format is not perfect, (e.g. Spark timestamp can only
            be represented as a formatted string in Swagger, but the string is not directly convertible by Spark back to
            a timestamp type, even when presented with this hint), we first process the incoming payload and fix some of
            these inconsistencies
            """ for i in range ( 0 , len ( input_data ) ) : input_data [ i ] = SparkParameterType . _preprocess_json_input ( input_data [ i ] , self . sample_input . schema ) data_frame = sql_context . createDataFrame ( data = input_data , schema = self . sample_input . schema ) else : data_frame = sql_context . createDataFrame ( data = input_data ) return data_frame
Convert the provided spark-like object into a spark dataframe.
Convert the provided spark - like object into a spark dataframe .
"""
        Convert the provided spark-like object into a spark dataframe. Will attempt to enforce column type and array
        shape as specified when constructed.

        :param input_data: The spark-like object to convert.
        :type input_data: list
        :return: The converted spark dataframe.
        :rtype: DataFrame
        """ """
            Because the transform from Spark format to Swagger format is not perfect, (e.g. Spark timestamp can only
            be represented as a formatted string in Swagger, but the string is not directly convertible by Spark back to
            a timestamp type, even when presented with this hint), we first process the incoming payload and fix some of
            these inconsistencies
            """
The converted spark dataframe.
The converted spark dataframe .
DataFrame
self
input_data
The spark-like object to convert.
The spark - like object to convert .