spark/example_udf.py
def udf(sc, data_path, sampling_ratio, seed):
    """
    This is an example UDF function to load training data. It loads data from a text file
    with base64-encoded pickled numpy arrays on each line.
    """

    # Compute the number of cores in our cluster - used below to heuristically set the number of partitions
    total_cores = int(sc._conf.get('spark.executor.instances')) * int(sc._conf.get('spark.executor.cores'))

    # Load and sample down the dataset
    d = sc.textFile(data_path, total_cores * 3).sample(False, sampling_ratio, seed)

    deserialize_vec = lambda s: pkl.loads(base64.decodestring(s))
    vecs = d.map(deserialize_vec)

    return vecs

    This is an example UDF function to load training data. It loads data from a text file
    with base64-encoded pickled numpy arrays on each line.
    
This is an example UDF function to load training data. It loads data from a text file
with base64-encoded pickled numpy arrays on each line.
This is an example UDF function to load training data . It loads data from a text file with base64 - encoded pickled numpy arrays on each line .
def udf(sc, data_path, sampling_ratio, seed):
    total_cores = int(sc._conf.get('spark.executor.instances')) * int(sc._conf.get('spark.executor.cores'))
    d = sc.textFile(data_path, total_cores * 3).sample(False, sampling_ratio, seed)
    deserialize_vec = lambda s: pkl.loads(base64.decodestring(s))
    vecs = d.map(deserialize_vec)
    return vecs
def udf ( sc , data_path , sampling_ratio , seed ) : total_cores = int ( sc . _conf . get ( 'spark.executor.instances' ) ) * int ( sc . _conf . get ( 'spark.executor.cores' ) ) d = sc . textFile ( data_path , total_cores * 3 ) . sample ( False , sampling_ratio , seed ) deserialize_vec = lambda s : pkl . loads ( base64 . decodestring ( s ) ) vecs = d . map ( deserialize_vec ) return vecs
This is an example UDF function to load training data.
This is an example UDF function to load training data .
"""
    This is an example UDF function to load training data. It loads data from a text file
    with base64-encoded pickled numpy arrays on each line.
    """ # Compute the number of cores in our cluster - used below to heuristically set the number of partitions # Load and sample down the dataset
sc
data_path
sampling_ratio
seed