src/malware_classifier.py
def classify(likelihoods, priors, data):
    """
    creates classifications for each document in data
    parameters:
    likelihoods: an rdd of likelihoods for each class for each word
    priors: an rdd of priors for each class
    data: a pair rdd of tokens for each file
    """
    # join likelihoods onto tokens from test data
    data = data.map(lambda x: (x[1], x[0])).join(likelihoods).map(lambda x: x[1])

    # convert to log likelihoods to avoid overflow
    data = data.mapValues(lambda x: [log(i) for i in x])

    # reduce by filename to create likelihood sums
    data = data.reduceByKey(lambda x, y: [i+j for i, j in zip(x, y)])

    # create and broadcast log priors
    log_priors = priors.mapValues(lambda x: log(x)).values().collect()
    log_priors = sc.broadcast(log_priors)

    # add priors to reduced likelihoods
    data = data.mapValues(lambda x: [i+j for i, j in zip(x, log_priors.value)])

    # argmax for class
    classes = data.mapValues(lambda x: x.index(max(x))+1)
    return classes

    creates classifications for each document in data
    parameters:
    likelihoods: an rdd of likelihoods for each class for each word
    priors: an rdd of priors for each class
    data: a pair rdd of tokens for each file
    
creates classifications for each document in data
parameters:
likelihoods: an rdd of likelihoods for each class for each word
priors: an rdd of priors for each class
data: a pair rdd of tokens for each file
creates classifications for each document in data parameters : likelihoods : an rdd of likelihoods for each class for each word priors : an rdd of priors for each class data : a pair rdd of tokens for each file
def classify(likelihoods, priors, data):
    data = data.map(lambda x: (x[1], x[0])).join(likelihoods).map(lambda x: x[1])
    data = data.mapValues(lambda x: [log(i) for i in x])
    data = data.reduceByKey(lambda x, y: [i+j for i, j in zip(x, y)])
    log_priors = priors.mapValues(lambda x: log(x)).values().collect()
    log_priors = sc.broadcast(log_priors)
    data = data.mapValues(lambda x: [i+j for i, j in zip(x, log_priors.value)])
    classes = data.mapValues(lambda x: x.index(max(x))+1)
    return classes
def classify ( likelihoods , priors , data ) : data = data . map ( lambda x : ( x [ 1 ] , x [ 0 ] ) ) . join ( likelihoods ) . map ( lambda x : x [ 1 ] ) data = data . mapValues ( lambda x : [ log ( i ) for i in x ] ) data = data . reduceByKey ( lambda x , y : [ i + j for i , j in zip ( x , y ) ] ) log_priors = priors . mapValues ( lambda x : log ( x ) ) . values ( ) . collect ( ) log_priors = sc . broadcast ( log_priors ) data = data . mapValues ( lambda x : [ i + j for i , j in zip ( x , log_priors . value ) ] ) classes = data . mapValues ( lambda x : x . index ( max ( x ) ) + 1 ) return classes
creates classifications for each document in data
parameters:
likelihoods: an rdd of likelihoods for each class for each word
priors: an rdd of priors for each class
data: a pair rdd of tokens for each file
creates classifications for each document in data parameters : likelihoods : an rdd of likelihoods for each class for each word priors : an rdd of priors for each class data : a pair rdd of tokens for each file
"""
    creates classifications for each document in data
    parameters:
    likelihoods: an rdd of likelihoods for each class for each word
    priors: an rdd of priors for each class
    data: a pair rdd of tokens for each file
    """ # join likelihoods onto tokens from test data # convert to log likelihoods to avoid overflow # reduce by filename to create likelihood sums # create and broadcast log priors # add priors to reduced likelihoods # argmax for class
likelihoods
priors
data