src/malware_classifier.py
def smoothing(train_dat, test_dat):
	'''The Function being tested should take labeled training data and
        tokenized testing data and number of classes and return an RDD of the training data with a
        new entry added for each unique (label, word) pair to increase the
        count of all vocab by 1 for each label to avoid 0 probabilities
        '''

		
	label_dat = train_dat.flatMap(lambda x : x)
	label_dat = label_dat.filter(lambda x : len(x) != 2) # retaining only the labels
	label_dat = label_dat.distinct() # retaining only the distinct labels
	n = label.count() # taking the count of distincr labels

	word_dat = train_dat.flatMap(lambda x : x)
	word_dat = word_dat.filter(lambda x : len(x) == 2) # retaining only the words
	word_dat = word_dat.distinct() # retaining only the distinct words

	mod_dat = label_dat.cartesian(word_dat) # taking cartesian product of each word in the training data with each class
	train_complete = train_dat.union(mod_dat)
	
	mod_train = train_dat.flatMap(lambda x : x)
	mod_train = mod_train.filter(lambda x : len(x) == 2) # retaining only the hex binaries

	mod_test = test_dat.flatMap(lambda x : x)
	mod_test = mod_test.filter(lambda x : len(x) == 2) # retaining only the hex binaries

	diff = mod_test.subtract(mod_train) # retaining the hex binaries which are present in test but not in training data
	new_rdd = diff.flatMap(lambda x: ((str(i),x)for i in range(1,n+1))) # adding missing hex binaries with each label in an rdd

	complete_train = train_complete.union(new_rdd) # appending the rdd with missing hex binaries to the training data rdd
	return complete_train
The Function being tested should take labeled training data and
        tokenized testing data and number of classes and return an RDD of the training data with a
        new entry added for each unique (label, word) pair to increase the
        count of all vocab by 1 for each label to avoid 0 probabilities
        
The Function being tested should take labeled training data and
tokenized testing data and number of classes and return an RDD of the training data with a
new entry added for each unique (label, word) pair to increase the
count of all vocab by 1 for each label to avoid 0 probabilities
The Function being tested should take labeled training data and tokenized testing data and number of classes and return an RDD of the training data with a new entry added for each unique ( label word ) pair to increase the count of all vocab by 1 for each label to avoid 0 probabilities
def smoothing(train_dat, test_dat):
	label_dat = train_dat.flatMap(lambda x : x)
	label_dat = label_dat.filter(lambda x : len(x) != 2) 
	label_dat = label_dat.distinct() 
	n = label.count() 
	word_dat = train_dat.flatMap(lambda x : x)
	word_dat = word_dat.filter(lambda x : len(x) == 2) 
	word_dat = word_dat.distinct() 
	mod_dat = label_dat.cartesian(word_dat) 
	train_complete = train_dat.union(mod_dat)
	mod_train = train_dat.flatMap(lambda x : x)
	mod_train = mod_train.filter(lambda x : len(x) == 2) 
	mod_test = test_dat.flatMap(lambda x : x)
	mod_test = mod_test.filter(lambda x : len(x) == 2) 
	diff = mod_test.subtract(mod_train) 
	new_rdd = diff.flatMap(lambda x: ((str(i),x)for i in range(1,n+1))) 
	complete_train = train_complete.union(new_rdd) 
	return complete_train
def smoothing ( train_dat , test_dat ) : label_dat = train_dat . flatMap ( lambda x : x ) label_dat = label_dat . filter ( lambda x : len ( x ) != 2 ) label_dat = label_dat . distinct ( ) n = label . count ( ) word_dat = train_dat . flatMap ( lambda x : x ) word_dat = word_dat . filter ( lambda x : len ( x ) == 2 ) word_dat = word_dat . distinct ( ) mod_dat = label_dat . cartesian ( word_dat ) train_complete = train_dat . union ( mod_dat ) mod_train = train_dat . flatMap ( lambda x : x ) mod_train = mod_train . filter ( lambda x : len ( x ) == 2 ) mod_test = test_dat . flatMap ( lambda x : x ) mod_test = mod_test . filter ( lambda x : len ( x ) == 2 ) diff = mod_test . subtract ( mod_train ) new_rdd = diff . flatMap ( lambda x : ( ( str ( i ) , x ) for i in range ( 1 , n + 1 ) ) ) complete_train = train_complete . union ( new_rdd ) return complete_train
The Function being tested should take labeled training data and
tokenized testing data and number of classes and return an RDD of the training data with a
new entry added for each unique (label, word) pair to increase the
count of all vocab by 1 for each label to avoid 0 probabilities
The Function being tested should take labeled training data and tokenized testing data and number of classes and return an RDD of the training data with a new entry added for each unique ( label word ) pair to increase the count of all vocab by 1 for each label to avoid 0 probabilities
'''The Function being tested should take labeled training data and
        tokenized testing data and number of classes and return an RDD of the training data with a
        new entry added for each unique (label, word) pair to increase the
        count of all vocab by 1 for each label to avoid 0 probabilities
        ''' # retaining only the labels # retaining only the distinct labels # taking the count of distincr labels # retaining only the words # retaining only the distinct words # taking cartesian product of each word in the training data with each class # retaining only the hex binaries # retaining only the hex binaries # retaining the hex binaries which are present in test but not in training data # adding missing hex binaries with each label in an rdd # appending the rdd with missing hex binaries to the training data rdd
train_dat
test_dat