import sys
import datetime
import numpy as np
import pandas as pd

FREQUENCY_COLUMN_NAMES = [ 'term', 'counts','idf','percentage_of_documents' ]

frequency_path = sys.argv[1]
output_path = sys.argv[2]
mallet_stoplist_path = sys.argv[3]
custom_stoplist_path = sys.argv[4]

def load_tokens(path):
	def clean_token(token):
		return token.strip().replace('\n', '')

	if len(path) == 0:
		return []

	with open(path) as file:
		for token in file:
			yield clean_token(token)

def main():
	
	stopwords = list(load_tokens(mallet_stoplist_path))
	custom_stopwords = list(load_tokens(custom_stoplist_path))

	print('started creating extra stoplist file at {}'.format(str(datetime.datetime.now())))
	
	print(output_path)
	with open(output_path, 'w') as output:

		## add in the custom stopwords ...
		output.write(
			'\n'.join([ term for term in custom_stopwords if len(term) > 0 ]))
	
		## load frequency csv ...
		df = pd.read_csv(frequency_path, names=FREQUENCY_COLUMN_NAMES, skiprows=1)
		df.term = df.term.fillna('nan').astype(str)

		## want only words with idf between ~ .5 and 2 ...
		q1 = np.logical_and(df.idf > .5, df.idf < 2)
		q2 = np.logical_not(q1)

		## write terms out to stoplist file ...
		output.write(
			'\n'.join([term for term in df[q2].term.values if not term in stopwords]))
	
	print('completed at {}'.format(str(datetime.datetime.now())))

if __name__ == "__main__":
	main()