#для корпусов (пока небольших)

# from fortexts import normalize, tokenizing, checking
#не вышло нормально импортировать из fortext.py

import nltk.data
import pymorphy2
from nltk.tokenize import RegexpTokenizer
from gensim import corpora, models, similarities
morph = pymorphy2.MorphAnalyzer()

from lxml import etree
from lxml.etree import iterparse
import string

def normalize(token):
	try: 
		gram_info = morph.parse(token)
		return gram_info[0].normal_form
	except:
		return token

def tokenizing(sentences):
	tokenizer = RegexpTokenizer("[^\W\d]+((?:[^\W\d]|[-'])+[^\W\d]+)*")
	tokens = []
	n = 0
	for sentence in sentences:
		tokens.append([])
		tokenized_line = tokenizer.tokenize(sentence.strip())
		for token in tokenized_line:
			if token not in string.punctuation:
				tokens[n].append(normalize(token.lower()))
		n+=1
	return tokens


def checking(dictionary, corpus, stopwords):
	checked = set()
	index = similarities.SparseMatrixSimilarity(corpus, num_features=len(dictionary))
	for sword in stopwords:
		new_vec = dictionary.doc2bow(sword.split())
		simls = index[new_vec]
		for s in range(len(simls)):
			if simls[s] > 0:
				checked.add(s)
	return(checked)


print("Пожалуйста, введите имя файла \n с расширением. Например: file.vert")
print("Он должен находиться в папке 'resources'")

filename = (input()).strip

destination = "resources/"+filename

doc = iterparse(destination, ('start',))

def text_prep(doc):
	sentences = []
	sentence = ""
	for event, elem in doc:
		if event == 'start' and elem.tag == "s":
			sent = elem.text
			# print(sent)
			sent = sent.split('\n')
			for line in sent:
				if line != "":
					word = line.split()[0]
					if sentence == "":
						sentence += word+" "
					else:
						sentence += word +" "
		if elem.tag == "g":
			if elem.tail != None:
				punct = elem.tail
				punct = punct.split('\n')
				for line in punct:
					if line != "":
						obj = line.split()[0]
						if obj in string.punctuation:
							if obj == "," or obj == "." or obj == "(" or obj == ")":
								sentence = sentence[:-1]
								if obj == "," or obj == ")":
									sentence += obj+" "
								else:
									sentence += obj
							else:
								sentence += obj +" "
						else:
							sentence += obj+" "
						if line.split()[1] == "SENT":
							sentences.append(sentence)
							sentence = ""
	return sentences

text = text_prep(doc)
#text — массив очень красивых предложений, извлечённый из корпуса! Его мы будем печатать
prepareds = tokenizing(text)


dictionary = corpora.Dictionary(prepareds)
corpus = [dictionary.doc2bow(sentence) for sentence in prepareds]

with open("resources/stoplist.txt") as stopfile:
	stopwords = [line.strip() for line in stopfile]

prepareds = None

checked = checking(dictionary, corpus, stopwords)

corpus = None
dictionary = None
stopwords = None

with open('results/output_corpus.txt', 'w') as outfile:
	for n in range(len(text)):
		if n not in checked:
			outfile.write('\n'.join(text))


