<filename>src/wikisource_parser/step1_wordform_to_lemmas_with_variants.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Convert wordform to lemmas in the database WCorpus (Wikisource).
# The order of words remains the same.
# There are several variants generated by pymorphy2 joined by the pipe symbol '|'.

import logging
import sys

import os
from os import listdir
from os.path import isfile, join

import re

import pymorphy2
morph = pymorphy2.MorphAnalyzer()

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

#source_text = u'Отправились в швейцарскую'
#source_text = u'Швейцара Макара, чтоб он не подслушал и не донес, поспешили услать в казначейство.'
#source_text = u'Макар взял рассыльную книгу, надел шапку, но в казначейство не пошел, а спрятался под лестницей: он знал, что бунт будет…'
#source_text = u'Сельский Вестник — еженедельная народная газета, издаваемая при «Правительственном Вестнике» с сентября 1881 г.'

#filename = '22999.txt'

#script_dir = os.path.dirname(__file__) #<-- absolute dir the script is in
#read_path  = "sentences/sentences1_source"
#write_path = "sentences/sentences2_lemmas"

abs_dir_path       = "/data/all/projects/git/wcorpus.addon/src.addon/sentences.short/sentences1.short" 
abs_dir_path_write = "/data/all/projects/git/wcorpus.addon/src.addon/sentences.short/sentences2_lemmas.short"

onlyfiles = [f for f in listdir(abs_dir_path) if isfile(join(abs_dir_path, f))]

i = 0
for filename in onlyfiles:
    i += 1
    file_path       = os.path.join(abs_dir_path,       filename)
    file_path_write = os.path.join(abs_dir_path_write, filename)
    print u"{0}. {1}".format(i, file_path);
    # print u" . {0}".format( file_path_write );

    file_out = open(file_path_write, 'w')

    lines = [line.rstrip('\n') for line in open(file_path)]
    for source_text in lines:

        source_text = source_text.decode('utf-8')
        #print "source_text=" + source_text
        result_text = "";

        # split text to words[]
        delim = ' \n\t,.!?:;';  # see http://stackoverflow.com/a/790600/1173350
        sentence_words = re.split("[" + delim + "]", source_text.lower())
        sentence_words = filter(None, sentence_words)           # remove empty strings
        #print "Words in sentence: " + u', '.join(sentence_words)

        # morph.parse(u'стали')
        lemmas = set()
        lemmas_list = []
        for word in sentence_words:
            b_prep = False
            current_lemmas = set()
            for p in morph.parse( word ):
                # if 'PREP' in p.tag:
                bad = ['PREP','CONJ']
                for item in bad:
                    if item in p.tag:
                        b_prep = True

                current_lemmas.add( p.normal_form )
                #print p.tag
                #print p.normal_form
                #print

            if not b_prep:                  # if one of tags is "PREPosition", then skip all forms (в -> в век)
                lemmas.update( current_lemmas )
                lemmas_list.extend( current_lemmas )

                if len(result_text) > 0:
                    result_text += u', '                # comma separated
                result_text += u'|'.join( current_lemmas ) 

        file_out.write( "# " + source_text.encode('utf-8') + "\n" )
        file_out.write( result_text.encode('utf-8') + "\n" )
        
        #print "Lemmas (set)  in sentence: " + u', '.join(lemmas)
        #print "Lemmas (list) in sentence: " + u', '.join(lemmas_list)
        #print "Lemmas (text) in sentence: " + result_text
        #print

    file_out.close()

#sys.exit("\nLet's stop and think.")
