#!/usr/bin/python
# -*- coding: utf-8 -*-
import pandas as pd
import logging
import argparse
import os
import glob

from cptm.utils.inputgeneration import Perspective, remove_trailing_digits
from cptm.utils.dutchdata import pos_topic_words, pos_opinion_words, word_types
from cptm.utils.frog import get_frogclient, pos_and_lemmas

logger = logging.getLogger(__name__)
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)
logger.setLevel(logging.DEBUG)
logging.getLogger('inputgeneration').setLevel(logging.DEBUG)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('dir_in', help='directory containing the data '
                        '(manifesto project csv files)')
    parser.add_argument('dir_out', help='the name of the dir where the '
                        'CPT corpus should be saved.')
    args = parser.parse_args()

    dir_in = args.dir_in
    dir_out = args.dir_out

    frogclient = get_frogclient()

    if not os.path.exists(dir_out):
        os.makedirs(dir_out)

    p = Perspective('', pos_topic_words(), pos_opinion_words())

    data_files = glob.glob('{}/*.csv'.format(dir_in))

    for i, data_file in enumerate(data_files):
        if i % 5 == 0:
            logger.info('Processing text {} of {}'.format(i + 1,
                        len(data_files)))
        df = pd.read_csv(data_file, encoding='utf-8')
        text = ' '.join([line for line in df['content']])
        try:
            for pos, lemma in pos_and_lemmas(text, frogclient):
                if pos in word_types():
                    p.add(pos, remove_trailing_digits(lemma))
        except Exception, e:
            logger.warn(str(e))
            del frogclient
            frogclient = get_frogclient()
            logger.info('parsing pseudo sentences instead')
            for text in df['content']:
                for pos, lemma in pos_and_lemmas(text, frogclient):
                    if pos in word_types():
                        p.add(pos, remove_trailing_digits(lemma))

        file_name = os.path.basename(data_file).replace('.csv', '.txt')
        p.write2file(dir_out, file_name)

Add script to generate cptm corpus for manifesto data
Add script to generate cptm corpus for manifesto data

The script uses frog to parse text.
