yanrin/topic_modeling.py
def main():
    '''
    0. decide what date or range of dates to run this on
    1. create dictionary and corpus
    2. create model
    3. for each doc get top topic
    4. insert topic into topic table with date
    :return:
    '''
    datez = create_dates(2016)
    doc = []
    for date in datez:
        doc.extend(get_doc(date))
    doc_len = len(doc)
    train_texts = process_doc(doc)
    dictionary = process_dict(train_texts,doc_len)
    corpus = [dictionary.doc2bow(text) for text in train_texts]
    print('doc_len:',doc_len)
    print('corpus size:',len(corpus))
    print('lda+lsi model start!')
    num_times = 1
    lm, top_topics, high = ret_top_model(corpus,dictionary,train_texts,num_times)
    save_model = True
    load_model = False
    if save_model:
        lm.save('/Users/jaideepsingh/Projects/yanrin/lm2016Jan.ldamodel')
    if load_model:
        lm = LdaModel.load('/Users/jaideepsingh/Projects/yanrin/lm2016Jan.ldamodel')
    print('finished!')
    try:
        conn = psycopg2.connect("dbname=jaideepsingh user=jaideepsingh")
        cur = conn.cursor()
        for doc_id in range(doc_len):
            topics = get_doc_topics(doc, lm, dictionary, doc_id)
            print(topics)
            print('docid:',doc_id)
            print('mapping:',doc2id_mapping[doc_id])
            news_id = doc2id_mapping[doc_id]
            print('news_id:',news_id)
            if topics:
                for topic_name in topics:
                    print('Putting topic',topic_name,'into topics table')
                    insert_into_relevant_topic(cur, topic_name, news_id, date)
            conn.commit()
        cur.close()
        conn.close()
        print('db closed')
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()

    0. decide what date or range of dates to run this on
    1. create dictionary and corpus
    2. create model
    3. for each doc get top topic
    4. insert topic into topic table with date
    :return:
    
0. decide what date or range of dates to run this on
1. create dictionary and corpus
2. create model
3. for each doc get top topic
4. insert topic into topic table with date
0 . decide what date or range of dates to run this on 1 . create dictionary and corpus 2 . create model 3 . for each doc get top topic 4 . insert topic into topic table with date
def main():
    datez = create_dates(2016)
    doc = []
    for date in datez:
        doc.extend(get_doc(date))
    doc_len = len(doc)
    train_texts = process_doc(doc)
    dictionary = process_dict(train_texts,doc_len)
    corpus = [dictionary.doc2bow(text) for text in train_texts]
    print('doc_len:',doc_len)
    print('corpus size:',len(corpus))
    print('lda+lsi model start!')
    num_times = 1
    lm, top_topics, high = ret_top_model(corpus,dictionary,train_texts,num_times)
    save_model = True
    load_model = False
    if save_model:
        lm.save('/Users/jaideepsingh/Projects/yanrin/lm2016Jan.ldamodel')
    if load_model:
        lm = LdaModel.load('/Users/jaideepsingh/Projects/yanrin/lm2016Jan.ldamodel')
    print('finished!')
    try:
        conn = psycopg2.connect("dbname=jaideepsingh user=jaideepsingh")
        cur = conn.cursor()
        for doc_id in range(doc_len):
            topics = get_doc_topics(doc, lm, dictionary, doc_id)
            print(topics)
            print('docid:',doc_id)
            print('mapping:',doc2id_mapping[doc_id])
            news_id = doc2id_mapping[doc_id]
            print('news_id:',news_id)
            if topics:
                for topic_name in topics:
                    print('Putting topic',topic_name,'into topics table')
                    insert_into_relevant_topic(cur, topic_name, news_id, date)
            conn.commit()
        cur.close()
        conn.close()
        print('db closed')
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
def main ( ) : datez = create_dates ( 2016 ) doc = [ ] for date in datez : doc . extend ( get_doc ( date ) ) doc_len = len ( doc ) train_texts = process_doc ( doc ) dictionary = process_dict ( train_texts , doc_len ) corpus = [ dictionary . doc2bow ( text ) for text in train_texts ] print ( 'doc_len:' , doc_len ) print ( 'corpus size:' , len ( corpus ) ) print ( 'lda+lsi model start!' ) num_times = 1 lm , top_topics , high = ret_top_model ( corpus , dictionary , train_texts , num_times ) save_model = True load_model = False if save_model : lm . save ( '/Users/jaideepsingh/Projects/yanrin/lm2016Jan.ldamodel' ) if load_model : lm = LdaModel . load ( '/Users/jaideepsingh/Projects/yanrin/lm2016Jan.ldamodel' ) print ( 'finished!' ) try : conn = psycopg2 . connect ( "dbname=jaideepsingh user=jaideepsingh" ) cur = conn . cursor ( ) for doc_id in range ( doc_len ) : topics = get_doc_topics ( doc , lm , dictionary , doc_id ) print ( topics ) print ( 'docid:' , doc_id ) print ( 'mapping:' , doc2id_mapping [ doc_id ] ) news_id = doc2id_mapping [ doc_id ] print ( 'news_id:' , news_id ) if topics : for topic_name in topics : print ( 'Putting topic' , topic_name , 'into topics table' ) insert_into_relevant_topic ( cur , topic_name , news_id , date ) conn . commit ( ) cur . close ( ) conn . close ( ) print ( 'db closed' ) except ( Exception , psycopg2 . DatabaseError ) as error : print ( error ) finally : if conn is not None : conn . close ( )
0. decide what date or range of dates to run this on
1. create dictionary and corpus
2. create model
3. for each doc get top topic
4. insert topic into topic table with date
0 . decide what date or range of dates to run this on 1 . create dictionary and corpus 2 . create model 3 . for each doc get top topic 4 . insert topic into topic table with date
'''
    0. decide what date or range of dates to run this on
    1. create dictionary and corpus
    2. create model
    3. for each doc get top topic
    4. insert topic into topic table with date
    :return:
    '''
None