src/analyse/db_updater.py
def db_update(stopwords=True, bigrams=True):
    '''
    Clean up articles from the database, remove stopwords, extract bigrams and add a bigram column
    '''
    #get rows, per row ID and full text
    db = dbconnect()
    #Get all rows
    cur_all_rows = db.query("SELECT id, full_text, full_without_stop FROM artikelen", (), True)
    for row in tqdm(cur_all_rows):
        id = row[0]
        full = row[1]
        full_without_stop = row[2]
        
        #remove stop words if wanted
        if stopwords:
            counter = Counter(1)
            filtered = counter.remove_stopwords(counter.remove_special_characters(full, True))
            full_without_stop = filtered
            db.query(f"UPDATE artikelen SET full_without_stop = %s WHERE id = %s", (filtered, id))
        
        #Create bigrams if wanted
        if bigrams:
            counter=Counter(1)
            counter.all_text = full_without_stop
            grams = counter.get_bigrams()
            counted_bigrams = {}
            for k,v in grams.items():
                key = (f'{k[0]} {k[1]}')
                counted_bigrams[key] = v
            gram_str = (json.dumps(counted_bigrams))
            db.query(f'UPDATE artikelen SET bigrams = %s WHERE id = %s', (gram_str, id))
    db.close()

    Clean up articles from the database, remove stopwords, extract bigrams and add a bigram column
    
Clean up articles from the database, remove stopwords, extract bigrams and add a bigram column
Clean up articles from the database remove stopwords extract bigrams and add a bigram column
def db_update(stopwords=True, bigrams=True):
    db = dbconnect()
    cur_all_rows = db.query("SELECT id, full_text, full_without_stop FROM artikelen", (), True)
    for row in tqdm(cur_all_rows):
        id = row[0]
        full = row[1]
        full_without_stop = row[2]
        if stopwords:
            counter = Counter(1)
            filtered = counter.remove_stopwords(counter.remove_special_characters(full, True))
            full_without_stop = filtered
            db.query(f"UPDATE artikelen SET full_without_stop = %s WHERE id = %s", (filtered, id))
        if bigrams:
            counter=Counter(1)
            counter.all_text = full_without_stop
            grams = counter.get_bigrams()
            counted_bigrams = {}
            for k,v in grams.items():
                key = (f'{k[0]} {k[1]}')
                counted_bigrams[key] = v
            gram_str = (json.dumps(counted_bigrams))
            db.query(f'UPDATE artikelen SET bigrams = %s WHERE id = %s', (gram_str, id))
    db.close()
def db_update ( stopwords = True , bigrams = True ) : db = dbconnect ( ) cur_all_rows = db . query ( "SELECT id, full_text, full_without_stop FROM artikelen" , ( ) , True ) for row in tqdm ( cur_all_rows ) : id = row [ 0 ] full = row [ 1 ] full_without_stop = row [ 2 ] if stopwords : counter = Counter ( 1 ) filtered = counter . remove_stopwords ( counter . remove_special_characters ( full , True ) ) full_without_stop = filtered db . query ( f"UPDATE artikelen SET full_without_stop = %s WHERE id = %s" , ( filtered , id ) ) if bigrams : counter = Counter ( 1 ) counter . all_text = full_without_stop grams = counter . get_bigrams ( ) counted_bigrams = { } for k , v in grams . items ( ) : key = ( f'{k[0]} {k[1]}' ) counted_bigrams [ key ] = v gram_str = ( json . dumps ( counted_bigrams ) ) db . query ( f'UPDATE artikelen SET bigrams = %s WHERE id = %s' , ( gram_str , id ) ) db . close ( )
Clean up articles from the database, remove stopwords, extract bigrams and add a bigram column
Clean up articles from the database remove stopwords extract bigrams and add a bigram column
'''
    Clean up articles from the database, remove stopwords, extract bigrams and add a bigram column
    ''' #get rows, per row ID and full text #Get all rows #remove stop words if wanted #Create bigrams if wanted
stopwords
bigrams