# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import os
import pandas as pd
import nltk
import pickle as pkl
import re, string
#import unicodedata
import gensim
from gensim import corpora, models, similarities 
from collections import Counter
from sklearn import datasets

# +
DATA_DIR = os.getcwd()+'\\bbc'  #Windows's dir delimiter
#subdir are CATEGORIES = ['business', 'entertainment', 'politics', 'sport', 'tech']
CATEGORIES = os.listdir(path='./bbc')

#print(DATA_DIR)
print(CATEGORIES)
# -

#This will load all articles into a list of articles.
raw_text = datasets.load_files(DATA_DIR,
    categories=CATEGORIES, encoding="Latin-1")

#Let see the content of few articles
raw_text.data[0]

raw_text.target_names[3]


raw_text.data[3]


raw_text.target_names[1]

raw_text.data[1]

raw_text.target_names[2]

raw_text.data[2]

#there are 2225 text/news extracted, matches the dataset total files
len(raw_text.data)

#Save all text files to .plk for faster reload next time
pkl.dump(raw_text.data, open('raw_text.pkl','wb'))


raw_text_df = pd.DataFrame(raw_text.data,columns=['raw_text'])

raw_text.data[3]

raw_text.data[2]

clean_list= re.sub("[^a-zA-Z]", " ", raw_text.data[1].lower().strip())

clean_list.split(" ")

stopwords = nltk.corpus.stopwords.words('english')
print(sorted(stopwords))


wcount = Counter()
filtered_words= [word for word in clean_list[2].split() if word not in stopwords and len(word) > 2]  
wcount.update(filtered_words)
print(wcount.most_common(25))


# +
stopwords = nltk.corpus.stopwords.words('english') 
stemmer = nltk.stem.snowball.SnowballStemmer('english')

def news_to_tokenized_words (text):
    #  Basic text cleaning:  Lowering letter cases, and using alphabet without special characters.
    clean_text = re.sub("[^a-zA-Z]", " ", text.lower().strip())
    
    #  Filtering out Stop Words and words that are less than 3 chars
    filtered_words= [word for word in clean_text.split(" ") if word not in stopwords and len(word) > 2]

    #  Stemming 
    tokenized_words = [stemmer.stem(word) for word in filtered_words ]
    
    return tokenized_words


# -

raw_text_df.insert(raw_text_df.shape[1], 'clean_text', raw_text_df.raw_text.apply(news_to_tokenized_words))

#The tokenized list looks ok
raw_text_df.clean_text[1]

dictionary = corpora.Dictionary(raw_text_df.clean_text)
print(dictionary)


corpus=[dictionary.doc2bow(doc) for doc in raw_text_df.clean_text]

# %%time
ldamodel=gensim.models.LdaMulticore(corpus, num_topics=len(CATEGORIES), id2word=dictionary)


print(ldamodel.print_topics(num_topics=len(CATEGORIES), num_words=50))

# +
topics = ldamodel.show_topics(formatted=False, num_words=100)
    
for t in range(len(topics)):
    print("\nTopic {}, top {} words:".format(t+1,100))
    print(" ".join([w[0] for w in topics[t][1]]))
# -



StarcoderdataJupyterScriptsDedupFiltered