# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# Library Loading

import bs4 as bs
import urllib.request
import re
import pandas as pd
import nltk 
import string
string.punctuation
import nltk
nltk.download('punkt')
# nltk.download('stopwords')
# nltk.corpus.stopwords.words('english')
# -

class text_summary:
    """A library with  to get summary of the articles using multiple options"""
    pass


# clean punctuation, lower case, replace newline and extra white spaces
def clean_words(text):
    #removing stop words first
#   text = "".join([x for x in text if word not in stopwords])
    text  = "".join([char for char in text if char not in string.punctuation])
    text = re.sub('[0-9]+', '', text)
#     

    text = punctuation_rmv(text).lower().replace('\n', ' ').replace(" s ", " ")
    
    
    
    return text


# clean the text
def clean_data_text(data=data):
    # also included removing stop words part but not in this case later we will check
    data['tokenized_sents'] = data['tokenized_sents'].apply(clean_words)
    
    return data


# creates sentence list 
def create_sent(data=data):
    data['sentences_list'] = data['article_text'].apply(nltk.sent_tokenize)
    data['tokenized_sents'] = data['article_text'].astype(str)
    data = clean_data_text(data) # cleaning the article
    data['tokenized_sents_copy'] = data['tokenized_sents'] #creating copy of sentence without changes
    return data


def create_words_frequency_dictionary(data=data):
    from collections import Counter
    data['frequency'] = data['tokenized_sents_copy'].str.split(' ').apply(Counter)
    return data


def get_maximum_frequency(freq_dic):
    return max(freq_dic.values())


def create_word_weighted_freq(row_dictionary):
    max_weight = get_maximum_frequency(row_dictionary)
    for key, val in row_dictionary.items():
        row_dictionary[key] = val/max_weight
    return row_dictionary


#return max key value pair
def get_max_key_value(freq_dic):
    max_key_value_dic = {}
    k = max(freq_dic, key = freq_dic.get)
    max_key_value_dic[k] = freq_dic[k]
    return max_key_value_dic


#test_function
def insert_max_word_key_value(data=data):
    data['max'] = data['frequency'].apply(get_max_key_value)
    return data


# +
#calculate the sentence score
def calculate_sentence_score(sentence_list, 
                             word_weight_dictionary,
                            decided_len=30):
    sentence_score = {}
#     print('step 1')
    for sent in sentence_list:
#         print(f'sent: {sent}')
        if len(sent.split(' ')) < decided_len:
            for word in nltk.word_tokenize(sent):
#                 print(f'{word}')
                if sent not in sentence_score.keys():
                    
                    sentence_score[sent] = word_weight_dictionary[word]
#                     print(f'{sentence_score[sent]}')
#                     print(f'word searched {word}')
#                     print(f'word_weight_dictionary value First time:{sent} and value\n {word_weight_dictionary[word]}')
                else:
#                     print('why this is not printing')
                    sentence_score[sent] += word_weight_dictionary[word]
#                     print(f'word {word_weight_dictionary[word]} {sentence_score[word]}')
#                     print(f'word_weight_dictionary value other time time: {sent} and value\n {word_weight_dictionary[word]}')
    
#     sentence_score = sorted(sentence_score, key = sentence_score.get, reverse=True)
    return [sentence_score]

        
# -

#Data loading
data = pd.read_csv('tennis_articles.csv',encoding = 'unicode_escape')

# +
# create sentences

data = create_sent(data)
# create word frequency dictionary
data = create_words_frequency_dictionary(data)

data = insert_max_word_key_value(data)

# this function has one bug, it is changing the main column of frequency also.
data['weighted_freq'] = data['frequency'].apply(create_word_weighted_freq)
# -

data['sentences_scores_dict'] = 0
data['summary'] = 0
summary_sent_len = 7
data['summary'] = data['summary'].astype('O')
for i in range(data.shape[0]):
    dictionary_of_sent_scores = calculate_sentence_score(data['sentences_list'][i],
                                                         word_weight_dictionary=data['weighted_freq'][i]) 

    data.loc[i, 'sentences_scores_dict'] = dictionary_of_sent_scores
    data.at[data.index[i], 'summary'] = sorted(data.loc[data.index[i], 'sentences_scores_dict'],
                                 key = data.loc[data.index[i], 'sentences_scores_dict'].get)
    data.loc[i, 'summary'] = " ".join(list(data.loc[i, 'summary'])[:summary_sent_len])


data.head()

StarcoderdataJupyterScriptsDedupFiltered