# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
from io import StringIO
import nltk
import collections as co
import re

d = pd.read_csv("mortgage.csv",names=["complaintdate","product","subproduct" ,"issue","subissue","consumer_narrative","company_public_response","company","state" ,"zip","tags","companyconsent","via","send_to_company_date","company_response_consumer","timely","consumer_disputed","id"],skiprows=1) 
d=d[d['consumer_narrative'].notnull()]

s=StringIO()
d['consumer_narrative'].apply(lambda x: s.write(x))
k=s.getvalue()
s.close()
k=k.lower()
k=k.split()

words = co.Counter(nltk.corpus.words.words())
stopWords =co.Counter( nltk.corpus.stopwords.words() )
k=[i for i in k if i in words and i not in stopWords]
s=" ".join(k)
c = co.Counter(k)

c.most_common(10)

from nltk.tokenize import RegexpTokenizer 
tokenizer = RegexpTokenizer(r'\w+') 
d["tokens"] = d["consumer_narrative"].apply(tokenizer.tokenize)

import nltk
from nltk.corpus import stopwords 
from nltk.stem.porter import PorterStemmer 
stemmer = PorterStemmer() 
from nltk.stem import WordNetLemmatizer 
lemmatizer = WordNetLemmatizer();

mortgage_dict=['refinance','borrower','mortgage','lender','reporting','refinanced','financed','finance']


def stemandlemma(consumer_narrative):
    preprocess_list = list()
    words = consumer_narrative.lower().split()
    for w in words:
        if(w not in mortgage_dict):
            preprocess_list.append( lemmatizer.lemmatize(w) )
        else:
            preprocess_list.append(w)
    return " ".join(preprocess_list)


def removeNER(consumer_narrative):
    preprocess_list = list()
    words = consumer_narrative.lower().split()
    for w in words:
        preprocess_list.append( stemmer.stem(w) )
    return preprocess_list


def preprocess_consumer_narrative( consumer_narrative):
    preprocess_list = list()
    for text in consumer_narrative:
        letters_only = re.sub("[^a-zA-Z]", " ", text) 
        letters_only = re.sub("XX","",letters_only)
        words = letters_only.lower().split()                             
        stops = set(stopwords.words("english"))                  
        meaningful_words = [w for w in words if not w in stops]
        preprocess_list.append(" ".join( meaningful_words ))
    return preprocess_list 


def count_words(consumer_narrative):
    words = {}
    for w in consumer_narrative.split():
        words[w] =words.get(w,0) +1
        for w,c in words.consumer_narrative():
            print("%s: %d times" % (w,c))


d['consumer_newnarrative'] = preprocess_consumer_narrative( d.consumer_narrative)
d['consumer_newnarrative'] = d['consumer_newnarrative'].map(stemandlemma)

d.iloc[1].consumer_newnarrative

s=StringIO()
cf['consumer_newnarrative'].apply(lambda x: s.write(x))
k=s.getvalue()
s.close()
k=k.lower()
k=k.split()

words = co.Counter(nltk.corpus.words.words())
stopWords =co.Counter( nltk.corpus.stopwords.words() )
k=[i for i in k if i in words and i not in stopWords]
s=" ".join(k)
c = co.Counter(k)



StarcoderdataJupyterScriptsDedupFiltered