# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import findspark      # @uthor <NAME>
findspark.init()      # <EMAIL>, 8851129962

from pyspark import SparkContext, SparkConf

conf = SparkConf().setAppName("remove crutch_wrd")

sc = SparkContext(conf=conf) 


def crutch_wrd(filename): #here a mere assumed file is taken     
    strg=[]               #we can do this for many files in one go and get the result
    f=open(filename,'r')  #like if we have a list of names of files we can map the function
    str=f.readlines()     #crutch_wrd in RDD and get resulted list of lines. 
    for i in str:         # which can be then mapped with function cr to get the desired result
        i=i.strip('\n')
        strg=strg+[i]
    return(strg)
    f.close()


k=crutch_wrd("ab.txt") #input text file in place of <file_name>

k_rdd=sc.parallelize(k)  #as k is a list we can convert it 
                        #into RDD in order parallize the task


def cr(x):
    c=0
    ls=['um','ah','you know','uh','actually','really','literally','suddenly','basically','definitely','very','truly','honestly','absolutely','totally','seriously','obviously','well','maybe','i guess','somehow','i suppose','regardless','nevertheless','for what it’s worth','er','so','um']
    x=x.lower
    for i in ls:
        if(i in x):
            c=c+1
    x1=x.split(" ")
    return(c/len(x1))


from operator import add

k1=k_rdd.map(cr).reduce(add)# k1 is the degree of crutch
                            #words used in whole text

StarcoderdataJupyterScriptsDedupFiltered