code: from nltk.tokenize import RegexpTokenizer
  # from stop_words import get_stop_words
  from nltk.stem.porter import PorterStemmer
  from string import punctuation
  import re
  from nltk.corpus import stopwords
  en_stop = stopwords.words('english')
  from nltk.corpus import wordnet
  import html
  
  from common.commons import *
  CODE_PATH = os.environ["CODE_PATH"]
  
  import spacy
  nlp = spacy.load('en_core_web_lg', disable=['parser', 'tagger', 'ner'])
  nlp.max_length =100000000
  from sklearn.feature_extraction.text import TfidfVectorizer
  from sklearn.metrics.pairwise import cosine_similarity
  
  import sys
  
  def preprocessingCodeElementsList(res):
      printDetail = False
      if isinstance(res, list):
          merged = str()
          for r in res:
              if isinstance(r, list):
                  merged = merged + ' ' + ' '.join(r)
              else:
                  merged = merged +' ' + r
      else:
          merged=res
  
      res = html.unescape(merged)
  
      tokens = getTokens(res,printDetail)
  
      stripped = []
      for t in tokens:
          splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-',t)
          for s in splits:
              stripped.append(s)
      punc = removeEndingPunct(stripped,printDetail)
  
      non_empty = [i for i in punc if i != '']
  
      stripped = removeEndingPunct(non_empty,printDetail)
  
      camelCase = handleCamelCase(stripped,printDetail,True)
  
      underScore = handleUnderScore(camelCase,printDetail,True)
  
      lower = [i.lower() for i in underScore]
  
      stopped_tokens = [i for i in lower if not i in en_stop]
  
      stem2 = stem(stopped_tokens,printDetail)
      if printDetail:
          print('=====CLEANED=========')
          print(stem2)
  
      return stem2
  
  def preprocessingNL(res):
  
      printDetail = False
  
      if isinstance(res, list):
          merged = str()
          for r in res:
              if isinstance(r, list):
                  merged = merged + ' ' + ' '.join(r)
              else:
                  merged = merged +' ' + r
      else:
          merged=res
  
      res = html.unescape(merged)
      html_decoded_string = res.replace("&amp;", "&").replace("&quot;", '"').replace("&apos;", "'").replace("&gt;",
                                                                                                             ">").replace(
          "&lt;", "<")
      html_decoded_string = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '',html_decoded_string)
  
      tokens = getTokens(html_decoded_string,printDetail)
  
      stripped = []
      for t in tokens:
          splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-',t)
          for s in splits:
              stripped.append(s)
      punc = removeEndingPunct(stripped,printDetail)
  
      non_empty = [i for i in punc if i != '']
  
      stripped = removeEndingPunct(non_empty,printDetail)
  
      camelCase = handleCamelCase(stripped,printDetail,True)
  
      underScore = handleUnderScore(camelCase,printDetail,True)
  
      lower = [i.lower() for i in underScore]
  
      stopped_tokens = [i for i in lower if not i in en_stop]
  
      nonDigit = [i for i in stopped_tokens if (not i.isdigit())]
  
      doc = nlp(' '.join(nonDigit))
      newWord = []
      for token in doc:
          if(token.text in nlp.vocab):
              newWord.append(token.text)
  
      stem2 = stem(newWord,printDetail)
  
      if printDetail:
          print('=====CLEANED=========')
          print(stem2)
  
      return stem2
  
  def getTokens(re,printDetail=False):
      tokenizer = RegexpTokenizer(r'\S+')
      tokens = tokenizer.tokenize(re)
      if printDetail:
          print('=====TOKENS=========')
          print(tokens)
  
      return tokens
  
  def charLength(x, l=3):
      if x.isalpha() and len(x) >= l:
          return True
      else:
          return False
  
  
  def removeEndingPunct(re,printDetail):
      stripped = [i.strip(punctuation) for i in re]
      if printDetail:
          print('=====removeEndingPunct=========')
          print(stripped)
      return stripped
  
  def handleCamelCase(re,printDetail=False,keepOriginal = False):
      camelCased = list()
  
      for i in re:
          listOfCC = camel_case_split(i)
          camelCased.extend(listOfCC)
          if i not in listOfCC and keepOriginal:
              camelCased.append(i)
  
      if printDetail:
          print('=====CAMEL CASE=========')
          print(camelCased)
      return camelCased
  
  def handleUnderScore(re,printDetail=False,keepOriginal = False):
      underScored = list()
      for i in re:
          listOfCC = i.split('_')
          underScored.extend(listOfCC)
          if i not in listOfCC and keepOriginal:
              underScored.append(i)
  
      if printDetail:
          print('=====UNDER SCORE=========')
          print(underScored)
  
      return underScored
  
  def camel_case_split(identifier):
      matches = re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)', identifier)
      res = [m.group(0) for m in matches]
  
      return res
  
  def stem(res,printDetail):
      p_stemmer = PorterStemmer()
      stemmed_tokens = [p_stemmer.stem(i.strip()) for i in res if i]
      if printDetail:
          print('=====STEMMED=========')
          print(stemmed_tokens)
      return stemmed_tokens
  
  def isEnglish(word_to_test):
      if not wordnet.synsets(word_to_test):
          #Not an English Word
          #TODO
          word_to_test
          #print word_to_test
      else:
          return word_to_test
  
  
  def dummy_fun(doc):
      return doc
  
  def calculateTfIdfCodeElementsList(aCorpus):
      global progress
      progress = 0
      v = TfidfVectorizer(tokenizer=dummy_fun,stop_words=None,lowercase=False,sublinear_tf=True)#,max_df=0.7,min_df=3)
      m = v.fit(aCorpus)
      return v
  
  def calculateTfIdfNLList(aCorpus):
      global progress
      progress = 0
      v = TfidfVectorizer(tokenizer=dummy_fun,stop_words=None,lowercase=False,sublinear_tf=True)#,max_df=0.7,min_df=3)
      m = v.fit(aCorpus)
      return v
  
  def getDTMNL(x,v,corpus):
      ind =x.name
      v.tokenizer = dummy_fun
      return v.transform([corpus[ind]])
  def getDTMCE(x,v,corpus):
      ind =x.name
      v.tokenizer = dummy_fun
      return v.transform([corpus[ind]])
  
  def getBRDTM(x,v,corpus):
      ind =x.name
      v.tokenizer = dummy_fun
      return v.transform([corpus[ind]])
  
  
  def getBRDTMCEs(x,v,corpus):
      ind =x.name
      v.tokenizer = dummy_fun
      return v.transform([corpus[ind]])
  
apis: sklearn.feature_extraction.text.TfidfVectorizer