src/vectorizer.py
def init_tfidf_vectorizer():
  """
  DESCRIPTION: 
          Initializes the tfidf vectorizer by taking the parameters from options.py
  """
  print_dict_settings(algorithm['options']['TFIDF'], msg='tf-idf Vectorizer settings\n')
  
  if algorithm['options']['TFIDF']['number_of_stopwords'] != None:
      algorithm['options']['TFIDF']['number_of_stopwords'] = find_stopwords(number_of_stopwords=algorithm['options']['TFIDF']['number_of_stopwords'])
  if algorithm['options']['TFIDF']['tokenizer'] != None:
      algorithm['options']['TFIDF']['tokenizer'] = LemmaTokenizer()

  return TfidfVectorizer(
     min_df = algorithm['options']['TFIDF']['min_df'], 
     max_df = algorithm['options']['TFIDF']['max_df'], 
     sublinear_tf = algorithm['options']['TFIDF']['sublinear_tf'], 
     use_idf = algorithm['options']['TFIDF']['use_idf'],
     stop_words = algorithm['options']['TFIDF']['number_of_stopwords'], 
     tokenizer = algorithm['options']['TFIDF']['tokenizer'],
     ngram_range = algorithm['options']['TFIDF']['ngram_range'],
     max_features = algorithm['options']['TFIDF']['max_features']
  )

  DESCRIPTION: 
          Initializes the tfidf vectorizer by taking the parameters from options.py
  
Initializes the tfidf vectorizer by taking the parameters from options.py
Initializes the tfidf vectorizer by taking the parameters from options . py
def init_tfidf_vectorizer():
  print_dict_settings(algorithm['options']['TFIDF'], msg='tf-idf Vectorizer settings\n')
  if algorithm['options']['TFIDF']['number_of_stopwords'] != None:
      algorithm['options']['TFIDF']['number_of_stopwords'] = find_stopwords(number_of_stopwords=algorithm['options']['TFIDF']['number_of_stopwords'])
  if algorithm['options']['TFIDF']['tokenizer'] != None:
      algorithm['options']['TFIDF']['tokenizer'] = LemmaTokenizer()
  return TfidfVectorizer(
     min_df = algorithm['options']['TFIDF']['min_df'], 
     max_df = algorithm['options']['TFIDF']['max_df'], 
     sublinear_tf = algorithm['options']['TFIDF']['sublinear_tf'], 
     use_idf = algorithm['options']['TFIDF']['use_idf'],
     stop_words = algorithm['options']['TFIDF']['number_of_stopwords'], 
     tokenizer = algorithm['options']['TFIDF']['tokenizer'],
     ngram_range = algorithm['options']['TFIDF']['ngram_range'],
     max_features = algorithm['options']['TFIDF']['max_features']
  )
def init_tfidf_vectorizer ( ) : print_dict_settings ( algorithm [ 'options' ] [ 'TFIDF' ] , msg = 'tf-idf Vectorizer settings\n' ) if algorithm [ 'options' ] [ 'TFIDF' ] [ 'number_of_stopwords' ] != None : algorithm [ 'options' ] [ 'TFIDF' ] [ 'number_of_stopwords' ] = find_stopwords ( number_of_stopwords = algorithm [ 'options' ] [ 'TFIDF' ] [ 'number_of_stopwords' ] ) if algorithm [ 'options' ] [ 'TFIDF' ] [ 'tokenizer' ] != None : algorithm [ 'options' ] [ 'TFIDF' ] [ 'tokenizer' ] = LemmaTokenizer ( ) return TfidfVectorizer ( min_df = algorithm [ 'options' ] [ 'TFIDF' ] [ 'min_df' ] , max_df = algorithm [ 'options' ] [ 'TFIDF' ] [ 'max_df' ] , sublinear_tf = algorithm [ 'options' ] [ 'TFIDF' ] [ 'sublinear_tf' ] , use_idf = algorithm [ 'options' ] [ 'TFIDF' ] [ 'use_idf' ] , stop_words = algorithm [ 'options' ] [ 'TFIDF' ] [ 'number_of_stopwords' ] , tokenizer = algorithm [ 'options' ] [ 'TFIDF' ] [ 'tokenizer' ] , ngram_range = algorithm [ 'options' ] [ 'TFIDF' ] [ 'ngram_range' ] , max_features = algorithm [ 'options' ] [ 'TFIDF' ] [ 'max_features' ] )
DESCRIPTION:
Initializes the tfidf vectorizer by taking the parameters from options.py
DESCRIPTION : Initializes the tfidf vectorizer by taking the parameters from options . py
"""
  DESCRIPTION: 
          Initializes the tfidf vectorizer by taking the parameters from options.py
  """