src/vectorizer.py
def load_vectorizer(tweets, test_tweets):
  """
  DESCRIPTION: 
        If there exists a cached tfidf file then it is loaded and returned. Otherwisem a new vectorizer is fittied
        by the gived data.
  INPUT: 
        tweets: Dataframe of a set of tweets
        test_tweets: Dataframe of a set of test_tweets
  OUTPUT: 
        train_reptweets: TFIDF representation of train tweets (dimensions: |train tweets| x |vocabulary|)  
        test_reptweets: TFIDF representation of test tweets (dimensions: |train tweets| x |vocabulary|)
  """
  import os.path
  if(os.path.exists(TFIDF_TRAIN_FILE) and os.path.exists(TFIDF_TRAIN_FILE)):
    f = open(TFIDF_TRAIN_FILE,'rb')
    train_reptweets = pickle.load(f)
    f = open(TFIDF_TEST_FILE,'rb')
    test_reptweets = pickle.load(f)
  else:
    tfidf = init_tfidf_vectorizer()
    train_reptweets = tfidf.fit_transform(tweets['tweet'])
    if algorithm['options']['TFIDF']['cache_tfidf']:
      f = open(TFIDF_TRAIN_FILE,'wb')
      pickle.dump(train_reptweets, f)
    test_reptweets = tfidf.transform(test_tweets['tweet'])
    if algorithm['options']['TFIDF']['cache_tfidf']:
      f = open(TFIDF_TEST_FILE,'wb')
      pickle.dump(test_reptweets, f)

  return train_reptweets, test_reptweets

  DESCRIPTION: 
        If there exists a cached tfidf file then it is loaded and returned. Otherwisem a new vectorizer is fittied
        by the gived data.
  INPUT: 
        tweets: Dataframe of a set of tweets
        test_tweets: Dataframe of a set of test_tweets
  OUTPUT: 
        train_reptweets: TFIDF representation of train tweets (dimensions: |train tweets| x |vocabulary|)  
        test_reptweets: TFIDF representation of test tweets (dimensions: |train tweets| x |vocabulary|)
  
If there exists a cached tfidf file then it is loaded and returned. Otherwisem a new vectorizer is fittied
by the gived data.
If there exists a cached tfidf file then it is loaded and returned . Otherwisem a new vectorizer is fittied by the gived data .
def load_vectorizer(tweets, test_tweets):
  import os.path
  if(os.path.exists(TFIDF_TRAIN_FILE) and os.path.exists(TFIDF_TRAIN_FILE)):
    f = open(TFIDF_TRAIN_FILE,'rb')
    train_reptweets = pickle.load(f)
    f = open(TFIDF_TEST_FILE,'rb')
    test_reptweets = pickle.load(f)
  else:
    tfidf = init_tfidf_vectorizer()
    train_reptweets = tfidf.fit_transform(tweets['tweet'])
    if algorithm['options']['TFIDF']['cache_tfidf']:
      f = open(TFIDF_TRAIN_FILE,'wb')
      pickle.dump(train_reptweets, f)
    test_reptweets = tfidf.transform(test_tweets['tweet'])
    if algorithm['options']['TFIDF']['cache_tfidf']:
      f = open(TFIDF_TEST_FILE,'wb')
      pickle.dump(test_reptweets, f)
  return train_reptweets, test_reptweets
def load_vectorizer ( tweets , test_tweets ) : import os . path if ( os . path . exists ( TFIDF_TRAIN_FILE ) and os . path . exists ( TFIDF_TRAIN_FILE ) ) : f = open ( TFIDF_TRAIN_FILE , 'rb' ) train_reptweets = pickle . load ( f ) f = open ( TFIDF_TEST_FILE , 'rb' ) test_reptweets = pickle . load ( f ) else : tfidf = init_tfidf_vectorizer ( ) train_reptweets = tfidf . fit_transform ( tweets [ 'tweet' ] ) if algorithm [ 'options' ] [ 'TFIDF' ] [ 'cache_tfidf' ] : f = open ( TFIDF_TRAIN_FILE , 'wb' ) pickle . dump ( train_reptweets , f ) test_reptweets = tfidf . transform ( test_tweets [ 'tweet' ] ) if algorithm [ 'options' ] [ 'TFIDF' ] [ 'cache_tfidf' ] : f = open ( TFIDF_TEST_FILE , 'wb' ) pickle . dump ( test_reptweets , f ) return train_reptweets , test_reptweets
DESCRIPTION:
If there exists a cached tfidf file then it is loaded and returned.
DESCRIPTION : If there exists a cached tfidf file then it is loaded and returned .
"""
  DESCRIPTION: 
        If there exists a cached tfidf file then it is loaded and returned. Otherwisem a new vectorizer is fittied
        by the gived data.
  INPUT: 
        tweets: Dataframe of a set of tweets
        test_tweets: Dataframe of a set of test_tweets
  OUTPUT: 
        train_reptweets: TFIDF representation of train tweets (dimensions: |train tweets| x |vocabulary|)  
        test_reptweets: TFIDF representation of test tweets (dimensions: |train tweets| x |vocabulary|)
  """
tweets
test_tweets