src/tfidf_embdedding_vectorizer.py
def average_vectors(tweets, words, tfidf, X):
    """
    DESCRIPTION: 
            Given a pandas Dataframe of tweets and the trained word embedings (of some dimension d)
            this function returns the tweet embeddings of the same d dimension by just averaging the
            vectors of each word in the same tweet and multipliying the corresponding word embedding
            with it's tfidf value. 
    INPUT: 
            tweets: Dataframe of a set of tweets as a python strings
            words: python dictionary (word, vector of embeddings of the corresponding word)
    OUTPUT: 
            we_tweets: tweet embeddings of d dimension
    """
    we_tweets = np.zeros((tweets.shape[0], len(next(iter(words.values())))))
    for i, tweet in enumerate(tweets['tweet']):
        try:
            split_tweet = tweet.split()
        except:
            continue;

        foundEmbeddings = 0
        for word in split_tweet:
            try:
                try:
                    weight = X[i,tfidf.vocabulary_[word]]
                except:                    
                    weight = 1
                
                we_tweets[i] += words[word] * weight
                foundEmbeddings+=1
            except:
                if (not word.startswith("#")):
                    word = "#" + word
                tokens=split_hashtag_to_words(word)
                #print('before: ', word, ' after: ', split_hashtag_to_words(word))
                for token in tokens.split():
                    if((len(token) != 1) or (token == "a") or (token == "i")):
                        try:
                            try:
                                weight = X[i,tfidf.vocabulary_[token]]
                            except:                    
                                weight = 1
                            we_tweets[i] += words[token] * weight
                            foundEmbeddings+=1
                        except:
                            #print('Not found: ', token)
                            continue;
                continue;
        if (foundEmbeddings != 0):
            we_tweets[i] /= foundEmbeddings
    return we_tweets

    DESCRIPTION: 
            Given a pandas Dataframe of tweets and the trained word embedings (of some dimension d)
            this function returns the tweet embeddings of the same d dimension by just averaging the
            vectors of each word in the same tweet and multipliying the corresponding word embedding
            with it's tfidf value. 
    INPUT: 
            tweets: Dataframe of a set of tweets as a python strings
            words: python dictionary (word, vector of embeddings of the corresponding word)
    OUTPUT: 
            we_tweets: tweet embeddings of d dimension
    
Given a pandas Dataframe of tweets and the trained word embedings (of some dimension d)
this function returns the tweet embeddings of the same d dimension by just averaging the
vectors of each word in the same tweet and multipliying the corresponding word embedding
with it's tfidf value.
Given a pandas Dataframe of tweets and the trained word embedings ( of some dimension d ) this function returns the tweet embeddings of the same d dimension by just averaging the vectors of each word in the same tweet and multipliying the corresponding word embedding with it ' s tfidf value .
def average_vectors(tweets, words, tfidf, X):
    we_tweets = np.zeros((tweets.shape[0], len(next(iter(words.values())))))
    for i, tweet in enumerate(tweets['tweet']):
        try:
            split_tweet = tweet.split()
        except:
            continue;
        foundEmbeddings = 0
        for word in split_tweet:
            try:
                try:
                    weight = X[i,tfidf.vocabulary_[word]]
                except:                    
                    weight = 1
                we_tweets[i] += words[word] * weight
                foundEmbeddings+=1
            except:
                if (not word.startswith("#")):
                    word = "#" + word
                tokens=split_hashtag_to_words(word)
                for token in tokens.split():
                    if((len(token) != 1) or (token == "a") or (token == "i")):
                        try:
                            try:
                                weight = X[i,tfidf.vocabulary_[token]]
                            except:                    
                                weight = 1
                            we_tweets[i] += words[token] * weight
                            foundEmbeddings+=1
                        except:
                            continue;
                continue;
        if (foundEmbeddings != 0):
            we_tweets[i] /= foundEmbeddings
    return we_tweets
def average_vectors ( tweets , words , tfidf , X ) : we_tweets = np . zeros ( ( tweets . shape [ 0 ] , len ( next ( iter ( words . values ( ) ) ) ) ) ) for i , tweet in enumerate ( tweets [ 'tweet' ] ) : try : split_tweet = tweet . split ( ) except : continue ; foundEmbeddings = 0 for word in split_tweet : try : try : weight = X [ i , tfidf . vocabulary_ [ word ] ] except : weight = 1 we_tweets [ i ] += words [ word ] * weight foundEmbeddings += 1 except : if ( not word . startswith ( "#" ) ) : word = "#" + word tokens = split_hashtag_to_words ( word ) for token in tokens . split ( ) : if ( ( len ( token ) != 1 ) or ( token == "a" ) or ( token == "i" ) ) : try : try : weight = X [ i , tfidf . vocabulary_ [ token ] ] except : weight = 1 we_tweets [ i ] += words [ token ] * weight foundEmbeddings += 1 except : continue ; continue ; if ( foundEmbeddings != 0 ) : we_tweets [ i ] /= foundEmbeddings return we_tweets
DESCRIPTION:
Given a pandas Dataframe of tweets and the trained word embedings (of some dimension d)
this function returns the tweet embeddings of the same d dimension by just averaging the
vectors of each word in the same tweet and multipliying the corresponding word embedding
with it's tfidf value.
DESCRIPTION : Given a pandas Dataframe of tweets and the trained word embedings ( of some dimension d ) this function returns the tweet embeddings of the same d dimension by just averaging the vectors of each word in the same tweet and multipliying the corresponding word embedding with it ' s tfidf value .
"""
    DESCRIPTION: 
            Given a pandas Dataframe of tweets and the trained word embedings (of some dimension d)
            this function returns the tweet embeddings of the same d dimension by just averaging the
            vectors of each word in the same tweet and multipliying the corresponding word embedding
            with it's tfidf value. 
    INPUT: 
            tweets: Dataframe of a set of tweets as a python strings
            words: python dictionary (word, vector of embeddings of the corresponding word)
    OUTPUT: 
            we_tweets: tweet embeddings of d dimension
    """ #print('before: ', word, ' after: ', split_hashtag_to_words(word)) #print('Not found: ', token)
tweets
words
tfidf
X