text: import numpy as np
  import gensim
  from os import path
  from collections import defaultdict
  
  from sklearn.feature_extraction.text import TfidfVectorizer
  from sklearn.pipeline import Pipeline
  from sklearn.ensemble import ExtraTreesClassifier
  from sklearn.metrics import accuracy_score
  from sklearn.metrics import classification_report
  
  from tokenization import tag_subtitles
  from tokenization import randomize
  
  
  class MeanEmbeddingVectorizer(object):
      def __init__(self, word2vec):
          self.word2vec = word2vec
          # if a text is empty we should return a vector of zeros
          # with the same dimensionality as all the other vectors
          self.dim = len(word2vec.values())
  
      def fit(self, X, y):
          return self
  
      def transform(self, X):
          return np.array([
              np.mean([self.word2vec[w] for w in words if w in self.word2vec]
                      or [np.zeros(self.dim)], axis=0)
              for words in X
          ])
  
  
  class TfidfEmbeddingVectorizer(object):
      def __init__(self, word2vec):
          self.word2vec = word2vec
          self.word2weight = None
          self.dim = len(word2vec.values())
  
      def fit(self, X, y):
          tfidf = TfidfVectorizer(analyzer=lambda x: x)
          tfidf.fit(X)
          # if a word was never seen - it must be at least as infrequent
          # as any of the known words - so the default idf is the max of
          # known idf's
          max_idf = max(tfidf.idf_)
          self.word2weight = defaultdict(
              lambda: max_idf,
              [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()])
  
          return self
  
      def transform(self, X):
          return np.array([
                  np.mean([self.word2vec[w] * self.word2weight[w]
                           for w in words if w in self.word2vec] or
                          [np.zeros(self.dim)], axis=0)
                  for words in X
              ])
  
  
  with open("../../glove_model/glove.6B.50d.txt", "rb") as lines:
      w2v = {line.split()[0]: np.array(map(float, line.split()[1:]))
             for line in lines}
  
  # let X be a list of tokenized texts (i.e. list of lists of tokens)
  text, genre = tag_subtitles(path.relpath('CategoryData'))
  text, genre = randomize(text, genre)
  
  print('dataset size:', len(genre))
  test_size = 150
  X = text[test_size:]
  y = genre[test_size:]
  x_test = text[:test_size]
  y_test = genre[:test_size]
  
  model = gensim.models.Word2Vec(X, size=100)
  w2v = dict(zip(model.index2word, model.syn0))
  
  etree_w2v = Pipeline([
      ("word2vec vectorizer", MeanEmbeddingVectorizer(w2v)),
      ("extra trees", ExtraTreesClassifier(n_estimators=200))])
  etree_w2v_tfidf = Pipeline([
      ("word2vec vectorizer", TfidfEmbeddingVectorizer(w2v)),
      ("extra trees", ExtraTreesClassifier(n_estimators=200))])
  
  etree_w2v.fit(X, y)
  predicted = etree_w2v.predict(x_test)
  print(predicted)
  print(y_test)
  print(accuracy_score(y_test, predicted))
  print(classification_report(y_test, predicted))
  
  