text: import utils
  import random
  import numpy as np
  from sklearn.ensemble import RandomForestClassifier
  from scipy.sparse import lil_matrix
  from sklearn.feature_extraction.text import TfidfTransformer
  
  # Performs classification using RandomForest classifier.
  
  FREQ_DIST_FILE = '../train-processed-freqdist.pkl'
  BI_FREQ_DIST_FILE = '../train-processed-freqdist-bi.pkl'
  TRAIN_PROCESSED_FILE = '../train-processed.csv'
  TEST_PROCESSED_FILE = '../test-processed.csv'
  TRAIN = True
  UNIGRAM_SIZE = 15000
  VOCAB_SIZE = UNIGRAM_SIZE
  USE_BIGRAMS = False
  if USE_BIGRAMS:
      BIGRAM_SIZE = 10000
      VOCAB_SIZE = UNIGRAM_SIZE + BIGRAM_SIZE
  FEAT_TYPE = 'presence'
  
  
  def get_feature_vector(tweet):
      uni_feature_vector = []
      bi_feature_vector = []
      words = tweet.split()
      for i in xrange(len(words) - 1):
          word = words[i]
          next_word = words[i + 1]
          if unigrams.get(word):
              uni_feature_vector.append(word)
          if USE_BIGRAMS:
              if bigrams.get((word, next_word)):
                  bi_feature_vector.append((word, next_word))
      if len(words) >= 1:
          if unigrams.get(words[-1]):
              uni_feature_vector.append(words[-1])
      return uni_feature_vector, bi_feature_vector
  
  
  def extract_features(tweets, batch_size=500, test_file=True, feat_type='presence'):
      num_batches = int(np.ceil(len(tweets) / float(batch_size)))
      for i in xrange(num_batches):
          batch = tweets[i * batch_size: (i + 1) * batch_size]
          features = lil_matrix((batch_size, VOCAB_SIZE))
          labels = np.zeros(batch_size)
          for j, tweet in enumerate(batch):
              if test_file:
                  tweet_words = tweet[1][0]
                  tweet_bigrams = tweet[1][1]
              else:
                  tweet_words = tweet[2][0]
                  tweet_bigrams = tweet[2][1]
                  labels[j] = tweet[1]
              if feat_type == 'presence':
                  tweet_words = set(tweet_words)
                  tweet_bigrams = set(tweet_bigrams)
              for word in tweet_words:
                  idx = unigrams.get(word)
                  if idx:
                      features[j, idx] += 1
              if USE_BIGRAMS:
                  for bigram in tweet_bigrams:
                      idx = bigrams.get(bigram)
                      if idx:
                          features[j, UNIGRAM_SIZE + idx] += 1
          yield features, labels
  
  
  def apply_tf_idf(X):
      transformer = TfidfTransformer(smooth_idf=True, sublinear_tf=True, use_idf=True)
      transformer.fit(X)
      return transformer
  
  
  def process_tweets(csv_file, test_file=True):
      """Returns a list of tuples of type (tweet_id, feature_vector)
              or (tweet_id, sentiment, feature_vector)
  
      Args:
          csv_file (str): Name of processed csv file generated by preprocess.py
          test_file (bool, optional): If processing test file
  
      Returns:
          list: Of tuples
      """
      tweets = []
      print 'Generating feature vectors'
      with open(csv_file, 'r') as csv:
          lines = csv.readlines()
          total = len(lines)
          for i, line in enumerate(lines):
              if test_file:
                  tweet_id, tweet = line.split(',')
              else:
                  tweet_id, sentiment, tweet = line.split(',')
              feature_vector = get_feature_vector(tweet)
              if test_file:
                  tweets.append((tweet_id, feature_vector))
              else:
                  tweets.append((tweet_id, int(sentiment), feature_vector))
              utils.write_status(i + 1, total)
      print '\n'
      return tweets
  
  
  if __name__ == '__main__':
      np.random.seed(1337)
      unigrams = utils.top_n_words(FREQ_DIST_FILE, UNIGRAM_SIZE)
      if USE_BIGRAMS:
          bigrams = utils.top_n_bigrams(BI_FREQ_DIST_FILE, BIGRAM_SIZE)
      tweets = process_tweets(TRAIN_PROCESSED_FILE, test_file=False)
      if TRAIN:
          train_tweets, val_tweets = utils.split_data(tweets)
      else:
          random.shuffle(tweets)
          train_tweets = tweets
      del tweets
      print 'Extracting features & training batches'
      clf = RandomForestClassifier(n_jobs=2, random_state=0)
      batch_size = len(train_tweets)
      i = 1
      n_train_batches = int(np.ceil(len(train_tweets) / float(batch_size)))
      for training_set_X, training_set_y in extract_features(train_tweets, test_file=False, feat_type=FEAT_TYPE, batch_size=batch_size):
          utils.write_status(i, n_train_batches)
          i += 1
          if FEAT_TYPE == 'frequency':
              tfidf = apply_tf_idf(training_set_X)
              training_set_X = tfidf.transform(training_set_X)
          clf.fit(training_set_X, training_set_y)
      print '\n'
      print 'Testing'
      if TRAIN:
          correct, total = 0, len(val_tweets)
          i = 1
          batch_size = len(val_tweets)
          n_val_batches = int(np.ceil(len(val_tweets) / float(batch_size)))
          for val_set_X, val_set_y in extract_features(val_tweets, test_file=False, feat_type=FEAT_TYPE, batch_size=batch_size):
              if FEAT_TYPE == 'frequency':
                  val_set_X = tfidf.transform(val_set_X)
              prediction = clf.predict(val_set_X)
              correct += np.sum(prediction == val_set_y)
              utils.write_status(i, n_val_batches)
              i += 1
          print '\nCorrect: %d/%d = %.4f %%' % (correct, total, correct * 100. / total)
      else:
          del train_tweets
          test_tweets = process_tweets(TEST_PROCESSED_FILE, test_file=True)
          n_test_batches = int(np.ceil(len(test_tweets) / float(batch_size)))
          predictions = np.array([])
          print 'Predicting batches'
          i = 1
          for test_set_X, _ in extract_features(test_tweets, test_file=True, feat_type=FEAT_TYPE):
              if FEAT_TYPE == 'frequency':
                  test_set_X = tfidf.transform(test_set_X)
              prediction = clf.predict(test_set_X)
              predictions = np.concatenate((predictions, prediction))
              utils.write_status(i, n_test_batches)
              i += 1
          predictions = [(str(j), int(predictions[j]))
                         for j in range(len(test_tweets))]
          utils.save_results_to_csv(predictions, 'randomforest.csv')
          print '\nSaved to randomforest.csv'
  