<NME> util.py
<BEF> import os
import pickle
import numpy as np
import chainer
from multiprocessing import Pool
from functools import partial
from chainer import Variable
from chainer.backends import cuda
from qanta.datasets.quiz_bowl import QuizBowlDataset
from qanta.guesser.abstract import AbstractGuesser
from qanta.util.constants import BUZZER_DEV_FOLD, BUZZER_TRAIN_FOLD

# constansts
N_GUESSES = 10
os.makedirs("output/buzzer", exist_ok=True)
dataset_dir = "output/buzzer/{}_data.pkl"


def vector_converter_0(guesses_sequence):
    """vector converter / feature extractor with only prob

    Args:
        guesses_sequence: a sequence (length of question) of list of guesses
            (n_guesses), each entry is (guess, prob)
    Returns:
        a sequence of vectors
    """
    length = len(guesses_sequence)
    prev_prob_vec = [0.0 for _ in range(N_GUESSES)]
    prev_dict = dict()

    vecs = []
    for i in range(length):
        prob_vec = []
        prob_diff_vec = []
        isnew_vec = []
        guesses = guesses_sequence[i]
        for guess, prob in guesses:
            prob_vec.append(prob)
            if i > 0 and guess in prev_dict:
                prev_prob = prev_dict[guess]
                prob_diff_vec.append(prob - prev_prob)
                isnew_vec.append(0)
            else:
                prob_diff_vec.append(prob)
                isnew_vec.append(1)
        if len(guesses) < N_GUESSES:
            for k in range(max(N_GUESSES - len(guesses), 0)):
                prob_vec.append(0)
                prob_diff_vec.append(0)
                isnew_vec.append(0)
        features = (
            prob_vec[:3]
            + isnew_vec[:3]
            + prob_diff_vec[:3]
            + [prob_vec[0] - prob_vec[1], prob_vec[1] - prob_vec[2]]
            + [prob_vec[0] - prev_prob_vec[0], prob_vec[1] - prev_prob_vec[1]]
            + [sum(isnew_vec[:5])]
            + [np.average(prob_vec), np.average(prev_prob_vec)]
            + [np.average(prob_vec[:6]), np.average(prev_prob_vec[:5])]
            + [np.var(prob_vec), np.var(prev_prob_vec)]
            + [np.var(prob_vec[:5]), np.var(prev_prob_vec[:5])]
        )
        vecs.append(np.array(features, dtype=np.float32))
        prev_prob_vec = prob_vec
        prev_dict = {g: p for g, p in guesses}
    return vecs


def vector_converter_1(guesses_sequence):
    """vector converter / feature extractor with both logit and prob

    return buzz_dict

def _process_question(option2id: Dict[str, int], 
        all_questions: Dict[int, Question], 
        word2vec, normalize, qnum, question) -> \
            Tuple[int, int, List[List[Dict[str, int]]], List[List[int]]]:
    '''Process one question.
    return:
    prev_logit_vec = [0.0 for _ in range(N_GUESSES)]
    prev_prob_vec = [0.0 for _ in range(N_GUESSES)]
    prev_dict = dict()

    vecs = []
    for i in range(length):
        logit_vec = []
        prob_vec = []
        logit_diff_vec = []
        prob_diff_vec = []
        isnew_vec = []
        guesses = guesses_sequence[i]
        for guess, logit, prob in guesses:
            logit_vec.append(logit)

    guess_dicts = []
    results = []
    wordvecs = None
    if word2vec is not None:
        wordvecs = []
    for pos, pos_group in question.groupby(['sentence', 'token']):
        pos_group = pos_group.groupby('guesser')
        guess_dicts.append([])
        results.append([])
        if word2vec is not None:
            wordvecs.append([])
        for guesser in GUESSERS:
            if guesser not in pos_group.groups:
                log.info("{0} missing guesser {1}.".format(qnum, guesser))
                guess_dicts[-1].append({})
                results[-1].append(0)
                if word2vec is not None:
                    wordvecs[-1].append(word2vec.get_zero_vec())
            else:
                guesses = pos_group.get_group(guesser)
                guesses = guesses.sort_values('score', ascending=False)
                top_guess = guesses.iloc[0].guess
                results[-1].append(int(top_guess == answer))
                
                s = sum(guesses.score) if normalize else 1
                dic = {x.guess: x.score / s for x in guesses.itertuples()}
                
                guess_dicts[-1].append(dic)
                if word2vec is not None:
                    wordvecs[-1].append(word2vec.get_avg_vec(top_guess))

    return qnum, answer_id, guess_dicts, results, wordvecs


class Word2Vec:

    def __init__(self, wordvec_dir, wordvec_dim):
        self.word2vec = gensim.models.KeyedVectors.load_word2vec_format(
                wordvec_dir, binary=True)
        self.wordvec_dim = wordvec_dim

    def get_avg_vec(self, guess):
        vecs = []
        for word in guess.split('_'):
            if word in self.word2vec:
                vecs.append(self.word2vec[word])
        if len(vecs) > 0:
            return sum(vecs) / len(vecs)
        return self.get_zero_vec()

    def get_zero_vec(self):
        return np.zeros(self.wordvec_dim, dtype=np.float32)


def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \
                    -> Tuple[Dict[str, int], Dict[str, list]]:
    merge_dfs()
    log.info('Loading data')
    question_db = QuestionDatabase()
    quizbowl_db = QuizBowlDataset(bc.MIN_ANSWERS, guesser_train=True, buzzer_train=True)
            + [np.var(logit_vec), np.var(prev_logit_vec)]
            + [np.var(prob_vec), np.var(prev_prob_vec)]
            + [np.var(logit_vec[:5]), np.var(prev_logit_vec[:5])]
            + [np.var(prob_vec[:5]), np.var(prev_prob_vec[:5])]
        )
        vecs.append(np.array(features, dtype=np.float32))
        prev_logit_vec = logit_vec
        prev_prob_vec = prob_vec
        prev_dict = {x: (y, z) for x, y, z in guesses}
    return vecs


def process_question(questions, vector_converter, item):
    """multiprocessing worker that converts the guesser output of a single
        question into format used by the buzzer
    """
    qid, q_rows = item
    qid = q_rows.qanta_id.tolist()[0]
    answer = questions[qid].page
    q_rows = q_rows.groupby("char_index")
    char_indices = sorted(q_rows.groups.keys())
    guesses_sequence = []
    labels = []
    for idx in char_indices:
        log.info('Processing {0} guesses'.format(fold))
        guesses = AbstractGuesser.load_guesses(bc.GUESSES_DIR, folds=[fold])

        worker = partial(_process_question, option2id, all_questions, word2vec, normalize)
        inputs = guesses.groupby('qnum')
        guesses_by_fold[fold] = _multiprocess(worker, inputs, info='df data',
                multi=True)
def read_data(
    fold,
    output_type="char",
    guesser_module="qanta.guesser.rnn",
    guesser_class="RnnGuesser",
    guesser_config_num=0,
    vector_converter=vector_converter_0,
):

    if os.path.isfile(dataset_dir.format(fold)):
        with open(dataset_dir.format(fold), "rb") as f:
            return pickle.load(f)

    g_dir = AbstractGuesser.output_path(
        guesser_module, guesser_class, guesser_config_num, ""
    )
    g_path = AbstractGuesser.guess_path(g_dir, fold, output_type)
    with open(g_path, "rb") as f:
        df = pickle.load(f)
    df_groups = df.groupby("qanta_id")

    questions = QuizBowlDataset(buzzer_train=True).questions_by_fold()
    questions = {q.qanta_id: q for q in questions[fold]}

    pool = Pool(8)
    worker = partial(process_question, questions, vector_converter)
    dataset = pool.map(worker, df_groups)

    with open(dataset_dir.format(fold), "wb") as f:
        pickle.dump(dataset, f)

    return dataset


def convert_seq(batch, device=None):
    def to_device_batch(batch):
        if device is None:
            return batch
        elif device < 0:
            return [chainer.dataset.to_device(device, x) for x in batch]
        else:
            xp = cuda.cupy.get_array_module(*batch)
            concat = xp.concatenate(batch, axis=0)
            sections = np.cumsum([len(x) for x in batch[:-1]], dtype=np.int32)
            concat_dev = chainer.dataset.to_device(device, concat)
            batch_dev = cuda.cupy.split(concat_dev, sections)
            return batch_dev

    qids, vectors, labels, positions = list(map(list, zip(*batch)))
    xs = [Variable(x) for x in to_device_batch(vectors)]
    ys = to_device_batch(labels)
    return {"xs": xs, "ys": ys}


if __name__ == "__main__":
    data = read_data(BUZZER_TRAIN_FOLD)
    print(data)
if __name__ == "__main__":
    merge_dfs()

    processed_dirs = ['%s_processed.pickle' % (os.path.join(
        bc.GUESSES_DIR, fold)) for fold in c.BUZZER_INPUT_FOLDS]
    # if not all(os.path.isfile(d) for d in processed_dirs):
    #     log.info('Loading {0}'.format(bc.WORDVEC_DIR))
    #     word2vec = Word2Vec(bc.WORDVEC_DIR, bc.WORDVEC_DIM)
    # else:
    word2vec = None

    option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS, word2vec)
    load_protobowl()

<MSG> don't normalize!

<DFF> @@ -73,8 +73,7 @@ def stupid_buzzer(iterator) -> Dict[int, int]:
     return buzz_dict
 
 def _process_question(option2id: Dict[str, int], 
-        all_questions: Dict[int, Question], 
-        word2vec, normalize, qnum, question) -> \
+        all_questions: Dict[int, Question], qnum, question) -> \
             Tuple[int, int, List[List[Dict[str, int]]], List[List[int]]]:
     '''Process one question.
     return:
@@ -95,61 +94,32 @@ def _process_question(option2id: Dict[str, int],
 
     guess_dicts = []
     results = []
-    wordvecs = None
-    if word2vec is not None:
-        wordvecs = []
     for pos, pos_group in question.groupby(['sentence', 'token']):
         pos_group = pos_group.groupby('guesser')
         guess_dicts.append([])
         results.append([])
-        if word2vec is not None:
-            wordvecs.append([])
         for guesser in GUESSERS:
             if guesser not in pos_group.groups:
                 log.info("{0} missing guesser {1}.".format(qnum, guesser))
                 guess_dicts[-1].append({})
                 results[-1].append(0)
-                if word2vec is not None:
-                    wordvecs[-1].append(word2vec.get_zero_vec())
             else:
                 guesses = pos_group.get_group(guesser)
                 guesses = guesses.sort_values('score', ascending=False)
                 top_guess = guesses.iloc[0].guess
                 results[-1].append(int(top_guess == answer))
                 
-                s = sum(guesses.score) if normalize else 1
+                # s = sum(guesses.score)
+                s = 1
                 dic = {x.guess: x.score / s for x in guesses.itertuples()}
                 
                 guess_dicts[-1].append(dic)
-                if word2vec is not None:
-                    wordvecs[-1].append(word2vec.get_avg_vec(top_guess))
 
-    return qnum, answer_id, guess_dicts, results, wordvecs
+    return qnum, answer_id, guess_dicts, results
 
-
-class Word2Vec:
-
-    def __init__(self, wordvec_dir, wordvec_dim):
-        self.word2vec = gensim.models.KeyedVectors.load_word2vec_format(
-                wordvec_dir, binary=True)
-        self.wordvec_dim = wordvec_dim
-
-    def get_avg_vec(self, guess):
-        vecs = []
-        for word in guess.split('_'):
-            if word in self.word2vec:
-                vecs.append(self.word2vec[word])
-        if len(vecs) > 0:
-            return sum(vecs) / len(vecs)
-        return self.get_zero_vec()
-
-    def get_zero_vec(self):
-        return np.zeros(self.wordvec_dim, dtype=np.float32)
-
-
-def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \
+def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, normalize=True) \
                     -> Tuple[Dict[str, int], Dict[str, list]]:
-    merge_dfs()
+    # merge_dfs()
     log.info('Loading data')
     question_db = QuestionDatabase()
     quizbowl_db = QuizBowlDataset(bc.MIN_ANSWERS, guesser_train=True, buzzer_train=True)
@@ -180,7 +150,7 @@ def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \
         log.info('Processing {0} guesses'.format(fold))
         guesses = AbstractGuesser.load_guesses(bc.GUESSES_DIR, folds=[fold])
 
-        worker = partial(_process_question, option2id, all_questions, word2vec, normalize)
+        worker = partial(_process_question, option2id, all_questions)
         inputs = guesses.groupby('qnum')
         guesses_by_fold[fold] = _multiprocess(worker, inputs, info='df data',
                 multi=True)
@@ -285,13 +255,6 @@ def load_protobowl():
 if __name__ == "__main__":
     merge_dfs()
 
-    processed_dirs = ['%s_processed.pickle' % (os.path.join(
-        bc.GUESSES_DIR, fold)) for fold in c.BUZZER_INPUT_FOLDS]
-    # if not all(os.path.isfile(d) for d in processed_dirs):
-    #     log.info('Loading {0}'.format(bc.WORDVEC_DIR))
-    #     word2vec = Word2Vec(bc.WORDVEC_DIR, bc.WORDVEC_DIM)
-    # else:
-    word2vec = None
-
-    option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS, word2vec)
-    load_protobowl()
+    # option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS)
+    # load_protobowl()
+    option2id, guesses_by_fold = load_quizbowl()

{'commit': 'diff --git a/qanta/buzzer/configs.py b/qanta/buzzer/configs.py\nindex 47800d3..a440c31 100644\n--- a/qanta/buzzer/configs.py\n+++ b/qanta/buzzer/configs.py\n@@ -7,7 +7,56 @@ class mlp():\n     step_size = 1\n     neg_weight = 1\n     batch_norm = False\n-    model_name = \'mlp_{}\'.format(n_hidden)\n+    make_vector = \'dense_vector_0\'\n+    model_name = \'rnn.{}.{}\'.format(n_hidden, make_vector)\n+    ckp_dir = \'output/buzzer/{}.ckp\'.format(model_name)\n+    model_dir = \'output/buzzer/{}.npz\'.format(model_name)\n+\n+class rnn_200_0():\n+    max_grad_norm = 5\n+    batch_size = 128\n+    n_hidden = 200\n+    dropout = 0\n+    step_size = 1\n+    neg_weight = 1\n+    make_vector = \'dense_vector_0\'\n+    model_name = \'rnn.{}.{}\'.format(n_hidden, make_vector)\n+    ckp_dir = \'output/buzzer/{}.ckp\'.format(model_name)\n+    model_dir = \'output/buzzer/{}.npz\'.format(model_name)\n+\n+class new_rnn_200():\n+    max_grad_norm = 5\n+    batch_size = 128\n+    n_hidden = 200\n+    dropout = 0\n+    step_size = 1\n+    neg_weight = 1\n+    make_vector = \'dense_vector_0\'\n+    model_name = \'new_rnn_200\'\n+    ckp_dir = \'output/buzzer/{}.ckp\'.format(model_name)\n+    model_dir = \'output/buzzer/{}.npz\'.format(model_name)\n+\n+class new_rnn_300():\n+    max_grad_norm = 5\n+    batch_size = 128\n+    n_hidden = 300\n+    dropout = 0\n+    step_size = 1\n+    neg_weight = 1\n+    make_vector = \'dense_vector_0\'\n+    model_name = \'new_rnn_300\'\n+    ckp_dir = \'output/buzzer/{}.ckp\'.format(model_name)\n+    model_dir = \'output/buzzer/{}.npz\'.format(model_name)\n+\n+class rnn_200_1():\n+    max_grad_norm = 5\n+    batch_size = 128\n+    n_hidden = 200\n+    dropout = 0\n+    step_size = 1\n+    neg_weight = 1\n+    make_vector = \'dense_vector_1\'\n+    model_name = \'rnn.{}.{}\'.format(n_hidden, make_vector)\n     ckp_dir = \'output/buzzer/{}.ckp\'.format(model_name)\n     model_dir = \'output/buzzer/{}.npz\'.format(model_name)\n \ndiff --git a/qanta/buzzer/cost_sensitive.py b/qanta/buzzer/cost_sensitive.py\nindex f6e3095..d26ad5f 100644\n--- a/qanta/buzzer/cost_sensitive.py\n+++ b/qanta/buzzer/cost_sensitive.py\n@@ -13,38 +13,26 @@ from qanta.buzzer import configs\n from qanta.buzzer.progress import ProgressBar\n from qanta.buzzer.trainer import Trainer\n from qanta.buzzer.iterator import QuestionIterator\n+from qanta.buzzer import iterator \n from qanta.buzzer.util import load_quizbowl, GUESSERS\n from qanta.buzzer.models import MLP, RNN\n from qanta.buzzer import constants as bc\n from qanta.util import constants as c\n-from qanta import logging\n-\n-import os\n-import sys\n-import random\n-import numpy as np\n-import pickle\n-\n \n log = logging.get(__name__)\n \n def train_cost_sensitive(config, folds):\n     N_GUESSERS = len(GUESSERS)\n     cfg = getattr(configs, config)()\n+    make_vector = getattr(iterator, cfg.make_vector)\n \n     option2id, all_guesses = load_quizbowl()\n     iterators = dict()\n     for fold in c.BUZZER_INPUT_FOLDS:\n         iterators[fold] = QuestionIterator(all_guesses[fold], option2id,\n-            batch_size=cfg.batch_size)\n-\n-    if isinstance(cfg, configs.mlp):\n-        model = MLP(n_input=eval_iter.n_input, n_hidden=cfg.n_hidden,\n-                n_output=N_GUESSERS + 1, n_layers=cfg.n_layers, \n-                dropout=cfg.dropout, batch_norm=cfg.batch_norm)\n+            batch_size=cfg.batch_size, make_vector=make_vector)\n \n-    if isinstance(cfg, configs.rnn):\n-        model = RNN(eval_iter.n_input, cfg.n_hidden, N_GUESSERS + 1)\n+    model = RNN(eval_iter.n_input, cfg.n_hidden, N_GUESSERS + 1)\n \n     gpu = conf[\'buzzer\'][\'gpu\']\n     if gpu != -1 and chainer.cuda.available:\n@@ -65,9 +53,12 @@ def train_cost_sensitive(config, folds):\n             pickle.dump(buzzes, outfile)\n         log.info(\'Buzzes saved to {0}.\'.format(buzzes_dir))\n \n+        if fold == \'expo\':\n+            buzzer2vwexpo(all_guesses[\'expo\'], buzzes, fold)\n+\n def parse_args():\n     parser = argparse.ArgumentParser()\n-    parser.add_argument(\'-c\', \'--config\', type=str, default=\'rnn\')\n+    parser.add_argument(\'-c\', \'--config\', type=str, default=\'rnn_200_1\')\n     parser.add_argument(\'-f\', \'--fold\', type=str, default=None)\n     return parser.parse_args()\n \ndiff --git a/qanta/buzzer/interface.py b/qanta/buzzer/interface.py\nindex 5f8e1e9..35e2fae 100644\n--- a/qanta/buzzer/interface.py\n+++ b/qanta/buzzer/interface.py\n@@ -11,6 +11,7 @@ from multiprocessing import Pool, Manager\n from typing import List, Dict, Tuple, Optional\n \n from qanta.util import constants as c\n+from qanta.buzzer import constants as bc\n from qanta.util.io import safe_path\n from qanta.config import conf\n from qanta.datasets.quiz_bowl import QuestionDatabase\n@@ -122,3 +123,11 @@ def buzzer2vwexpo(guesses_df: pd.DataFrame,\n                 itertools.chain(*metaf))\n         meta_file.write(meta_out)\n         log.info(\'vw_meta file written\')\n+\n+if __name__ == \'__main__\':\n+    model_name = \'neo_0\'\n+    guesses_df = AbstractGuesser.load_guesses(bc.GUESSES_DIR, folds=[\'expo\'])\n+    expo_buzzes_dir = \'output/buzzer/neo/expo_buzzes.{}.pkl\'.format(model_name)\n+    with open(expo_buzzes_dir, \'rb\') as f:\n+        expo_buzzes = pickle.load(f)\n+    buzzer2vwexpo(guesses_df, expo_buzzes, \'expo\')\ndiff --git a/qanta/buzzer/iterator.py b/qanta/buzzer/iterator.py\nindex 93fc87d..fe84005 100644\n--- a/qanta/buzzer/iterator.py\n+++ b/qanta/buzzer/iterator.py\n@@ -18,8 +18,8 @@ N_GUESSES = conf[\'buzzer\'][\'n_guesses\']\n \n log = logging.get(__name__)\n \n-def dense_vector(dicts: List[List[Dict[str, float]]],\n-        wordvecs: List[List[np.ndarray]], step_size=1) -> List[List[float]]:\n+def dense_vector_0(dicts: List[List[Dict[str, float]]], \n+        step_size=1) -> List[List[float]]:\n \n     def get_guesser_acc(i, length):\n         if i == length:\n@@ -79,6 +79,49 @@ def dense_vector(dicts: List[List[Dict[str, float]]],\n             prev_vecs = prev_vecs[-step_size:]\n     return vecs\n \n+def dense_vector_1(dicts: List[List[Dict[str, float]]],\n+        step_size=1) -> List[List[float]]:\n+\n+    length = len(dicts)\n+    prev_vec = [0.02 for _ in range(N_GUESSERS * N_GUESSES)]\n+    vecs = []\n+    for i in range(length):\n+        if len(dicts[i]) != N_GUESSERS:\n+            raise ValueError("Inconsistent number of guessers ({0}, {1}).".format(\n+                N_GUESSERS, len(dicts)))\n+        vec = []\n+        diff_vec = []\n+        isnew_vec = []\n+        for j in range(N_GUESSERS):\n+            dic = sorted(dicts[i][j].items(), key=lambda x: x[1], reverse=True)\n+            for guess, score in dic:\n+                vec.append(score)\n+                if i > 0 and guess in dicts[i-1][j]:\n+                    diff_vec.append(score - dicts[i-1][j][guess])\n+                    isnew_vec.append(0)\n+                else:\n+                    diff_vec.append(score) \n+                    isnew_vec.append(1)\n+            if len(dic) < N_GUESSES:\n+                for k in range(max(N_GUESSES - len(dic), 0)):\n+                    vec.append(0)\n+                    diff_vec.append(0)\n+                    isnew_vec.append(0)\n+        features = [vec[0], vec[1], vec[2],\n+                    np.average(vec[:10]), np.average(prev_vec[:10]),\n+                    np.var(vec[:10]), np.var(prev_vec[:10]),\n+                    sum(isnew_vec[:10]),\n+                    isnew_vec[0], isnew_vec[1], isnew_vec[2],\n+                    diff_vec[0], diff_vec[1],\n+                    vec[0] - vec[1], vec[1] - vec[2], \n+                    vec[0] / vec[1], vec[0] / prev_vec[0],\n+                    vec[0] - prev_vec[0], vec[1] - prev_vec[1]\n+                    ]\n+\n+        vecs.append(features)\n+        prev_vec = vec\n+    return vecs\n+\n class QuestionIterator(object):\n     \'\'\'Each batch contains:\n         qids: list, (batch_size,)\n@@ -89,7 +132,7 @@ class QuestionIterator(object):\n     \'\'\'\n \n     def __init__(self, dataset: list, option2id: Dict[str, int], batch_size:int,\n-            make_vector=dense_vector, bucket_size=4, step_size=1, neg_weight=1, shuffle=True):\n+            make_vector=dense_vector_1, bucket_size=4, step_size=1, neg_weight=1, shuffle=True):\n         self.dataset = dataset\n         self.option2id = option2id\n         self.batch_size = batch_size\n@@ -107,7 +150,7 @@ class QuestionIterator(object):\n         self.create_batches()\n         # log.info(\'Finish creating batches {}\'.format(self.n_input))\n \n-    def _process_example(self, qid, answer, dicts, results, wordvecs):\n+    def _process_example(self, qid, answer, dicts, results):\n         \n         results = np.asarray(results, dtype=np.int32)\n         length, n_guessers = results.shape\n@@ -127,7 +170,7 @@ class QuestionIterator(object):\n \n         if len(dicts) != length:\n             raise ValueError("Inconsistant shape of results and vecs.")\n-        vecs = self.make_vector(dicts, wordvecs, self.step_size)\n+        vecs = self.make_vector(dicts, self.step_size)\n         vecs = np.asarray(vecs, dtype=np.float32)\n         assert length == vecs.shape[0]\n         self.n_input = len(vecs[0])\ndiff --git a/qanta/buzzer/report.py b/qanta/buzzer/report.py\nnew file mode 100644\nindex 0000000..4592d21\n--- /dev/null\n+++ b/qanta/buzzer/report.py\n@@ -0,0 +1,57 @@\n+import os\n+import pickle\n+import argparse\n+import numpy as np\n+from itertools import cycle\n+from collections import defaultdict\n+from functools import partial\n+from typing import List, Dict, Tuple\n+\n+from qanta.guesser.abstract import AbstractGuesser\n+from qanta.datasets.quiz_bowl import QuestionDatabase\n+from qanta.util import constants as c\n+from qanta.buzzer import constants as bc\n+from qanta import logging\n+from qanta.buzzer.util import GUESSERS, load_protobowl\n+from qanta.reporting.report_generator import ReportGenerator\n+from qanta.util.multiprocess import _multiprocess\n+\n+import matplotlib\n+matplotlib.use(\'Agg\')\n+import matplotlib.pyplot as plt\n+\n+from qanta.reporting.new_performance import get_protobowl, _get_top_guesses\n+\n+def report(buzzes_dir):\n+    all_questions = QuestionDatabase().all_questions()\n+    answers = {k: v.page for k, v in all_questions.items()}\n+    question_texts = {k: v.text for k, v in all_questions.items()}\n+    protobowl_ids = {k: all_questions[k].protobowl \n+                for k in all_questions if all_questions[k].protobowl != \'\'}\n+    protobowl_df, user_count = load_protobowl()\n+    guesses_df = AbstractGuesser.load_guesses(bc.GUESSES_DIR,\n+            folds=[c.BUZZER_DEV_FOLD])\n+    questions = guesses_df.groupby(\'qnum\')\n+    top_guesses = _multiprocess(_get_top_guesses, questions, info=\'Top guesses\', multi=True)\n+    top_guesses = {k: v for k, v in top_guesses}\n+    \n+    with open(buzzes_dir, \'rb\') as infile:\n+        buzzes = pickle.load(infile)\n+            \n+    save_dir = \'output/summary/new_performance/\'\n+    inputs = [top_guesses, buzzes, answers, None, c.BUZZER_DEV_FOLD, save_dir]\n+    user_answers_thresholds = [1,10,50,100,500,1000,2000]\n+    threshold_stats = []\n+    for threshold in user_answers_thresholds:\n+        pdf1 = protobowl_df[protobowl_df.user_answers > threshold]\n+        p_inputs = [question_texts, protobowl_ids, pdf1.groupby(\'qid\'), questions] + inputs\n+        pstats = get_protobowl(p_inputs)\n+        threshold_stats.append(pstats)\n+        print(threshold, stats)\n+    with open(buzzes_dir + \'.pstats\', \'wb\') as f:\n+        pickle.dump(threshold_stats, f)\n+    print([x[\'reward\'] for x in threshold_stats])\n+\n+if __name__ == \'__main__\':\n+    buzzes_dir = \'output/buzzer/neo/dev_buzzes.neo_0.pkl\'\n+    report(buzzes_dir)\ndiff --git a/qanta/buzzer/rnn_0.py b/qanta/buzzer/rnn_0.py\nnew file mode 100644\nindex 0000000..75d9990\n--- /dev/null\n+++ b/qanta/buzzer/rnn_0.py\n@@ -0,0 +1,107 @@\n+import os\n+import numpy as np\n+import argparse\n+import chainer\n+import pickle\n+from collections import defaultdict, namedtuple\n+from typing import List, Dict, Tuple, Optional\n+\n+from qanta import logging\n+from qanta.config import conf\n+from qanta.guesser.abstract import AbstractGuesser\n+\n+from qanta.buzzer import configs\n+from qanta.buzzer.progress import ProgressBar\n+from qanta.buzzer.trainer import Trainer\n+from qanta.buzzer.iterator import QuestionIterator\n+from qanta.buzzer import iterator \n+from qanta.buzzer.util import load_quizbowl, GUESSERS\n+from qanta.buzzer.models import MLP, RNN\n+from qanta.buzzer import constants as bc\n+from qanta.buzzer.report import report\n+from qanta.util import constants as c\n+\n+N_GUESSERS = len(GUESSERS)\n+N_GUESSES = 10\n+\n+log = logging.get(__name__)\n+\n+\n+def dense_vector(dicts: List[List[Dict[str, float]]],\n+        step_size=1) -> List[List[float]]:\n+    length = len(dicts)\n+    prev_vec = [0. for _ in range(N_GUESSERS * N_GUESSES)]\n+            \n+    vecs = []\n+    for i in range(length):\n+        if len(dicts[i]) != N_GUESSERS:\n+            raise ValueError("Inconsistent number of guessers ({0}, {1}).".format(\n+                N_GUESSERS, len(dicts)))\n+        vec = []\n+        diff_vec = []\n+        isnew_vec = []\n+        for j in range(N_GUESSERS):\n+            dic = sorted(dicts[i][j].items(), key=lambda x: x[1],\n+                    reverse=True)[:N_GUESSES]\n+            for guess, score in dic:\n+                vec.append(score)\n+                if i > 0 and guess in dicts[i-1][j]:\n+                    diff_vec.append(score - dicts[i-1][j][guess])\n+                    isnew_vec.append(0)\n+                else:\n+                    diff_vec.append(score) \n+                    isnew_vec.append(1)\n+            if len(dic) < N_GUESSES:\n+                for k in range(max(N_GUESSES - len(dic), 0)):\n+                    vec.append(0)\n+                    diff_vec.append(0)\n+                    isnew_vec.append(0)\n+        # guesser_acc = get_guesser_acc(i, length)\n+        features = [sum(isnew_vec), np.average(vec), vec[0], vec[1], vec[2],\n+                isnew_vec[0], isnew_vec[1], vec[0] - vec[1], vec[1] -\n+                vec[2], isnew_vec[2], diff_vec[0], \n+                vec[0] - prev_vec[0], np.var(vec),\n+                np.var(prev_vec)]\n+                # i, int(i < 10), int(i < 20), int(i > 30),\n+                # guesser_acc]\n+\n+        vecs.append(features)\n+        prev_vec = vec\n+    return vecs\n+\n+def main():\n+    option2id, all_guesses = load_quizbowl()\n+    train_iter = QuestionIterator(all_guesses[c.BUZZER_TRAIN_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    dev_iter = QuestionIterator(all_guesses[c.BUZZER_DEV_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    expo_iter = QuestionIterator(all_guesses[\'expo\'], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+\n+    n_hidden = 200\n+    model_name = \'neo_0\'\n+    model_dir = \'output/buzzer/neo/{}.npz\'.format(model_name)\n+    model = RNN(train_iter.n_input, n_hidden, N_GUESSERS + 1)\n+\n+    chainer.cuda.get_device(0).use()\n+    model.to_gpu(0)\n+\n+    trainer = Trainer(model, model_dir)\n+    trainer.run(train_iter, dev_iter, 20)\n+\n+    dev_buzzes = trainer.test(dev_iter)\n+    dev_buzzes_dir = \'output/buzzer/neo/dev_buzzes.{}.pkl\'.format(model_name)\n+    with open(dev_buzzes_dir, \'wb\') as f:\n+        pickle.dump(dev_buzzes, f)\n+    print(\'Dev buzz {} saved to {}\'.format(len(dev_buzzes), dev_buzzes_dir))\n+\n+    expo_buzzes = trainer.test(expo_iter)\n+    expo_buzzes_dir = \'output/buzzer/neo/expo_buzzes.{}.pkl\'.format(model_name)\n+    with open(expo_buzzes_dir, \'wb\') as f:\n+        pickle.dump(expo_buzzes, f)\n+    print(\'Expo buzz {} saved to {}\'.format(len(expo_buzzes), expo_buzzes_dir))\n+\n+    report(dev_buzzes_dir)\n+\n+if __name__ == \'__main__\':\n+    main()\ndiff --git a/qanta/buzzer/rnn_1.py b/qanta/buzzer/rnn_1.py\nnew file mode 100644\nindex 0000000..e6485a4\n--- /dev/null\n+++ b/qanta/buzzer/rnn_1.py\n@@ -0,0 +1,107 @@\n+import os\n+import numpy as np\n+import argparse\n+import chainer\n+import pickle\n+from collections import defaultdict, namedtuple\n+from typing import List, Dict, Tuple, Optional\n+\n+from qanta import logging\n+from qanta.config import conf\n+from qanta.guesser.abstract import AbstractGuesser\n+\n+from qanta.buzzer import configs\n+from qanta.buzzer.progress import ProgressBar\n+from qanta.buzzer.trainer import Trainer\n+from qanta.buzzer.iterator import QuestionIterator\n+from qanta.buzzer import iterator \n+from qanta.buzzer.util import load_quizbowl, GUESSERS\n+from qanta.buzzer.models import MLP, RNN\n+from qanta.buzzer import constants as bc\n+from qanta.buzzer.report import report\n+from qanta.util import constants as c\n+\n+N_GUESSERS = len(GUESSERS)\n+N_GUESSES = 10\n+\n+log = logging.get(__name__)\n+\n+\n+def dense_vector(dicts: List[List[Dict[str, float]]],\n+        step_size=1) -> List[List[float]]:\n+    length = len(dicts)\n+    prev_vec = [0. for _ in range(N_GUESSERS * N_GUESSES)]\n+            \n+    vecs = []\n+    for i in range(length):\n+        if len(dicts[i]) != N_GUESSERS:\n+            raise ValueError("Inconsistent number of guessers ({0}, {1}).".format(\n+                N_GUESSERS, len(dicts)))\n+        vec = []\n+        diff_vec = []\n+        isnew_vec = []\n+        for j in range(N_GUESSERS):\n+            dic = sorted(dicts[i][j].items(), key=lambda x: x[1],\n+                    reverse=True)[:N_GUESSES]\n+            for guess, score in dic:\n+                vec.append(score)\n+                if i > 0 and guess in dicts[i-1][j]:\n+                    diff_vec.append(score - dicts[i-1][j][guess])\n+                    isnew_vec.append(0)\n+                else:\n+                    diff_vec.append(score) \n+                    isnew_vec.append(1)\n+            if len(dic) < N_GUESSES:\n+                for k in range(max(N_GUESSES - len(dic), 0)):\n+                    vec.append(0)\n+                    diff_vec.append(0)\n+                    isnew_vec.append(0)\n+        # guesser_acc = get_guesser_acc(i, length)\n+        features = [sum(isnew_vec), np.average(vec), vec[0], vec[1], vec[2],\n+                isnew_vec[0], isnew_vec[1], vec[0] - vec[1], vec[1] -\n+                vec[2], isnew_vec[2], diff_vec[0], \n+                vec[0] - prev_vec[0], np.var(vec),\n+                np.var(prev_vec)]\n+                # i, int(i < 10), int(i < 20), int(i > 30),\n+                # guesser_acc]\n+\n+        vecs.append(features)\n+        prev_vec = vec\n+    return vecs\n+\n+def main():\n+    option2id, all_guesses = load_quizbowl()\n+    train_iter = QuestionIterator(all_guesses[c.BUZZER_TRAIN_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    dev_iter = QuestionIterator(all_guesses[c.BUZZER_DEV_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    expo_iter = QuestionIterator(all_guesses[\'expo\'], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+\n+    n_hidden = 300\n+    model_name = \'neo_1\'\n+    model_dir = \'output/buzzer/neo/{}.npz\'.format(model_name)\n+    model = RNN(train_iter.n_input, n_hidden, N_GUESSERS + 1)\n+\n+    chainer.cuda.get_device(1).use()\n+    model.to_gpu(1)\n+\n+    trainer = Trainer(model, model_dir)\n+    trainer.run(train_iter, dev_iter, 20)\n+\n+    dev_buzzes = trainer.test(dev_iter)\n+    dev_buzzes_dir = \'output/buzzer/neo/dev_buzzes.{}.pkl\'.format(model_name)\n+    with open(dev_buzzes_dir, \'wb\') as f:\n+        pickle.dump(dev_buzzes, f)\n+    print(\'Dev buzz {} saved to {}\'.format(len(dev_buzzes), dev_buzzes_dir))\n+\n+    expo_buzzes = trainer.test(expo_iter)\n+    expo_buzzes_dir = \'output/buzzer/neo/expo_buzzes.{}.pkl\'.format(model_name)\n+    with open(expo_buzzes_dir, \'wb\') as f:\n+        pickle.dump(expo_buzzes, f)\n+    print(\'Expo buzz {} saved to {}\'.format(len(expo_buzzes), expo_buzzes_dir))\n+\n+    report(dev_buzzes_dir)\n+\n+if __name__ == \'__main__\':\n+    main()\ndiff --git a/qanta/buzzer/rnn_2.py b/qanta/buzzer/rnn_2.py\nnew file mode 100644\nindex 0000000..00dae47\n--- /dev/null\n+++ b/qanta/buzzer/rnn_2.py\n@@ -0,0 +1,109 @@\n+import os\n+import numpy as np\n+import argparse\n+import chainer\n+import pickle\n+from collections import defaultdict, namedtuple\n+from typing import List, Dict, Tuple, Optional\n+\n+from qanta import logging\n+from qanta.config import conf\n+from qanta.guesser.abstract import AbstractGuesser\n+\n+from qanta.buzzer import configs\n+from qanta.buzzer.progress import ProgressBar\n+from qanta.buzzer.trainer import Trainer\n+from qanta.buzzer.iterator import QuestionIterator\n+from qanta.buzzer import iterator \n+from qanta.buzzer.util import load_quizbowl, GUESSERS\n+from qanta.buzzer.models import MLP, RNN\n+from qanta.buzzer import constants as bc\n+from qanta.buzzer.report import report\n+from qanta.util import constants as c\n+\n+N_GUESSERS = len(GUESSERS)\n+N_GUESSES = 10\n+\n+log = logging.get(__name__)\n+\n+\n+def dense_vector(dicts: List[List[Dict[str, float]]],\n+        step_size=1) -> List[List[float]]:\n+    length = len(dicts)\n+    prev_vec = [0. for _ in range(N_GUESSERS * N_GUESSES)]\n+            \n+    vecs = []\n+    for i in range(length):\n+        if len(dicts[i]) != N_GUESSERS:\n+            raise ValueError("Inconsistent number of guessers ({0}, {1}).".format(\n+                N_GUESSERS, len(dicts)))\n+        vec = []\n+        diff_vec = []\n+        isnew_vec = []\n+        for j in range(N_GUESSERS):\n+            dic = sorted(dicts[i][j].items(), key=lambda x: x[1],\n+                    reverse=True)[:N_GUESSES]\n+            for guess, score in dic:\n+                vec.append(score)\n+                if i > 0 and guess in dicts[i-1][j]:\n+                    diff_vec.append(score - dicts[i-1][j][guess])\n+                    isnew_vec.append(0)\n+                else:\n+                    diff_vec.append(score) \n+                    isnew_vec.append(1)\n+            if len(dic) < N_GUESSES:\n+                for k in range(max(N_GUESSES - len(dic), 0)):\n+                    vec.append(0)\n+                    diff_vec.append(0)\n+                    isnew_vec.append(0)\n+        features = [vec[0], vec[1], vec[2],\n+                    isnew_vec[0], isnew_vec[1], isnew_vec[2], \n+                    prev_vec[0], prev_vec[1], prev_vec[2],\n+                    diff_vec[0], diff_vec[1], diff_vec[2],\n+                    sum(isnew_vec[:10]), sum(isnew_vec[:5]), \n+                    np.average(vec[:10]), np.average(prev_vec[:10]),\n+                    np.var(prev_vec[:10]), np.var(prev_vec[:10]),\n+                    vec[0] - vec[1], vec[1] - vec[2], \n+                    vec[0] - prev_vec[0], vec[1] - prev_vec[1]\n+                    ]\n+\n+        vecs.append(features)\n+        prev_vec = vec\n+    return vecs\n+\n+def main():\n+    option2id, all_guesses = load_quizbowl()\n+    train_iter = QuestionIterator(all_guesses[c.BUZZER_TRAIN_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    dev_iter = QuestionIterator(all_guesses[c.BUZZER_DEV_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    expo_iter = QuestionIterator(all_guesses[\'expo\'], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+\n+    n_hidden = 200\n+    model_name = \'neo_2\'\n+    model_dir = \'output/buzzer/neo/{}.npz\'.format(model_name)\n+    model = RNN(train_iter.n_input, n_hidden, N_GUESSERS + 1)\n+\n+    chainer.cuda.get_device(2).use()\n+    model.to_gpu(2)\n+\n+    trainer = Trainer(model, model_dir)\n+    trainer.run(train_iter, dev_iter, 20)\n+\n+    dev_buzzes = trainer.test(dev_iter)\n+    dev_buzzes_dir = \'output/buzzer/neo/dev_buzzes.{}.pkl\'.format(model_name)\n+    with open(dev_buzzes_dir, \'wb\') as f:\n+        pickle.dump(dev_buzzes, f)\n+    print(\'Dev buzz {} saved to {}\'.format(len(dev_buzzes), dev_buzzes_dir))\n+\n+    expo_buzzes = trainer.test(expo_iter)\n+    expo_buzzes_dir = \'output/buzzer/neo/expo_buzzes.{}.pkl\'.format(model_name)\n+    with open(expo_buzzes_dir, \'wb\') as f:\n+        pickle.dump(expo_buzzes, f)\n+    print(\'Expo buzz {} saved to {}\'.format(len(expo_buzzes), expo_buzzes_dir))\n+\n+    report(dev_buzzes_dir)\n+\n+if __name__ == \'__main__\':\n+    main()\ndiff --git a/qanta/buzzer/rnn_3.py b/qanta/buzzer/rnn_3.py\nnew file mode 100644\nindex 0000000..97264ce\n--- /dev/null\n+++ b/qanta/buzzer/rnn_3.py\n@@ -0,0 +1,109 @@\n+import os\n+import numpy as np\n+import argparse\n+import chainer\n+import pickle\n+from collections import defaultdict, namedtuple\n+from typing import List, Dict, Tuple, Optional\n+\n+from qanta import logging\n+from qanta.config import conf\n+from qanta.guesser.abstract import AbstractGuesser\n+\n+from qanta.buzzer import configs\n+from qanta.buzzer.progress import ProgressBar\n+from qanta.buzzer.trainer import Trainer\n+from qanta.buzzer.iterator import QuestionIterator\n+from qanta.buzzer import iterator \n+from qanta.buzzer.util import load_quizbowl, GUESSERS\n+from qanta.buzzer.models import MLP, RNN\n+from qanta.buzzer import constants as bc\n+from qanta.buzzer.report import report\n+from qanta.util import constants as c\n+\n+N_GUESSERS = len(GUESSERS)\n+N_GUESSES = 10\n+\n+log = logging.get(__name__)\n+\n+\n+def dense_vector(dicts: List[List[Dict[str, float]]],\n+        step_size=1) -> List[List[float]]:\n+    length = len(dicts)\n+    prev_vec = [0. for _ in range(N_GUESSERS * N_GUESSES)]\n+            \n+    vecs = []\n+    for i in range(length):\n+        if len(dicts[i]) != N_GUESSERS:\n+            raise ValueError("Inconsistent number of guessers ({0}, {1}).".format(\n+                N_GUESSERS, len(dicts)))\n+        vec = []\n+        diff_vec = []\n+        isnew_vec = []\n+        for j in range(N_GUESSERS):\n+            dic = sorted(dicts[i][j].items(), key=lambda x: x[1],\n+                    reverse=True)[:N_GUESSES]\n+            for guess, score in dic:\n+                vec.append(score)\n+                if i > 0 and guess in dicts[i-1][j]:\n+                    diff_vec.append(score - dicts[i-1][j][guess])\n+                    isnew_vec.append(0)\n+                else:\n+                    diff_vec.append(score) \n+                    isnew_vec.append(1)\n+            if len(dic) < N_GUESSES:\n+                for k in range(max(N_GUESSES - len(dic), 0)):\n+                    vec.append(0)\n+                    diff_vec.append(0)\n+                    isnew_vec.append(0)\n+        features = [vec[0], vec[1], vec[2],\n+                    isnew_vec[0], isnew_vec[1],\n+                    prev_vec[0], prev_vec[1],\n+                    diff_vec[0], diff_vec[1],\n+                    sum(isnew_vec[:10]),\n+                    np.average(vec[:10]), np.average(prev_vec[:10]),\n+                    np.var(prev_vec[:10]), np.var(prev_vec[:10]),\n+                    vec[0] - vec[1], vec[1] - vec[2], \n+                    vec[0] - prev_vec[0], vec[1] - prev_vec[1]\n+                    ]\n+\n+        vecs.append(features)\n+        prev_vec = vec\n+    return vecs\n+\n+def main():\n+    option2id, all_guesses = load_quizbowl()\n+    train_iter = QuestionIterator(all_guesses[c.BUZZER_TRAIN_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    dev_iter = QuestionIterator(all_guesses[c.BUZZER_DEV_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    expo_iter = QuestionIterator(all_guesses[\'expo\'], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+\n+    n_hidden = 300\n+    model_name = \'neo_3\'\n+    model_dir = \'output/buzzer/neo/{}.npz\'.format(model_name)\n+    model = RNN(train_iter.n_input, n_hidden, N_GUESSERS + 1)\n+\n+    chainer.cuda.get_device(3).use()\n+    model.to_gpu(3)\n+\n+    trainer = Trainer(model, model_dir)\n+    trainer.run(train_iter, dev_iter, 25)\n+\n+    dev_buzzes = trainer.test(dev_iter)\n+    dev_buzzes_dir = \'output/buzzer/neo/dev_buzzes.{}.pkl\'.format(model_name)\n+    with open(dev_buzzes_dir, \'wb\') as f:\n+        pickle.dump(dev_buzzes, f)\n+    print(\'Dev buzz {} saved to {}\'.format(len(dev_buzzes), dev_buzzes_dir))\n+\n+    expo_buzzes = trainer.test(expo_iter)\n+    expo_buzzes_dir = \'output/buzzer/neo/expo_buzzes.{}.pkl\'.format(model_name)\n+    with open(expo_buzzes_dir, \'wb\') as f:\n+        pickle.dump(expo_buzzes, f)\n+    print(\'Expo buzz {} saved to {}\'.format(len(expo_buzzes), expo_buzzes_dir))\n+\n+    report(dev_buzzes_dir)\n+\n+if __name__ == \'__main__\':\n+    main()\ndiff --git a/qanta/buzzer/rnn_4.py b/qanta/buzzer/rnn_4.py\nnew file mode 100644\nindex 0000000..289c416\n--- /dev/null\n+++ b/qanta/buzzer/rnn_4.py\n@@ -0,0 +1,109 @@\n+import os\n+import numpy as np\n+import argparse\n+import chainer\n+import pickle\n+from collections import defaultdict, namedtuple\n+from typing import List, Dict, Tuple, Optional\n+\n+from qanta import logging\n+from qanta.config import conf\n+from qanta.guesser.abstract import AbstractGuesser\n+\n+from qanta.buzzer import configs\n+from qanta.buzzer.progress import ProgressBar\n+from qanta.buzzer.trainer import Trainer\n+from qanta.buzzer.iterator import QuestionIterator\n+from qanta.buzzer import iterator \n+from qanta.buzzer.util import load_quizbowl, GUESSERS\n+from qanta.buzzer.models import MLP, RNN\n+from qanta.buzzer import constants as bc\n+from qanta.buzzer.report import report\n+from qanta.util import constants as c\n+\n+N_GUESSERS = len(GUESSERS)\n+N_GUESSES = 10\n+\n+log = logging.get(__name__)\n+\n+\n+def dense_vector(dicts: List[List[Dict[str, float]]],\n+        step_size=1) -> List[List[float]]:\n+    length = len(dicts)\n+    prev_vec = [0. for _ in range(N_GUESSERS * N_GUESSES)]\n+            \n+    vecs = []\n+    for i in range(length):\n+        if len(dicts[i]) != N_GUESSERS:\n+            raise ValueError("Inconsistent number of guessers ({0}, {1}).".format(\n+                N_GUESSERS, len(dicts)))\n+        vec = []\n+        diff_vec = []\n+        isnew_vec = []\n+        for j in range(N_GUESSERS):\n+            dic = sorted(dicts[i][j].items(), key=lambda x: x[1],\n+                    reverse=True)[:N_GUESSES]\n+            for guess, score in dic:\n+                vec.append(score)\n+                if i > 0 and guess in dicts[i-1][j]:\n+                    diff_vec.append(score - dicts[i-1][j][guess])\n+                    isnew_vec.append(0)\n+                else:\n+                    diff_vec.append(score) \n+                    isnew_vec.append(1)\n+            if len(dic) < N_GUESSES:\n+                for k in range(max(N_GUESSES - len(dic), 0)):\n+                    vec.append(0)\n+                    diff_vec.append(0)\n+                    isnew_vec.append(0)\n+        features = [vec[0], vec[1], vec[2],\n+                    isnew_vec[0], isnew_vec[1], isnew_vec[2], \n+                    prev_vec[0], prev_vec[1], prev_vec[2],\n+                    diff_vec[0], diff_vec[1], diff_vec[2],\n+                    sum(isnew_vec[:10]), sum(isnew_vec[:5]), \n+                    np.average(vec[:10]), np.average(prev_vec[:10]),\n+                    np.var(prev_vec[:10]), np.var(prev_vec[:10]),\n+                    vec[0] - vec[1], vec[1] - vec[2], \n+                    vec[0] - prev_vec[0], vec[1] - prev_vec[1]\n+                    ]\n+\n+        vecs.append(features)\n+        prev_vec = vec\n+    return vecs\n+\n+def main():\n+    option2id, all_guesses = load_quizbowl()\n+    train_iter = QuestionIterator(all_guesses[c.BUZZER_TRAIN_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    dev_iter = QuestionIterator(all_guesses[c.BUZZER_DEV_FOLD], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+    expo_iter = QuestionIterator(all_guesses[\'expo\'], option2id,\n+            batch_size=128, make_vector=dense_vector)\n+\n+    n_hidden = 200\n+    model_name = \'neo_4\'\n+    model_dir = \'output/buzzer/neo/{}.npz\'.format(model_name)\n+    model = RNN(train_iter.n_input, n_hidden, N_GUESSERS + 1)\n+\n+    chainer.cuda.get_device(1).use()\n+    model.to_gpu(1)\n+\n+    trainer = Trainer(model, model_dir)\n+    trainer.run(train_iter, dev_iter, 20)\n+\n+    dev_buzzes = trainer.test(dev_iter)\n+    dev_buzzes_dir = \'output/buzzer/neo/dev_buzzes.{}.pkl\'.format(model_name)\n+    with open(dev_buzzes_dir, \'wb\') as f:\n+        pickle.dump(dev_buzzes, f)\n+    print(\'Dev buzz {} saved to {}\'.format(len(dev_buzzes), dev_buzzes_dir))\n+\n+    expo_buzzes = trainer.test(expo_iter)\n+    expo_buzzes_dir = \'output/buzzer/neo/expo_buzzes.{}.pkl\'.format(model_name)\n+    with open(expo_buzzes_dir, \'wb\') as f:\n+        pickle.dump(expo_buzzes, f)\n+    print(\'Expo buzz {} saved to {}\'.format(len(expo_buzzes), expo_buzzes_dir))\n+\n+    report(dev_buzzes_dir)\n+\n+if __name__ == \'__main__\':\n+    main()\ndiff --git a/qanta/buzzer/test.py b/qanta/buzzer/test.py\nindex 8c8011f..65517e2 100644\n--- a/qanta/buzzer/test.py\n+++ b/qanta/buzzer/test.py\n@@ -9,6 +9,7 @@ from qanta.config import conf\n \n from qanta.buzzer import configs\n from qanta.buzzer.interface import buzzer2vwexpo\n+from qanta.buzzer import iterator\n from qanta.buzzer.iterator import QuestionIterator\n from qanta.buzzer.util import load_quizbowl, GUESSERS, stupid_buzzer\n from qanta.buzzer.trainer import Trainer\n@@ -24,24 +25,19 @@ def generate(config, folds):\n     option2id, all_guesses = load_quizbowl(folds)\n \n     cfg = getattr(configs, config)()\n-    cfg = pickle.load(open(cfg.ckp_dir, \'rb\'))\n+    # cfg = pickle.load(open(cfg.ckp_dir, \'rb\'))\n+    make_vector = getattr(iterator, cfg.make_vector)\n \n     iterators = dict()\n     for fold in folds:\n         iterators[fold] = QuestionIterator(all_guesses[fold], option2id,\n-            batch_size=cfg.batch_size)\n+            batch_size=cfg.batch_size, make_vector=make_vector)\n     \n     if not os.path.exists(cfg.model_dir):\n         log.info(\'Model {0} not available\'.format(cfg.model_dir))\n         exit(0)\n \n-    if isinstance(cfg, configs.mlp):\n-        model = MLP(n_input=iterators[folds[0]].n_input, n_hidden=cfg.n_hidden,\n-                n_output=N_GUESSERS+1, n_layers=cfg.n_layers,\n-                dropout=cfg.dropout, batch_norm=cfg.batch_norm)\n-\n-    if isinstance(cfg, configs.rnn):\n-        model = RNN(iterators[folds[0]].n_input, cfg.n_hidden, N_GUESSERS + 1)\n+    model = RNN(iterators[folds[0]].n_input, cfg.n_hidden, N_GUESSERS + 1)\n \n     log.info(\'Loading model {0}\'.format(cfg.model_dir))\n     chainer.serializers.load_npz(cfg.model_dir, model)\ndiff --git a/qanta/buzzer/util.py b/qanta/buzzer/util.py\nindex c885ce7..cc1b341 100644\n--- a/qanta/buzzer/util.py\n+++ b/qanta/buzzer/util.py\n@@ -73,8 +73,7 @@ def stupid_buzzer(iterator) -> Dict[int, int]:\n     return buzz_dict\n \n def _process_question(option2id: Dict[str, int], \n-        all_questions: Dict[int, Question], \n-        word2vec, normalize, qnum, question) -> \\\n+        all_questions: Dict[int, Question], qnum, question) -> \\\n             Tuple[int, int, List[List[Dict[str, int]]], List[List[int]]]:\n     \'\'\'Process one question.\n     return:\n@@ -95,61 +94,32 @@ def _process_question(option2id: Dict[str, int],\n \n     guess_dicts = []\n     results = []\n-    wordvecs = None\n-    if word2vec is not None:\n-        wordvecs = []\n     for pos, pos_group in question.groupby([\'sentence\', \'token\']):\n         pos_group = pos_group.groupby(\'guesser\')\n         guess_dicts.append([])\n         results.append([])\n-        if word2vec is not None:\n-            wordvecs.append([])\n         for guesser in GUESSERS:\n             if guesser not in pos_group.groups:\n                 log.info("{0} missing guesser {1}.".format(qnum, guesser))\n                 guess_dicts[-1].append({})\n                 results[-1].append(0)\n-                if word2vec is not None:\n-                    wordvecs[-1].append(word2vec.get_zero_vec())\n             else:\n                 guesses = pos_group.get_group(guesser)\n                 guesses = guesses.sort_values(\'score\', ascending=False)\n                 top_guess = guesses.iloc[0].guess\n                 results[-1].append(int(top_guess == answer))\n                 \n-                s = sum(guesses.score) if normalize else 1\n+                # s = sum(guesses.score)\n+                s = 1\n                 dic = {x.guess: x.score / s for x in guesses.itertuples()}\n                 \n                 guess_dicts[-1].append(dic)\n-                if word2vec is not None:\n-                    wordvecs[-1].append(word2vec.get_avg_vec(top_guess))\n \n-    return qnum, answer_id, guess_dicts, results, wordvecs\n+    return qnum, answer_id, guess_dicts, results\n \n-\n-class Word2Vec:\n-\n-    def __init__(self, wordvec_dir, wordvec_dim):\n-        self.word2vec = gensim.models.KeyedVectors.load_word2vec_format(\n-                wordvec_dir, binary=True)\n-        self.wordvec_dim = wordvec_dim\n-\n-    def get_avg_vec(self, guess):\n-        vecs = []\n-        for word in guess.split(\'_\'):\n-            if word in self.word2vec:\n-                vecs.append(self.word2vec[word])\n-        if len(vecs) > 0:\n-            return sum(vecs) / len(vecs)\n-        return self.get_zero_vec()\n-\n-    def get_zero_vec(self):\n-        return np.zeros(self.wordvec_dim, dtype=np.float32)\n-\n-\n-def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \\\n+def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, normalize=True) \\\n                     -> Tuple[Dict[str, int], Dict[str, list]]:\n-    merge_dfs()\n+    # merge_dfs()\n     log.info(\'Loading data\')\n     question_db = QuestionDatabase()\n     quizbowl_db = QuizBowlDataset(bc.MIN_ANSWERS, guesser_train=True, buzzer_train=True)\n@@ -180,7 +150,7 @@ def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \\\n         log.info(\'Processing {0} guesses\'.format(fold))\n         guesses = AbstractGuesser.load_guesses(bc.GUESSES_DIR, folds=[fold])\n \n-        worker = partial(_process_question, option2id, all_questions, word2vec, normalize)\n+        worker = partial(_process_question, option2id, all_questions)\n         inputs = guesses.groupby(\'qnum\')\n         guesses_by_fold[fold] = _multiprocess(worker, inputs, info=\'df data\',\n                 multi=True)\n@@ -285,13 +255,6 @@ def load_protobowl():\n if __name__ == "__main__":\n     merge_dfs()\n \n-    processed_dirs = [\'%s_processed.pickle\' % (os.path.join(\n-        bc.GUESSES_DIR, fold)) for fold in c.BUZZER_INPUT_FOLDS]\n-    # if not all(os.path.isfile(d) for d in processed_dirs):\n-    #     log.info(\'Loading {0}\'.format(bc.WORDVEC_DIR))\n-    #     word2vec = Word2Vec(bc.WORDVEC_DIR, bc.WORDVEC_DIM)\n-    # else:\n-    word2vec = None\n-\n-    option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS, word2vec)\n-    load_protobowl()\n+    # option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS)\n+    # load_protobowl()\n+    option2id, guesses_by_fold = load_quizbowl()\ndiff --git a/rnn.ipynb b/rnn.ipynb\nindex 5ab714a..80f8549 100644\n--- a/rnn.ipynb\n+++ b/rnn.ipynb\n@@ -57,7 +57,7 @@\n    },\n    "outputs": [],\n    "source": [\n-    "def dense_vector(dicts: List[List[Dict[str, float]]],\\n",\n+    "def new_dense_vector(dicts: List[List[Dict[str, float]]],\\n",\n     "        wordvecs: List[List[np.ndarray]], step_size=1) -> List[List[float]]:\\n",\n     "\\n",\n     "    length = len(dicts)\\n",\n@@ -103,31 +103,185 @@\n   },\n   {\n    "cell_type": "code",\n-   "execution_count": null,\n+   "execution_count": 3,\n    "metadata": {\n     "collapsed": false\n    },\n-   "outputs": [],\n+   "outputs": [\n+    {\n+     "name": "stderr",\n+     "output_type": "stream",\n+     "text": [\n+      "2017-05-26 13:18:00,145 - qanta.buzzer.util - INFO - Merging guesser DataFrames.\\n",\n+      "2017-05-26 13:18:00,146 - qanta.buzzer.util - INFO - Merged buzzertrain exists, skipping.\\n",\n+      "2017-05-26 13:18:00,147 - qanta.buzzer.util - INFO - Merged buzzerdev exists, skipping.\\n",\n+      "2017-05-26 13:18:00,148 - qanta.buzzer.util - INFO - Merged dev exists, skipping.\\n",\n+      "2017-05-26 13:18:00,150 - qanta.buzzer.util - INFO - Merged test exists, skipping.\\n",\n+      "2017-05-26 13:18:00,151 - qanta.buzzer.util - INFO - Merged expo exists, skipping.\\n",\n+      "2017-05-26 13:18:00,152 - qanta.buzzer.util - INFO - Loading data\\n",\n+      "2017-05-26 13:18:00,154 - qanta.datasets.quiz_bowl - WARNING - Using QuizBowlDataset with guesser and buzzer training data, make sure you know what you are doing!\\n",\n+      "2017-05-26 13:18:06,976 - qanta.buzzer.util - INFO - Number of options 8247\\n",\n+      "2017-05-26 13:18:27,531 - qanta.buzzer.util - INFO - Loading buzzertrain guesses\\n",\n+      "2017-05-26 13:18:32,471 - qanta.buzzer.util - INFO - Loading buzzerdev guesses\\n",\n+      "2017-05-26 13:18:38,994 - qanta.buzzer.util - INFO - Loading dev guesses\\n",\n+      "2017-05-26 13:18:39,762 - qanta.buzzer.util - INFO - Loading test guesses\\n",\n+      "2017-05-26 13:18:39,778 - qanta.buzzer.util - INFO - Loading expo guesses\\n",\n+      "2017-05-26 13:18:40,023 - qanta.buzzer.iterator - INFO - Creating batches\\n",\n+      "[creat batches] (1) done: 23210/23211\\n",\n+      "2017-05-26 13:26:35,211 - qanta.buzzer.iterator - INFO - Creating batches\\n",\n+      "[creat batches] (1) done: 7586/7587\\n",\n+      "2017-05-26 13:29:14,547 - qanta.buzzer.iterator - INFO - Creating batches\\n",\n+      "[creat batches] (1) done: 2088/2089\\n",\n+      "2017-05-26 13:30:00,000 - qanta.buzzer.iterator - INFO - Creating batches\\n",\n+      "[creat batches] (1) done: 1411/1412\\n",\n+      "2017-05-26 13:30:30,878 - qanta.buzzer.iterator - INFO - Creating batches\\n",\n+      "[creat batches] (1) done: 42/43\\n"\n+     ]\n+    }\n+   ],\n    "source": [\n-    "option2id, all_guesses = load_quizbowl()\\n",\n+    "option2id, all_guesses = load_quizbowl(normalize=True)\\n",\n     "iterators = dict()\\n",\n     "for fold in c.BUZZER_INPUT_FOLDS:\\n",\n     "    iterators[fold] = QuestionIterator(all_guesses[fold], option2id,\\n",\n-    "        batch_size=128, make_vector=dense_vector)"\n+    "        batch_size=128, make_vector=new_dense_vector)"\n    ]\n   },\n   {\n    "cell_type": "code",\n-   "execution_count": null,\n+   "execution_count": 4,\n    "metadata": {\n     "collapsed": false\n    },\n-   "outputs": [],\n+   "outputs": [\n+    {\n+     "data": {\n+      "text/plain": [\n+       "19"\n+      ]\n+     },\n+     "execution_count": 4,\n+     "metadata": {},\n+     "output_type": "execute_result"\n+    }\n+   ],\n+   "source": [\n+    "iterators[\'expo\'].n_input"\n+   ]\n+  },\n+  {\n+   "cell_type": "code",\n+   "execution_count": 5,\n+   "metadata": {\n+    "collapsed": false\n+   },\n+   "outputs": [\n+    {\n+     "name": "stderr",\n+     "output_type": "stream",\n+     "text": [\n+      "2017-05-26 13:30:31,787 - qanta.buzzer.trainer - INFO - epoch 0\\n",\n+      "2017-05-26 13:31:18,785 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.56  \\n",\n+      "2017-05-26 13:31:32,194 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:31:32,211 - qanta.buzzer.trainer - INFO - epoch 1\\n",\n+      "2017-05-26 13:32:01,036 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:32:10,760 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:32:10,771 - qanta.buzzer.trainer - INFO - epoch 2\\n",\n+      "2017-05-26 13:32:53,690 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:33:01,315 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:33:01,326 - qanta.buzzer.trainer - INFO - epoch 3\\n",\n+      "2017-05-26 13:33:33,181 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:33:43,310 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:33:43,324 - qanta.buzzer.trainer - INFO - epoch 4\\n",\n+      "2017-05-26 13:34:33,864 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:34:41,516 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:34:41,527 - qanta.buzzer.trainer - INFO - epoch 5\\n",\n+      "2017-05-26 13:35:20,247 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:35:27,034 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:35:27,048 - qanta.buzzer.trainer - INFO - epoch 6\\n",\n+      "2017-05-26 13:36:08,099 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:36:30,843 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:36:30,857 - qanta.buzzer.trainer - INFO - epoch 7\\n",\n+      "2017-05-26 13:37:15,452 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:37:20,652 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:37:20,664 - qanta.buzzer.trainer - INFO - epoch 8\\n",\n+      "2017-05-26 13:38:04,129 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:38:09,308 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:38:09,318 - qanta.buzzer.trainer - INFO - epoch 9\\n",\n+      "2017-05-26 13:38:44,532 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:39:00,090 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:39:00,101 - qanta.buzzer.trainer - INFO - epoch 10\\n",\n+      "2017-05-26 13:39:47,432 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:39:58,447 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:39:58,461 - qanta.buzzer.trainer - INFO - epoch 11\\n",\n+      "2017-05-26 13:40:45,519 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:40:53,162 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:40:53,175 - qanta.buzzer.trainer - INFO - epoch 12\\n",\n+      "2017-05-26 13:41:49,485 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:42:04,996 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:42:05,010 - qanta.buzzer.trainer - INFO - epoch 13\\n",\n+      "2017-05-26 13:42:38,190 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:42:49,217 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:42:49,227 - qanta.buzzer.trainer - INFO - epoch 14\\n",\n+      "2017-05-26 13:43:20,695 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:43:38,453 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:43:38,466 - qanta.buzzer.trainer - INFO - epoch 15\\n",\n+      "2017-05-26 13:44:12,905 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:44:18,066 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:44:18,076 - qanta.buzzer.trainer - INFO - epoch 16\\n",\n+      "2017-05-26 13:44:42,034 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:44:50,615 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:44:50,626 - qanta.buzzer.trainer - INFO - epoch 17\\n",\n+      "2017-05-26 13:45:33,054 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:45:46,215 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:45:46,227 - qanta.buzzer.trainer - INFO - epoch 18\\n",\n+      "2017-05-26 13:46:36,038 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:46:46,441 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:46:46,453 - qanta.buzzer.trainer - INFO - epoch 19\\n",\n+      "2017-05-26 13:47:21,608 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:47:26,764 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:47:26,774 - qanta.buzzer.trainer - INFO - epoch 20\\n",\n+      "2017-05-26 13:48:04,414 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:48:09,575 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:48:09,588 - qanta.buzzer.trainer - INFO - epoch 21\\n",\n+      "2017-05-26 13:48:51,297 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:48:58,286 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:48:58,298 - qanta.buzzer.trainer - INFO - epoch 22\\n",\n+      "2017-05-26 13:49:38,014 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:49:45,903 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:49:45,914 - qanta.buzzer.trainer - INFO - epoch 23\\n",\n+      "2017-05-26 13:50:31,968 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:50:37,120 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:50:37,129 - qanta.buzzer.trainer - INFO - epoch 24\\n",\n+      "2017-05-26 13:51:10,158 - qanta.buzzer.trainer - INFO - train loss: nan  acc: 0.55  \\n",\n+      "2017-05-26 13:51:15,311 - qanta.buzzer.trainer - INFO - eval loss: nan  acc: 0.61  \\n",\n+      "2017-05-26 13:51:30,421 - __main__ - INFO - buzzerdev buzzes generated. Size 7587.\\n",\n+      "2017-05-26 13:51:30,570 - __main__ - INFO - Buzzes saved to output/buzzer/buzzerdev_buzzes_rnn.200.pkl.\\n",\n+      "2017-05-26 13:51:36,797 - __main__ - INFO - dev buzzes generated. Size 2089.\\n",\n+      "2017-05-26 13:51:36,853 - __main__ - INFO - Buzzes saved to output/buzzer/dev_buzzes_rnn.200.pkl.\\n",\n+      "2017-05-26 13:51:38,821 - __main__ - INFO - test buzzes generated. Size 1412.\\n",\n+      "2017-05-26 13:51:38,855 - __main__ - INFO - Buzzes saved to output/buzzer/test_buzzes_rnn.200.pkl.\\n",\n+      "2017-05-26 13:51:39,079 - __main__ - INFO - expo buzzes generated. Size 43.\\n",\n+      "2017-05-26 13:51:39,081 - __main__ - INFO - Buzzes saved to output/buzzer/expo_buzzes_rnn.200.pkl.\\n"\n+     ]\n+    },\n+    {\n+     "ename": "NameError",\n+     "evalue": "name \'buzzer2vwexpo\' is not defined",\n+     "output_type": "error",\n+     "traceback": [\n+      "\\u001b[0;31m---------------------------------------------------------------------------\\u001b[0m",\n+      "\\u001b[0;31mNameError\\u001b[0m                                 Traceback (most recent call last)",\n+      "\\u001b[0;32m<ipython-input-5-b9748c5c04e9>\\u001b[0m in \\u001b[0;36m<module>\\u001b[0;34m()\\u001b[0m\\n\\u001b[1;32m     22\\u001b[0m         \\u001b[0;32mif\\u001b[0m \\u001b[0mfold\\u001b[0m \\u001b[0;34m==\\u001b[0m \\u001b[0;34m\'expo\'\\u001b[0m\\u001b[0;34m:\\u001b[0m\\u001b[0;34m\\u001b[0m\\u001b[0m\\n\\u001b[1;32m     23\\u001b[0m             \\u001b[0mguesses_df\\u001b[0m \\u001b[0;34m=\\u001b[0m \\u001b[0mAbstractGuesser\\u001b[0m\\u001b[0;34m.\\u001b[0m\\u001b[0mload_guesses\\u001b[0m\\u001b[0;34m(\\u001b[0m\\u001b[0mbc\\u001b[0m\\u001b[0;34m.\\u001b[0m\\u001b[0mGUESSES_DIR\\u001b[0m\\u001b[0;34m,\\u001b[0m \\u001b[0mfolds\\u001b[0m\\u001b[0;34m=\\u001b[0m\\u001b[0;34m[\\u001b[0m\\u001b[0mfold\\u001b[0m\\u001b[0;34m]\\u001b[0m\\u001b[0;34m)\\u001b[0m\\u001b[0;34m\\u001b[0m\\u001b[0m\\n\\u001b[0;32m---> 24\\u001b[0;31m             \\u001b[0mbuzzer2vwexpo\\u001b[0m\\u001b[0;34m(\\u001b[0m\\u001b[0mguesses_df\\u001b[0m\\u001b[0;34m,\\u001b[0m \\u001b[0mbuzzes\\u001b[0m\\u001b[0;34m,\\u001b[0m \\u001b[0mfold\\u001b[0m\\u001b[0;34m)\\u001b[0m\\u001b[0;34m\\u001b[0m\\u001b[0m\\n\\u001b[0m",\n+      "\\u001b[0;31mNameError\\u001b[0m: name \'buzzer2vwexpo\' is not defined"\n+     ]\n+    }\n+   ],\n    "source": [\n-    "for n_hidden in [300]:\\n",\n+    "for n_hidden in [200, 400]:\\n",\n     "    cfg = configs.rnn()\\n",\n     "    cfg.n_hidden = n_hidden\\n",\n-    "    cfg.model_name = \'rnn_{}\'.format(n_hidden)\\n",\n+    "    cfg.model_name = \'rnn.{}\'.format(n_hidden, \'dense_vector_1\')\\n",\n     "    cfg.model_dir = \'output/buzzer/{}.npz\'.format(cfg.model_name)\\n",\n     "\\n",\n     "    model = RNN(iterators[c.BUZZER_TRAIN_FOLD].n_input, cfg.n_hidden, N_GUESSERS + 1)\\n",\n@@ -144,94 +298,57 @@\n     "        buzzes_dir = bc.BUZZES_DIR.format(fold, cfg.model_name)\\n",\n     "        with open(buzzes_dir, \'wb\') as outfile:\\n",\n     "            pickle.dump(buzzes, outfile)\\n",\n-    "        log.info(\'Buzzes saved to {0}.\'.format(buzzes_dir))"\n+    "        log.info(\'Buzzes saved to {0}.\'.format(buzzes_dir))\\n",\n+    "        if fold == \'expo\':\\n",\n+    "            guesses_df = AbstractGuesser.load_guesses(bc.GUESSES_DIR, folds=[fold])\\n",\n+    "            buzzer2vwexpo(guesses_df, buzzes, fold)"\n    ]\n   },\n   {\n    "cell_type": "code",\n-   "execution_count": 3,\n+   "execution_count": null,\n+   "metadata": {\n+    "collapsed": true\n+   },\n+   "outputs": [],\n+   "source": []\n+  },\n+  {\n+   "cell_type": "code",\n+   "execution_count": null,\n    "metadata": {\n     "collapsed": false\n    },\n-   "outputs": [\n-    {\n-     "name": "stderr",\n-     "output_type": "stream",\n-     "text": [\n-      "2017-05-26 10:32:26,233 - qanta.buzzer.util - INFO - Merging guesser DataFrames.\\n",\n-      "2017-05-26 10:32:26,238 - qanta.buzzer.util - INFO - Merged buzzertrain exists, skipping.\\n",\n-      "2017-05-26 10:32:26,240 - qanta.buzzer.util - INFO - Merged buzzerdev exists, skipping.\\n",\n-      "2017-05-26 10:32:26,242 - qanta.buzzer.util - INFO - Merged dev exists, skipping.\\n",\n-      "2017-05-26 10:32:26,244 - qanta.buzzer.util - INFO - Merged test exists, skipping.\\n",\n-      "2017-05-26 10:32:26,247 - qanta.buzzer.util - INFO - Merged expo exists, skipping.\\n",\n-      "2017-05-26 10:32:26,248 - qanta.buzzer.util - INFO - Loading data\\n",\n-      "2017-05-26 10:32:26,250 - qanta.datasets.quiz_bowl - WARNING - Using QuizBowlDataset with guesser and buzzer training data, make sure you know what you are doing!\\n",\n-      "2017-05-26 10:32:48,157 - qanta.buzzer.util - INFO - Number of options 8247\\n",\n-      "2017-05-26 10:32:48,160 - qanta.buzzer.util - INFO - Processing expo guesses\\n",\n-      "[df data] (16) done: 43/43\\n",\n-      "2017-05-26 10:34:26,778 - qanta.buzzer.util - INFO - Processed expo guesses saved to output/guesser/merged/expo_processed.pickle\\n",\n-      "2017-05-26 10:34:26,787 - qanta.buzzer.iterator - INFO - Creating batches\\n",\n-      "[creat batches] (1) done: 42/43\\n"\n-     ]\n-    }\n-   ],\n+   "outputs": [],\n    "source": [\n-    "option2id, all_guesses = load_quizbowl([\'expo\'])\\n",\n+    "option2id, all_guesses = load_quizbowl([\'expo\'], normalize=True)\\n",\n     "iterators = dict()\\n",\n     "iterators[\'expo\'] = QuestionIterator(all_guesses[\'expo\'], option2id,\\n",\n-    "    batch_size=128, make_vector=dense_vector)"\n+    "    batch_size=128, make_vector=old_dense_vector )"\n    ]\n   },\n   {\n    "cell_type": "code",\n-   "execution_count": 5,\n+   "execution_count": null,\n    "metadata": {\n     "collapsed": false\n    },\n-   "outputs": [\n-    {\n-     "data": {\n-      "text/plain": [\n-       "43"\n-      ]\n-     },\n-     "execution_count": 5,\n-     "metadata": {},\n-     "output_type": "execute_result"\n-    }\n-   ],\n+   "outputs": [],\n    "source": [\n-    "len(all_guesses[\'expo\'])"\n+    "iterators[\'expo\'].n_input"\n    ]\n   },\n   {\n    "cell_type": "code",\n-   "execution_count": 4,\n+   "execution_count": null,\n    "metadata": {\n     "collapsed": false\n    },\n-   "outputs": [\n-    {\n-     "name": "stderr",\n-     "output_type": "stream",\n-     "text": [\n-      "2017-05-26 10:34:29,099 - __main__ - INFO - expo buzzes generated. Size 43.\\n",\n-      "2017-05-26 10:34:29,104 - __main__ - INFO - Buzzes saved to output/buzzer/expo_buzzes_rnn_200.pkl.\\n",\n-      "[buzzer2vwexpo] (16) done: 43/43\\n",\n-      "2017-05-26 10:34:32,899 - qanta.buzzer.interface - INFO - \\n",\n-      "\\n",\n-      "[buzzer2vwexpo] writing to files\\n",\n-      "2017-05-26 10:34:32,972 - qanta.buzzer.interface - INFO - buzz file written\\n",\n-      "2017-05-26 10:34:32,973 - qanta.buzzer.interface - INFO - final file written\\n",\n-      "2017-05-26 10:34:33,032 - qanta.buzzer.interface - INFO - vw_pred file written\\n",\n-      "2017-05-26 10:34:33,087 - qanta.buzzer.interface - INFO - vw_meta file written\\n"\n-     ]\n-    }\n-   ],\n+   "outputs": [],\n    "source": [\n     "from qanta.buzzer.interface import buzzer2vwexpo\\n",\n     "\\n",\n-    "n_hidden = 200\\n",\n+    "n_hidden = 100\\n",\n     "cfg = configs.rnn()\\n",\n     "cfg.n_hidden = n_hidden\\n",\n     "cfg.model_name = \'rnn_{}\'.format(n_hidden)\\n",\n@@ -241,6 +358,9 @@\n     "chainer.cuda.get_device(0).use()\\n",\n     "model.to_gpu(0)\\n",\n     "\\n",\n+    "log.info(\'Loading model {0}\'.format(cfg.model_dir))\\n",\n+    "chainer.serializers.load_npz(cfg.model_dir, model)\\n",\n+    "\\n",\n     "trainer = Trainer(model, cfg.model_dir)\\n",\n     "\\n",\n     "fold = \'expo\'\\n",\n', 'commit_subject': "don't normalize!", 'repo_name': 'Pinafore/qb', 'committer': {'name': 'ihsgnef', 'email': '6f5ec810ecf149e7b4bf6c8b19528a1d2704904a@gmail.com', 'time_sec': '1495861190', 'tz_offset': '-240', 'date': {'seconds': '1495861190'}}, 'commit_tree': '45adcc9bd4041eda865aa3593f73c7b602187f04', 'commit_message': "don't normalize!\n", 'language_name': 'Python', 'license': 'mit', 'hunks': '@@ -73,8 +73,7 @@ def stupid_buzzer(iterator) -> Dict[int, int]:\n     return buzz_dict\n \n def _process_question(option2id: Dict[str, int], \n-        all_questions: Dict[int, Question], \n-        word2vec, normalize, qnum, question) -> \\\n+        all_questions: Dict[int, Question], qnum, question) -> \\\n             Tuple[int, int, List[List[Dict[str, int]]], List[List[int]]]:\n     \'\'\'Process one question.\n     return:\n@@ -95,61 +94,32 @@ def _process_question(option2id: Dict[str, int],\n \n     guess_dicts = []\n     results = []\n-    wordvecs = None\n-    if word2vec is not None:\n-        wordvecs = []\n     for pos, pos_group in question.groupby([\'sentence\', \'token\']):\n         pos_group = pos_group.groupby(\'guesser\')\n         guess_dicts.append([])\n         results.append([])\n-        if word2vec is not None:\n-            wordvecs.append([])\n         for guesser in GUESSERS:\n             if guesser not in pos_group.groups:\n                 log.info("{0} missing guesser {1}.".format(qnum, guesser))\n                 guess_dicts[-1].append({})\n                 results[-1].append(0)\n-                if word2vec is not None:\n-                    wordvecs[-1].append(word2vec.get_zero_vec())\n             else:\n                 guesses = pos_group.get_group(guesser)\n                 guesses = guesses.sort_values(\'score\', ascending=False)\n                 top_guess = guesses.iloc[0].guess\n                 results[-1].append(int(top_guess == answer))\n                 \n-                s = sum(guesses.score) if normalize else 1\n+                # s = sum(guesses.score)\n+                s = 1\n                 dic = {x.guess: x.score / s for x in guesses.itertuples()}\n                 \n                 guess_dicts[-1].append(dic)\n-                if word2vec is not None:\n-                    wordvecs[-1].append(word2vec.get_avg_vec(top_guess))\n \n-    return qnum, answer_id, guess_dicts, results, wordvecs\n+    return qnum, answer_id, guess_dicts, results\n \n-\n-class Word2Vec:\n-\n-    def __init__(self, wordvec_dir, wordvec_dim):\n-        self.word2vec = gensim.models.KeyedVectors.load_word2vec_format(\n-                wordvec_dir, binary=True)\n-        self.wordvec_dim = wordvec_dim\n-\n-    def get_avg_vec(self, guess):\n-        vecs = []\n-        for word in guess.split(\'_\'):\n-            if word in self.word2vec:\n-                vecs.append(self.word2vec[word])\n-        if len(vecs) > 0:\n-            return sum(vecs) / len(vecs)\n-        return self.get_zero_vec()\n-\n-    def get_zero_vec(self):\n-        return np.zeros(self.wordvec_dim, dtype=np.float32)\n-\n-\n-def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \\\n+def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, normalize=True) \\\n                     -> Tuple[Dict[str, int], Dict[str, list]]:\n-    merge_dfs()\n+    # merge_dfs()\n     log.info(\'Loading data\')\n     question_db = QuestionDatabase()\n     quizbowl_db = QuizBowlDataset(bc.MIN_ANSWERS, guesser_train=True, buzzer_train=True)\n@@ -180,7 +150,7 @@ def load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \\\n         log.info(\'Processing {0} guesses\'.format(fold))\n         guesses = AbstractGuesser.load_guesses(bc.GUESSES_DIR, folds=[fold])\n \n-        worker = partial(_process_question, option2id, all_questions, word2vec, normalize)\n+        worker = partial(_process_question, option2id, all_questions)\n         inputs = guesses.groupby(\'qnum\')\n         guesses_by_fold[fold] = _multiprocess(worker, inputs, info=\'df data\',\n                 multi=True)\n@@ -285,13 +255,6 @@ def load_protobowl():\n if __name__ == "__main__":\n     merge_dfs()\n \n-    processed_dirs = [\'%s_processed.pickle\' % (os.path.join(\n-        bc.GUESSES_DIR, fold)) for fold in c.BUZZER_INPUT_FOLDS]\n-    # if not all(os.path.isfile(d) for d in processed_dirs):\n-    #     log.info(\'Loading {0}\'.format(bc.WORDVEC_DIR))\n-    #     word2vec = Word2Vec(bc.WORDVEC_DIR, bc.WORDVEC_DIM)\n-    # else:\n-    word2vec = None\n-\n-    option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS, word2vec)\n-    load_protobowl()\n+    # option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS)\n+    # load_protobowl()\n+    option2id, guesses_by_fold = load_quizbowl()\n', 'addition_count': 10, 'deletion_count': 47, 'src_file': 'a/qanta/buzzer/util.py', 'tgt_file': 'b/qanta/buzzer/util.py', 'file_extension': '.py', 'before_file': 'import os\nimport pickle\nimport numpy as np\nimport chainer\nfrom multiprocessing import Pool\nfrom functools import partial\nfrom chainer import Variable\nfrom chainer.backends import cuda\nfrom qanta.datasets.quiz_bowl import QuizBowlDataset\nfrom qanta.guesser.abstract import AbstractGuesser\nfrom qanta.util.constants import BUZZER_DEV_FOLD, BUZZER_TRAIN_FOLD\n\n# constansts\nN_GUESSES = 10\nos.makedirs("output/buzzer", exist_ok=True)\ndataset_dir = "output/buzzer/{}_data.pkl"\n\n\ndef vector_converter_0(guesses_sequence):\n    """vector converter / feature extractor with only prob\n\n    Args:\n        guesses_sequence: a sequence (length of question) of list of guesses\n            (n_guesses), each entry is (guess, prob)\n    Returns:\n        a sequence of vectors\n    """\n    length = len(guesses_sequence)\n    prev_prob_vec = [0.0 for _ in range(N_GUESSES)]\n    prev_dict = dict()\n\n    vecs = []\n    for i in range(length):\n        prob_vec = []\n        prob_diff_vec = []\n        isnew_vec = []\n        guesses = guesses_sequence[i]\n        for guess, prob in guesses:\n            prob_vec.append(prob)\n            if i > 0 and guess in prev_dict:\n                prev_prob = prev_dict[guess]\n                prob_diff_vec.append(prob - prev_prob)\n                isnew_vec.append(0)\n            else:\n                prob_diff_vec.append(prob)\n                isnew_vec.append(1)\n        if len(guesses) < N_GUESSES:\n            for k in range(max(N_GUESSES - len(guesses), 0)):\n                prob_vec.append(0)\n                prob_diff_vec.append(0)\n                isnew_vec.append(0)\n        features = (\n            prob_vec[:3]\n            + isnew_vec[:3]\n            + prob_diff_vec[:3]\n            + [prob_vec[0] - prob_vec[1], prob_vec[1] - prob_vec[2]]\n            + [prob_vec[0] - prev_prob_vec[0], prob_vec[1] - prev_prob_vec[1]]\n            + [sum(isnew_vec[:5])]\n            + [np.average(prob_vec), np.average(prev_prob_vec)]\n            + [np.average(prob_vec[:6]), np.average(prev_prob_vec[:5])]\n            + [np.var(prob_vec), np.var(prev_prob_vec)]\n            + [np.var(prob_vec[:5]), np.var(prev_prob_vec[:5])]\n        )\n        vecs.append(np.array(features, dtype=np.float32))\n        prev_prob_vec = prob_vec\n        prev_dict = {g: p for g, p in guesses}\n    return vecs\n\n\ndef vector_converter_1(guesses_sequence):\n    """vector converter / feature extractor with both logit and prob\n\n    return buzz_dict\n\ndef _process_question(option2id: Dict[str, int], \n        all_questions: Dict[int, Question], \n        word2vec, normalize, qnum, question) -> \\\n            Tuple[int, int, List[List[Dict[str, int]]], List[List[int]]]:\n    \'\'\'Process one question.\n    return:\n    prev_logit_vec = [0.0 for _ in range(N_GUESSES)]\n    prev_prob_vec = [0.0 for _ in range(N_GUESSES)]\n    prev_dict = dict()\n\n    vecs = []\n    for i in range(length):\n        logit_vec = []\n        prob_vec = []\n        logit_diff_vec = []\n        prob_diff_vec = []\n        isnew_vec = []\n        guesses = guesses_sequence[i]\n        for guess, logit, prob in guesses:\n            logit_vec.append(logit)\n\n    guess_dicts = []\n    results = []\n    wordvecs = None\n    if word2vec is not None:\n        wordvecs = []\n    for pos, pos_group in question.groupby([\'sentence\', \'token\']):\n        pos_group = pos_group.groupby(\'guesser\')\n        guess_dicts.append([])\n        results.append([])\n        if word2vec is not None:\n            wordvecs.append([])\n        for guesser in GUESSERS:\n            if guesser not in pos_group.groups:\n                log.info("{0} missing guesser {1}.".format(qnum, guesser))\n                guess_dicts[-1].append({})\n                results[-1].append(0)\n                if word2vec is not None:\n                    wordvecs[-1].append(word2vec.get_zero_vec())\n            else:\n                guesses = pos_group.get_group(guesser)\n                guesses = guesses.sort_values(\'score\', ascending=False)\n                top_guess = guesses.iloc[0].guess\n                results[-1].append(int(top_guess == answer))\n                \n                s = sum(guesses.score) if normalize else 1\n                dic = {x.guess: x.score / s for x in guesses.itertuples()}\n                \n                guess_dicts[-1].append(dic)\n                if word2vec is not None:\n                    wordvecs[-1].append(word2vec.get_avg_vec(top_guess))\n\n    return qnum, answer_id, guess_dicts, results, wordvecs\n\n\nclass Word2Vec:\n\n    def __init__(self, wordvec_dir, wordvec_dim):\n        self.word2vec = gensim.models.KeyedVectors.load_word2vec_format(\n                wordvec_dir, binary=True)\n        self.wordvec_dim = wordvec_dim\n\n    def get_avg_vec(self, guess):\n        vecs = []\n        for word in guess.split(\'_\'):\n            if word in self.word2vec:\n                vecs.append(self.word2vec[word])\n        if len(vecs) > 0:\n            return sum(vecs) / len(vecs)\n        return self.get_zero_vec()\n\n    def get_zero_vec(self):\n        return np.zeros(self.wordvec_dim, dtype=np.float32)\n\n\ndef load_quizbowl(folds=c.BUZZER_INPUT_FOLDS, word2vec=None, normalize=False) \\\n                    -> Tuple[Dict[str, int], Dict[str, list]]:\n    merge_dfs()\n    log.info(\'Loading data\')\n    question_db = QuestionDatabase()\n    quizbowl_db = QuizBowlDataset(bc.MIN_ANSWERS, guesser_train=True, buzzer_train=True)\n            + [np.var(logit_vec), np.var(prev_logit_vec)]\n            + [np.var(prob_vec), np.var(prev_prob_vec)]\n            + [np.var(logit_vec[:5]), np.var(prev_logit_vec[:5])]\n            + [np.var(prob_vec[:5]), np.var(prev_prob_vec[:5])]\n        )\n        vecs.append(np.array(features, dtype=np.float32))\n        prev_logit_vec = logit_vec\n        prev_prob_vec = prob_vec\n        prev_dict = {x: (y, z) for x, y, z in guesses}\n    return vecs\n\n\ndef process_question(questions, vector_converter, item):\n    """multiprocessing worker that converts the guesser output of a single\n        question into format used by the buzzer\n    """\n    qid, q_rows = item\n    qid = q_rows.qanta_id.tolist()[0]\n    answer = questions[qid].page\n    q_rows = q_rows.groupby("char_index")\n    char_indices = sorted(q_rows.groups.keys())\n    guesses_sequence = []\n    labels = []\n    for idx in char_indices:\n        log.info(\'Processing {0} guesses\'.format(fold))\n        guesses = AbstractGuesser.load_guesses(bc.GUESSES_DIR, folds=[fold])\n\n        worker = partial(_process_question, option2id, all_questions, word2vec, normalize)\n        inputs = guesses.groupby(\'qnum\')\n        guesses_by_fold[fold] = _multiprocess(worker, inputs, info=\'df data\',\n                multi=True)\ndef read_data(\n    fold,\n    output_type="char",\n    guesser_module="qanta.guesser.rnn",\n    guesser_class="RnnGuesser",\n    guesser_config_num=0,\n    vector_converter=vector_converter_0,\n):\n\n    if os.path.isfile(dataset_dir.format(fold)):\n        with open(dataset_dir.format(fold), "rb") as f:\n            return pickle.load(f)\n\n    g_dir = AbstractGuesser.output_path(\n        guesser_module, guesser_class, guesser_config_num, ""\n    )\n    g_path = AbstractGuesser.guess_path(g_dir, fold, output_type)\n    with open(g_path, "rb") as f:\n        df = pickle.load(f)\n    df_groups = df.groupby("qanta_id")\n\n    questions = QuizBowlDataset(buzzer_train=True).questions_by_fold()\n    questions = {q.qanta_id: q for q in questions[fold]}\n\n    pool = Pool(8)\n    worker = partial(process_question, questions, vector_converter)\n    dataset = pool.map(worker, df_groups)\n\n    with open(dataset_dir.format(fold), "wb") as f:\n        pickle.dump(dataset, f)\n\n    return dataset\n\n\ndef convert_seq(batch, device=None):\n    def to_device_batch(batch):\n        if device is None:\n            return batch\n        elif device < 0:\n            return [chainer.dataset.to_device(device, x) for x in batch]\n        else:\n            xp = cuda.cupy.get_array_module(*batch)\n            concat = xp.concatenate(batch, axis=0)\n            sections = np.cumsum([len(x) for x in batch[:-1]], dtype=np.int32)\n            concat_dev = chainer.dataset.to_device(device, concat)\n            batch_dev = cuda.cupy.split(concat_dev, sections)\n            return batch_dev\n\n    qids, vectors, labels, positions = list(map(list, zip(*batch)))\n    xs = [Variable(x) for x in to_device_batch(vectors)]\n    ys = to_device_batch(labels)\n    return {"xs": xs, "ys": ys}\n\n\nif __name__ == "__main__":\n    data = read_data(BUZZER_TRAIN_FOLD)\n    print(data)\nif __name__ == "__main__":\n    merge_dfs()\n\n    processed_dirs = [\'%s_processed.pickle\' % (os.path.join(\n        bc.GUESSES_DIR, fold)) for fold in c.BUZZER_INPUT_FOLDS]\n    # if not all(os.path.isfile(d) for d in processed_dirs):\n    #     log.info(\'Loading {0}\'.format(bc.WORDVEC_DIR))\n    #     word2vec = Word2Vec(bc.WORDVEC_DIR, bc.WORDVEC_DIM)\n    # else:\n    word2vec = None\n\n    option2id, guesses_by_fold = load_quizbowl(c.BUZZER_INPUT_FOLDS, word2vec)\n    load_protobowl()\n', 'message': "don't normalize!\n", 'author_name': 'ihsgnef', 'lang': 'py'}