'Returns:
xs = [memories, queries, memory_lengths, query_lengths]
ys = [labels, label_lengths] (if available, else None)
cands = list of candidates for each example in batch
valid_inds = list of indices for examples with valid observations'

def batchify(self, obs):

    exs = [ex for ex in obs if ('text' in ex)]
    valid_inds = [i for (i, ex) in enumerate(obs) if ('text' in ex)]
    parsed = [self.parse(ex['text']) for ex in exs]
    queries = torch.cat([x[0] for x in parsed])
    memories = torch.cat([x[1] for x in parsed])
    query_lengths = torch.cat([x[2] for x in parsed])
    memory_lengths = torch.LongTensor(len(exs), self.mem_size).zero_()
    for i in range(len(exs)):
        if (len(parsed[i][3]) > 0):
            memory_lengths[i, (- len(parsed[i][3])):] = parsed[i][3]
    xs = [memories, queries, memory_lengths, query_lengths]
    ys = None
    self.labels = [random.choice(ex['labels']) for ex in exs if ('labels' in ex)]
    if (len(self.labels) == len(exs)):
        parsed = [self.dict.txt2vec(l) for l in self.labels]
        parsed = [torch.LongTensor(p) for p in parsed]
        label_lengths = torch.LongTensor([len(p) for p in parsed]).unsqueeze(1)
        labels = torch.cat(parsed)
        ys = [labels, label_lengths]
    cands = [ex['label_candidates'] for ex in exs if ('label_candidates' in ex)]
    if (len(cands) < len(exs)):
        cands = build_cands(exs, self.dict)
    if (self.last_cands != cands):
        self.last_cands = cands
        self.last_cands_list = [list(c) for c in cands]
    cands = self.last_cands_list
    return (xs, ys, cands, valid_inds)
