def run_episode(batch, agent_models, batch_size, testing, render=False, initial_agent=0):
    '\n    turning testing on means, we disable stochasticity: always pick the argmax\n    '
    s = State(**batch)
    sieve = AliveSieve(batch_size=batch_size)
    actions_by_timestep = []
    alive_masks = []
    rewards = torch.zeros(batch_size, 3, device=FLAGS.device)
    num_steps = torch.full((batch_size,), FLAGS.max_timesteps, dtype=torch.int64, device=FLAGS.device)
    term_matches_argmax_count = 0
    utt_matches_argmax_count = 0
    utt_stochastic_draws = 0
    num_policy_runs = 0
    prop_matches_argmax_count = 0
    prop_stochastic_draws = 0
    utt_mask = torch.zeros(2, batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    prop_mask = torch.zeros(2, batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    entropy_loss_by_agent = [torch.zeros(1, device=FLAGS.device), torch.zeros(1, device=FLAGS.device)]
    if render:
        print('  ')
        print('           ', '{}   {}   {}'.format(*s.utilities[0][0].tolist()), '      ', '{} {} {}'.format(*s.pool[0].tolist()), '          ', '{}   {}   {}'.format(*s.utilities[0][1].tolist()))
    current_A_proposal = torch.zeros(sieve.batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    prev_A_proposal = torch.zeros(sieve.batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    current_A_message = torch.zeros(sieve.batch_size, FLAGS.utt_max_length, dtype=torch.int64, device=FLAGS.device)
    prev_A_message = torch.zeros(sieve.batch_size, FLAGS.utt_max_length, dtype=torch.int64, device=FLAGS.device)
    current_A_term = torch.zeros(sieve.batch_size, 1, dtype=torch.uint8)
    for t in range(FLAGS.max_timesteps):
        if FLAGS.linguistic:
            if (FLAGS.normal_form and ((t % 2) == 1)):
                _prev_message = prev_A_message
            else:
                _prev_message = s.m_prev
        else:
            _prev_message = torch.zeros(sieve.batch_size, 6, dtype=torch.int64, device=FLAGS.device)
        if FLAGS.proposal:
            if (FLAGS.normal_form and ((t % 2) == 1)):
                _prev_proposal = prev_A_proposal
            else:
                _prev_proposal = s.last_proposal
        else:
            _prev_proposal = torch.zeros(sieve.batch_size, 3, dtype=torch.int64, device=FLAGS.device)
        agent = ((initial_agent + t) % 2)
        agent_model = agent_models[agent]
        (nodes, term_a, s.m_prev, this_proposal, _entropy_loss, _term_matches_argmax_count, _utt_matches_argmax_count, _utt_stochastic_draws, _prop_matches_argmax_count, _prop_stochastic_draws, _utt_mask, _prop_mask) = agent_model(pool=s.pool, utility=s.utilities[:, agent], m_prev=_prev_message, prev_proposal=_prev_proposal, testing=testing)
        entropy_loss_by_agent[agent] += _entropy_loss
        actions_by_timestep.append(nodes)
        term_matches_argmax_count += _term_matches_argmax_count
        num_policy_runs += sieve.batch_size
        utt_matches_argmax_count += _utt_matches_argmax_count
        utt_stochastic_draws += _utt_stochastic_draws
        prop_matches_argmax_count += _prop_matches_argmax_count
        prop_stochastic_draws += _prop_stochastic_draws
        if FLAGS.force_masking_comm:
            utt_mask[agent][sieve.out_idxes] |= _utt_mask
            prop_mask[agent][sieve.out_idxes] |= _prop_mask
        if (FLAGS.proposal_termination and (not FLAGS.normal_form)):
            term_a = torch.prod((this_proposal == _prev_proposal), dim=1, keepdim=True)
        elif ((not FLAGS.proposal_termination) and FLAGS.normal_form):
            if ((t % 2) == 1):
                term_a = (term_a * current_A_term)
            else:
                current_A_term = term_a
                term_a = torch.zeros((sieve.batch_size, 1), dtype=torch.uint8, device=FLAGS.device)
        elif (FLAGS.proposal_termination and FLAGS.normal_form):
            if ((t % 2) == 1):
                term_a = torch.prod((this_proposal == current_A_proposal), dim=1, keepdim=True)
            else:
                term_a = torch.zeros((sieve.batch_size, 1), dtype=torch.uint8, device=FLAGS.device)
        if (render and (sieve.out_idxes[0] == 0)):
            render_action(t=t, s=s, term=term_a, prop=this_proposal)
        new_rewards = calc_rewards(t=t, s=s, term=term_a, agent=agent)
        rewards[sieve.out_idxes] = new_rewards
        s.last_proposal = this_proposal
        if (FLAGS.normal_form and ((t % 2) == 0)):
            prev_A_proposal = current_A_proposal
            current_A_proposal = this_proposal
            prev_A_message = current_A_message
            current_A_message = s.m_prev
        sieve.mark_dead(term_a)
        sieve.mark_dead(((t + 1) >= s.N))
        alive_masks.append(sieve.alive_mask.clone())
        sieve.set_dead_global(num_steps, (t + 1))
        if sieve.all_dead():
            break
        s.sieve_(sieve.alive_idxes)
        if FLAGS.normal_form:
            current_A_proposal = current_A_proposal[sieve.alive_idxes]
            prev_A_proposal = prev_A_proposal[sieve.alive_idxes]
            current_A_message = current_A_message[sieve.alive_idxes]
            prev_A_message = prev_A_message[sieve.alive_idxes]
        sieve.self_sieve_()
    if render:
        print(' rewards: {:2.2f} {:2.2f} {:2.2f}'.format(*rewards[0].tolist()))
        print('  ')
    utt_mask_count = utt_mask.sum(dim=[1, 2]).cpu().numpy()
    prop_mask_count = prop_mask.sum(dim=[1, 2]).cpu().numpy()
    return (actions_by_timestep, rewards, num_steps, alive_masks, entropy_loss_by_agent, term_matches_argmax_count, num_policy_runs, utt_matches_argmax_count, utt_stochastic_draws, prop_matches_argmax_count, prop_stochastic_draws, utt_mask_count, prop_mask_count)
turning testing on means, we disable stochasticity: always pick the argmax
def run_episode(batch, agent_models, batch_size, testing, render=False, initial_agent=0):
    '\n    \n    '
    s = State(**batch)
    sieve = AliveSieve(batch_size=batch_size)
    actions_by_timestep = []
    alive_masks = []
    rewards = torch.zeros(batch_size, 3, device=FLAGS.device)
    num_steps = torch.full((batch_size,), FLAGS.max_timesteps, dtype=torch.int64, device=FLAGS.device)
    term_matches_argmax_count = 0
    utt_matches_argmax_count = 0
    utt_stochastic_draws = 0
    num_policy_runs = 0
    prop_matches_argmax_count = 0
    prop_stochastic_draws = 0
    utt_mask = torch.zeros(2, batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    prop_mask = torch.zeros(2, batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    entropy_loss_by_agent = [torch.zeros(1, device=FLAGS.device), torch.zeros(1, device=FLAGS.device)]
    if render:
        print('  ')
        print('           ', '{}   {}   {}'.format(*s.utilities[0][0].tolist()), '      ', '{} {} {}'.format(*s.pool[0].tolist()), '          ', '{}   {}   {}'.format(*s.utilities[0][1].tolist()))
    current_A_proposal = torch.zeros(sieve.batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    prev_A_proposal = torch.zeros(sieve.batch_size, 3, dtype=torch.int64, device=FLAGS.device)
    current_A_message = torch.zeros(sieve.batch_size, FLAGS.utt_max_length, dtype=torch.int64, device=FLAGS.device)
    prev_A_message = torch.zeros(sieve.batch_size, FLAGS.utt_max_length, dtype=torch.int64, device=FLAGS.device)
    current_A_term = torch.zeros(sieve.batch_size, 1, dtype=torch.uint8)
    for t in range(FLAGS.max_timesteps):
        if FLAGS.linguistic:
            if (FLAGS.normal_form and ((t % 2) == 1)):
                _prev_message = prev_A_message
            else:
                _prev_message = s.m_prev
        else:
            _prev_message = torch.zeros(sieve.batch_size, 6, dtype=torch.int64, device=FLAGS.device)
        if FLAGS.proposal:
            if (FLAGS.normal_form and ((t % 2) == 1)):
                _prev_proposal = prev_A_proposal
            else:
                _prev_proposal = s.last_proposal
        else:
            _prev_proposal = torch.zeros(sieve.batch_size, 3, dtype=torch.int64, device=FLAGS.device)
        agent = ((initial_agent + t) % 2)
        agent_model = agent_models[agent]
        (nodes, term_a, s.m_prev, this_proposal, _entropy_loss, _term_matches_argmax_count, _utt_matches_argmax_count, _utt_stochastic_draws, _prop_matches_argmax_count, _prop_stochastic_draws, _utt_mask, _prop_mask) = agent_model(pool=s.pool, utility=s.utilities[:, agent], m_prev=_prev_message, prev_proposal=_prev_proposal, testing=testing)
        entropy_loss_by_agent[agent] += _entropy_loss
        actions_by_timestep.append(nodes)
        term_matches_argmax_count += _term_matches_argmax_count
        num_policy_runs += sieve.batch_size
        utt_matches_argmax_count += _utt_matches_argmax_count
        utt_stochastic_draws += _utt_stochastic_draws
        prop_matches_argmax_count += _prop_matches_argmax_count
        prop_stochastic_draws += _prop_stochastic_draws
        if FLAGS.force_masking_comm:
            utt_mask[agent][sieve.out_idxes] |= _utt_mask
            prop_mask[agent][sieve.out_idxes] |= _prop_mask
        if (FLAGS.proposal_termination and (not FLAGS.normal_form)):
            term_a = torch.prod((this_proposal == _prev_proposal), dim=1, keepdim=True)
        elif ((not FLAGS.proposal_termination) and FLAGS.normal_form):
            if ((t % 2) == 1):
                term_a = (term_a * current_A_term)
            else:
                current_A_term = term_a
                term_a = torch.zeros((sieve.batch_size, 1), dtype=torch.uint8, device=FLAGS.device)
        elif (FLAGS.proposal_termination and FLAGS.normal_form):
            if ((t % 2) == 1):
                term_a = torch.prod((this_proposal == current_A_proposal), dim=1, keepdim=True)
            else:
                term_a = torch.zeros((sieve.batch_size, 1), dtype=torch.uint8, device=FLAGS.device)
        if (render and (sieve.out_idxes[0] == 0)):
            render_action(t=t, s=s, term=term_a, prop=this_proposal)
        new_rewards = calc_rewards(t=t, s=s, term=term_a, agent=agent)
        rewards[sieve.out_idxes] = new_rewards
        s.last_proposal = this_proposal
        if (FLAGS.normal_form and ((t % 2) == 0)):
            prev_A_proposal = current_A_proposal
            current_A_proposal = this_proposal
            prev_A_message = current_A_message
            current_A_message = s.m_prev
        sieve.mark_dead(term_a)
        sieve.mark_dead(((t + 1) >= s.N))
        alive_masks.append(sieve.alive_mask.clone())
        sieve.set_dead_global(num_steps, (t + 1))
        if sieve.all_dead():
            break
        s.sieve_(sieve.alive_idxes)
        if FLAGS.normal_form:
            current_A_proposal = current_A_proposal[sieve.alive_idxes]
            prev_A_proposal = prev_A_proposal[sieve.alive_idxes]
            current_A_message = current_A_message[sieve.alive_idxes]
            prev_A_message = prev_A_message[sieve.alive_idxes]
        sieve.self_sieve_()
    if render:
        print(' rewards: {:2.2f} {:2.2f} {:2.2f}'.format(*rewards[0].tolist()))
        print('  ')
    utt_mask_count = utt_mask.sum(dim=[1, 2]).cpu().numpy()
    prop_mask_count = prop_mask.sum(dim=[1, 2]).cpu().numpy()
    return (actions_by_timestep, rewards, num_steps, alive_masks, entropy_loss_by_agent, term_matches_argmax_count, num_policy_runs, utt_matches_argmax_count, utt_stochastic_draws, prop_matches_argmax_count, prop_stochastic_draws, utt_mask_count, prop_mask_count)