def rollout(self, train_ml=None, train_rl=True, reset=True, speaker=None):
    '\n        :param train_ml:    The weight to train with maximum likelihood\n        :param train_rl:    whether use RL in training\n        :param reset:       Reset the environment\n        :param speaker:     Speaker used in back translation.\n                            If the speaker is not None, use back translation.\n                            O.w., normal training\n        :return:\n        '
    if ((self.feedback == 'teacher') or (self.feedback == 'argmax')):
        train_rl = False
    if reset:
        obs = np.array(self.env.reset())
    else:
        obs = np.array(self.env._get_obs())
    batch_size = len(obs)
    if (speaker is not None):
        noise = self.decoder.drop_env(torch.ones(self.feature_size).cuda())
        batch = self.env.batch.copy()
        speaker.env = self.env
        insts = speaker.infer_batch(featdropmask=noise)
        boss = (np.ones((batch_size, 1), np.int64) * self.tok.word_to_index['<BOS>'])
        insts = np.concatenate((boss, insts), 1)
        for (i, (datum, inst)) in enumerate(zip(batch, insts)):
            if (inst[(- 1)] != self.tok.word_to_index['<PAD>']):
                inst[(- 1)] = self.tok.word_to_index['<EOS>']
            datum.pop('instructions')
            datum.pop('instr_encoding')
            datum['instructions'] = self.tok.decode_sentence(inst)
            datum['instr_encoding'] = inst
        obs = np.array(self.env.reset(batch))
    (seq, seq_mask, seq_lengths, perm_idx) = self._sort_batch(obs)
    perm_obs = obs[perm_idx]
    (ctx, h_t, c_t) = self.encoder(seq, seq_lengths)
    ctx_mask = seq_mask
    last_dist = np.zeros(batch_size, np.float32)
    for (i, ob) in enumerate(perm_obs):
        last_dist[i] = ob['distance']
    traj = [{'instr_id': ob['instr_id'], 'path': [(ob['viewpoint'], ob['heading'], ob['elevation'])]} for ob in perm_obs]
    visited = [set() for _ in perm_obs]
    ended = np.array(([False] * batch_size))
    rewards = []
    hidden_states = []
    policy_log_probs = []
    masks = []
    entropys = []
    ml_loss = 0.0
    h1 = h_t
    for t in range(self.episode_len):
        (input_a_t, f_t, candidate_feat, candidate_leng) = self.get_input_feat(perm_obs)
        if (speaker is not None):
            candidate_feat[..., :(- args.angle_feat_size)] *= noise
            f_t[..., :(- args.angle_feat_size)] *= noise
        (h_t, c_t, logit, h1) = self.decoder(input_a_t, f_t, candidate_feat, h_t, h1, c_t, ctx, ctx_mask, already_dropfeat=(speaker is not None))
        hidden_states.append(h_t)
        candidate_mask = utils.length2mask(candidate_leng)
        if args.submit:
            for (ob_id, ob) in enumerate(perm_obs):
                visited[ob_id].add(ob['viewpoint'])
                for (c_id, c) in enumerate(ob['candidate']):
                    if (c['viewpointId'] in visited[ob_id]):
                        candidate_mask[ob_id][c_id] = 1
        logit.masked_fill_(candidate_mask, (- float('inf')))
        target = self._teacher_action(perm_obs, ended)
        ml_loss += self.criterion(logit, target)
        if (self.feedback == 'teacher'):
            a_t = target
        elif (self.feedback == 'argmax'):
            (_, a_t) = logit.max(1)
            a_t = a_t.detach()
            log_probs = F.log_softmax(logit, 1)
            policy_log_probs.append(log_probs.gather(1, a_t.unsqueeze(1)))
        elif (self.feedback == 'sample'):
            probs = F.softmax(logit, 1)
            c = torch.distributions.Categorical(probs)
            self.logs['entropy'].append(c.entropy().sum().item())
            entropys.append(c.entropy())
            a_t = c.sample().detach()
            policy_log_probs.append(c.log_prob(a_t))
        else:
            print(self.feedback)
            sys.exit('Invalid feedback option')
        cpu_a_t = a_t.cpu().numpy()
        for (i, next_id) in enumerate(cpu_a_t):
            if ((next_id == (candidate_leng[i] - 1)) or (next_id == args.ignoreid) or ended[i]):
                cpu_a_t[i] = (- 1)
        self.make_equiv_action(cpu_a_t, perm_obs, perm_idx, traj)
        obs = np.array(self.env._get_obs())
        perm_obs = obs[perm_idx]
        dist = np.zeros(batch_size, np.float32)
        reward = np.zeros(batch_size, np.float32)
        mask = np.ones(batch_size, np.float32)
        for (i, ob) in enumerate(perm_obs):
            dist[i] = ob['distance']
            if ended[i]:
                reward[i] = 0.0
                mask[i] = 0.0
            else:
                action_idx = cpu_a_t[i]
                if (action_idx == (- 1)):
                    if (dist[i] < 3):
                        reward[i] = 2.0
                    else:
                        reward[i] = (- 2.0)
                else:
                    reward[i] = (- (dist[i] - last_dist[i]))
                    if (reward[i] > 0):
                        reward[i] = 1
                    elif (reward[i] < 0):
                        reward[i] = (- 1)
                    else:
                        raise NameError("The action doesn't change the move")
        rewards.append(reward)
        masks.append(mask)
        last_dist[:] = dist
        ended[:] = np.logical_or(ended, (cpu_a_t == (- 1)))
        if ended.all():
            break
    if train_rl:
        (input_a_t, f_t, candidate_feat, candidate_leng) = self.get_input_feat(perm_obs)
        if (speaker is not None):
            candidate_feat[..., :(- args.angle_feat_size)] *= noise
            f_t[..., :(- args.angle_feat_size)] *= noise
        (last_h_, _, _, _) = self.decoder(input_a_t, f_t, candidate_feat, h_t, h1, c_t, ctx, ctx_mask, (speaker is not None))
        rl_loss = 0.0
        last_value__ = self.critic(last_h_).detach()
        discount_reward = np.zeros(batch_size, np.float32)
        for i in range(batch_size):
            if (not ended[i]):
                discount_reward[i] = last_value__[i]
        length = len(rewards)
        total = 0
        for t in range((length - 1), (- 1), (- 1)):
            discount_reward = ((discount_reward * args.gamma) + rewards[t])
            mask_ = Variable(torch.from_numpy(masks[t]), requires_grad=False).cuda()
            clip_reward = discount_reward.copy()
            r_ = Variable(torch.from_numpy(clip_reward), requires_grad=False).cuda()
            v_ = self.critic(hidden_states[t])
            a_ = (r_ - v_).detach()
            rl_loss += (((- policy_log_probs[t]) * a_) * mask_).sum()
            rl_loss += ((((r_ - v_) ** 2) * mask_).sum() * 0.5)
            if (self.feedback == 'sample'):
                rl_loss += (((- 0.01) * entropys[t]) * mask_).sum()
            self.logs['critic_loss'].append((((r_ - v_) ** 2) * mask_).sum().item())
            total = (total + np.sum(masks[t]))
        self.logs['total'].append(total)
        if (args.normalize_loss == 'total'):
            rl_loss /= total
        elif (args.normalize_loss == 'batch'):
            rl_loss /= batch_size
        else:
            assert (args.normalize_loss == 'none')
        self.loss += rl_loss
    if (train_ml is not None):
        self.loss += ((ml_loss * train_ml) / batch_size)
    if (type(self.loss) is int):
        self.losses.append(0.0)
    else:
        self.losses.append((self.loss.item() / self.episode_len))
    return traj
:param train_ml:    The weight to train with maximum likelihood
:param train_rl:    whether use RL in training
:param reset:       Reset the environment
:param speaker:     Speaker used in back translation.
                    If the speaker is not None, use back translation.
                    O.w., normal training
:return:
def rollout(self, train_ml=None, train_rl=True, reset=True, speaker=None):
    '\n        :param train_ml:    The weight to train with maximum likelihood\n        :param train_rl:    whether use RL in training\n        :param reset:       Reset the environment\n        :param speaker:     Speaker used in back translation.\n                            If the speaker is not None, use back translation.\n                            O.w., normal training\n        :return:\n        '
    if ((self.feedback == 'teacher') or (self.feedback == 'argmax')):
        train_rl = False
    if reset:
        obs = np.array(self.env.reset())
    else:
        obs = np.array(self.env._get_obs())
    batch_size = len(obs)
    if (speaker is not None):
        noise = self.decoder.drop_env(torch.ones(self.feature_size).cuda())
        batch = self.env.batch.copy()
        speaker.env = self.env
        insts = speaker.infer_batch(featdropmask=noise)
        boss = (np.ones((batch_size, 1), np.int64) * self.tok.word_to_index['<BOS>'])
        insts = np.concatenate((boss, insts), 1)
        for (i, (datum, inst)) in enumerate(zip(batch, insts)):
            if (inst[(- 1)] != self.tok.word_to_index['<PAD>']):
                inst[(- 1)] = self.tok.word_to_index['<EOS>']
            datum.pop('instructions')
            datum.pop('instr_encoding')
            datum['instructions'] = self.tok.decode_sentence(inst)
            datum['instr_encoding'] = inst
        obs = np.array(self.env.reset(batch))
    (seq, seq_mask, seq_lengths, perm_idx) = self._sort_batch(obs)
    perm_obs = obs[perm_idx]
    (ctx, h_t, c_t) = self.encoder(seq, seq_lengths)
    ctx_mask = seq_mask
    last_dist = np.zeros(batch_size, np.float32)
    for (i, ob) in enumerate(perm_obs):
        last_dist[i] = ob['distance']
    traj = [{'instr_id': ob['instr_id'], 'path': [(ob['viewpoint'], ob['heading'], ob['elevation'])]} for ob in perm_obs]
    visited = [set() for _ in perm_obs]
    ended = np.array(([False] * batch_size))
    rewards = []
    hidden_states = []
    policy_log_probs = []
    masks = []
    entropys = []
    ml_loss = 0.0
    h1 = h_t
    for t in range(self.episode_len):
        (input_a_t, f_t, candidate_feat, candidate_leng) = self.get_input_feat(perm_obs)
        if (speaker is not None):
            candidate_feat[..., :(- args.angle_feat_size)] *= noise
            f_t[..., :(- args.angle_feat_size)] *= noise
        (h_t, c_t, logit, h1) = self.decoder(input_a_t, f_t, candidate_feat, h_t, h1, c_t, ctx, ctx_mask, already_dropfeat=(speaker is not None))
        hidden_states.append(h_t)
        candidate_mask = utils.length2mask(candidate_leng)
        if args.submit:
            for (ob_id, ob) in enumerate(perm_obs):
                visited[ob_id].add(ob['viewpoint'])
                for (c_id, c) in enumerate(ob['candidate']):
                    if (c['viewpointId'] in visited[ob_id]):
                        candidate_mask[ob_id][c_id] = 1
        logit.masked_fill_(candidate_mask, (- float('inf')))
        target = self._teacher_action(perm_obs, ended)
        ml_loss += self.criterion(logit, target)
        if (self.feedback == 'teacher'):
            a_t = target
        elif (self.feedback == 'argmax'):
            (_, a_t) = logit.max(1)
            a_t = a_t.detach()
            log_probs = F.log_softmax(logit, 1)
            policy_log_probs.append(log_probs.gather(1, a_t.unsqueeze(1)))
        elif (self.feedback == 'sample'):
            probs = F.softmax(logit, 1)
            c = torch.distributions.Categorical(probs)
            self.logs['entropy'].append(c.entropy().sum().item())
            entropys.append(c.entropy())
            a_t = c.sample().detach()
            policy_log_probs.append(c.log_prob(a_t))
        else:
            print(self.feedback)
            sys.exit('Invalid feedback option')
        cpu_a_t = a_t.cpu().numpy()
        for (i, next_id) in enumerate(cpu_a_t):
            if ((next_id == (candidate_leng[i] - 1)) or (next_id == args.ignoreid) or ended[i]):
                cpu_a_t[i] = (- 1)
        self.make_equiv_action(cpu_a_t, perm_obs, perm_idx, traj)
        obs = np.array(self.env._get_obs())
        perm_obs = obs[perm_idx]
        dist = np.zeros(batch_size, np.float32)
        reward = np.zeros(batch_size, np.float32)
        mask = np.ones(batch_size, np.float32)
        for (i, ob) in enumerate(perm_obs):
            dist[i] = ob['distance']
            if ended[i]:
                reward[i] = 0.0
                mask[i] = 0.0
            else:
                action_idx = cpu_a_t[i]
                if (action_idx == (- 1)):
                    if (dist[i] < 3):
                        reward[i] = 2.0
                    else:
                        reward[i] = (- 2.0)
                else:
                    reward[i] = (- (dist[i] - last_dist[i]))
                    if (reward[i] > 0):
                        reward[i] = 1
                    elif (reward[i] < 0):
                        reward[i] = (- 1)
                    else:
                        raise NameError("The action doesn't change the move")
        rewards.append(reward)
        masks.append(mask)
        last_dist[:] = dist
        ended[:] = np.logical_or(ended, (cpu_a_t == (- 1)))
        if ended.all():
            break
    if train_rl:
        (input_a_t, f_t, candidate_feat, candidate_leng) = self.get_input_feat(perm_obs)
        if (speaker is not None):
            candidate_feat[..., :(- args.angle_feat_size)] *= noise
            f_t[..., :(- args.angle_feat_size)] *= noise
        (last_h_, _, _, _) = self.decoder(input_a_t, f_t, candidate_feat, h_t, h1, c_t, ctx, ctx_mask, (speaker is not None))
        rl_loss = 0.0
        last_value__ = self.critic(last_h_).detach()
        discount_reward = np.zeros(batch_size, np.float32)
        for i in range(batch_size):
            if (not ended[i]):
                discount_reward[i] = last_value__[i]
        length = len(rewards)
        total = 0
        for t in range((length - 1), (- 1), (- 1)):
            discount_reward = ((discount_reward * args.gamma) + rewards[t])
            mask_ = Variable(torch.from_numpy(masks[t]), requires_grad=False).cuda()
            clip_reward = discount_reward.copy()
            r_ = Variable(torch.from_numpy(clip_reward), requires_grad=False).cuda()
            v_ = self.critic(hidden_states[t])
            a_ = (r_ - v_).detach()
            rl_loss += (((- policy_log_probs[t]) * a_) * mask_).sum()
            rl_loss += ((((r_ - v_) ** 2) * mask_).sum() * 0.5)
            if (self.feedback == 'sample'):
                rl_loss += (((- 0.01) * entropys[t]) * mask_).sum()
            self.logs['critic_loss'].append((((r_ - v_) ** 2) * mask_).sum().item())
            total = (total + np.sum(masks[t]))
        self.logs['total'].append(total)
        if (args.normalize_loss == 'total'):
            rl_loss /= total
        elif (args.normalize_loss == 'batch'):
            rl_loss /= batch_size
        else:
            assert (args.normalize_loss == 'none')
        self.loss += rl_loss
    if (train_ml is not None):
        self.loss += ((ml_loss * train_ml) / batch_size)
    if (type(self.loss) is int):
        self.losses.append(0.0)
    else:
        self.losses.append((self.loss.item() / self.episode_len))
    return traj