parameters
                mean, log_std) of a Gaussian distribution over actions
                    sy_mean: (batch_size, self.ac_dim)
                    sy_logstd: (batch_size, self.ac_dim)

            sy_ac_na: (batch_size, self.ac_dim)

        returns:
            sy_lp_n: (batch_size)

        """
        sy_mean, sy_logstd = policy_parameters
        sy_lp_n = tfp.distributions.MultivariateNormalDiag(
            loc=sy_mean, scale_diag=tf.exp(sy_logstd)).log_prob(sy_ac_na)
        return sy_lp_n

    def build_computation_graph(self):
        """
        notes on notation:

        Symbolic variables have the prefix sy_, to distinguish them from the numerical values
        that are computed later in the function

        prefixes and suffixes:
        ob - observation
        ac - action
        _no - this tensor should have shape (batch self.size /n/, observation dim)
        _na - this tensor should have shape (batch self.size /n/, action dim)
        _n  - this tensor should have shape (batch self.size /n/)

        Note: batch self.size /n/ is defined at runtime, and until then, the shape for that axis
        is None

        ----------------------------------------------------------------------------------
        loss: a function of self.sy_lp_n and self.sy_adv_n that we will differentiate
            to get the policy gradient.
        """
        self.sy_ob_no, self.sy_ac_na, self.sy_adv_n, self.sy_hidden, self.sy_lp_n, self.sy_fixed_lp_n = self.define_placeholders()

        # The policy takes in an observation and produces a distribution over the action space
        policy_outputs = self.policy_forward_pass(self.sy_ob_no, self.sy_hidden)
        self.policy_parameters = policy_outputs[:-1]

        # unpack mean and variance
        self.policy_parameters = tf.split(self.policy_parameters[0], 2, axis=1)

        # We can sample actions from this action distribution.
        # This will be called in Agent.sample_trajectory() where we generate a rollout.
        self.sy_sampled_ac = self.sample_action(self.policy_parameters)

        # We can also compute the logprob of the actions that were actually taken by the policy
        # This is used in the loss function.
        self.sy_lp_n = self.get_log_prob(self.policy_parameters, self.sy_ac_na)

        # PPO critic update
        critic_regularizer = tf.contrib.layers.l2_regularizer(1e-3) if self.l2reg else None
        self.critic_prediction = tf.squeeze(build_critic(self.sy_ob_no, self.sy_hidden, 1, 'critic_network', n_layers=self.n_layers, size=self.size, gru_size=self.gru_size, recurrent=self.recurrent, regularizer=critic_regularizer))
        self.sy_target_n = tf.placeholder(shape=[None], name="critic_target", dtype=tf.float32)
        self.critic_loss = tf.losses.mean_squared_error(self.sy_target_n, self.critic_prediction)
        self.critic_weights = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='critic_network')
        self.critic_update_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.critic_loss)

        # PPO actor update
        self.sy_fixed_log_prob_n = tf.placeholder(shape=[None], name="fixed_log_prob", dtype=tf.float32)
        self.policy_surr_loss = self.ppo_loss(self.sy_lp_n, self.sy_fixed_lp_n, self.sy_adv_n)
        self.policy_weights = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.scope)
        optimizer = tf.train.AdamOptimizer(self.learning_rate)
        self.policy_update_op = minimize_and_clip(optimizer, self.policy_surr_loss, var_list=self.policy_weights, clip_val=40)

    def sample_trajectories(self, itr, env, min_timesteps, is_evaluation=False):
        # Collect paths until we have enough timesteps
        timesteps_this_batch = 0
        stats = []
        while True:
            animate_this_episode=(len(stats)==0 and (itr % 10 == 0) and self.animate)
            steps, s = self.sample_trajectory(env, animate_this_episode, is_evaluation=is_evaluation)
            stats += s
            timesteps_this_batch += steps
            if timesteps_this_batch > min_timesteps:
                break
        return stats, timesteps_this_batch

    def sample_trajectory(self, env, animate_this_episode, is_evaluation):
        """
        sample a task, then sample trajectories from that task until either
        max(self.history, self.max_path_length) timesteps have been sampled

        construct meta-observations by concatenating (s, a, r, d) into one vector
        inputs to the policy should have the shape (batch_size, self.history, self.meta_ob_dim)
        zero pad the input to maintain a consistent input shape

        add the entire input as observation to the replay buffer, along with a, r, d
        samples will be drawn from the replay buffer to update the policy

        arguments:
            env: the env to sample trajectories from
            animate_this_episode: if True then render
            val: whether this is training or evaluation
        """
        env.reset_task(is_evaluation=is_evaluation)
        stats = []
        #====================================================================================#
        #                           ----------PROBLEM 1----------
        #====================================================================================#
        ep_steps = 0
        steps = 0

        num_samples = max(self.history, self.max_path_length + 1)
        meta_obs = np.zeros((num_samples + self.history + 1, self.meta_ob_dim))
        rewards = []

        while True:
            if animate_this_episode:
                env.render()
                time.sleep(0.1)

            if ep_steps == 0:
                ob = env.reset()
                # first meta ob has only the observation
                # set a, r, d to zero, construct first meta observation in meta_obs
                # YOUR CODE HERE

                a = np.zeros((self.ac_dim,), dtype=np.float32)
                r = np.zeros((self.reward_dim,), dtype=np.float32)
                d = np.zeros((self.terminal_dim,), dtype=np.float32)

                meta_obs[ep_steps + self.history - 1,:] = np.concatenate([ob, a, r, d], axis=0)

                steps += 1

            # index into the meta_obs array to get the window that ends with the current timestep
            # please name the windowed observation `in_` for compatibilty with the code that adds to the replay buffer (lines 418, 420)
            # YOUR CODE HERE

            in_ = np.expand_dims(meta_obs[ep_steps:(ep_steps + self.history),:], axis=0)
            hidden = np.zeros((1, self.gru_size), dtype=np.float32)

            # get action from the policy
            # YOUR CODE HERE
            ac = self.sess.run(self.sy_sampled_ac, feed_dict={self.sy_ob_no: in_, self.sy_hidden: hidden})
            ac = ac[0]

            # step the environment
            # YOUR CODE HERE
            ob, rew, done, _ = env.step(ac)
            ep_steps += 1

            rew = np.array([rew], dtype=np.float32)
            done = bool(done) or ep_steps == self.max_path_length
            # construct the meta-observation and add it to meta_obs
            # YOUR CODE HERE            
            meta_obs[ep_steps + self