def learn(*,<EOL>network,<EOL>env,<EOL>total_timesteps,<EOL>timesteps_per_batch=<NUM_LIT>, <EOL>max_kl=<NUM_LIT>,<EOL>cg_iters=<NUM_LIT:10>,<EOL>gamma=<NUM_LIT>,<EOL>lam=<NUM_LIT:1.0>, <EOL>seed=None,<EOL>ent_coef=<NUM_LIT:0.0>,<EOL>cg_damping=<NUM_LIT>,<EOL>vf_stepsize=<NUM_LIT>,<EOL>vf_iters =<NUM_LIT:3>,<EOL>max_episodes=<NUM_LIT:0>, max_iters=<NUM_LIT:0>,  <EOL>callback=None,<EOL>load_path=None,<EOL>**network_kwargs<EOL>):
if MPI is not None:<EOL><INDENT>nworkers = MPI.COMM_WORLD.Get_size()<EOL>rank = MPI.COMM_WORLD.Get_rank()<EOL><DEDENT>else:<EOL><INDENT>nworkers = <NUM_LIT:1><EOL>rank = <NUM_LIT:0><EOL><DEDENT>cpus_per_worker = <NUM_LIT:1><EOL>U.get_session(config=tf.ConfigProto(<EOL>allow_soft_placement=True,<EOL>inter_op_parallelism_threads=cpus_per_worker,<EOL>intra_op_parallelism_threads=cpus_per_worker<EOL>))<EOL>policy = build_policy(env, network, value_network='<STR_LIT>', **network_kwargs)<EOL>set_global_seeds(seed)<EOL>np.set_printoptions(precision=<NUM_LIT:3>)<EOL>ob_space = env.observation_space<EOL>ac_space = env.action_space<EOL>ob = observation_placeholder(ob_space)<EOL>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>pi = policy(observ_placeholder=ob)<EOL><DEDENT>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>oldpi = policy(observ_placeholder=ob)<EOL><DEDENT>atarg = tf.placeholder(dtype=tf.float32, shape=[None]) <EOL>ret = tf.placeholder(dtype=tf.float32, shape=[None]) <EOL>ac = pi.pdtype.sample_placeholder([None])<EOL>kloldnew = oldpi.pd.kl(pi.pd)<EOL>ent = pi.pd.entropy()<EOL>meankl = tf.reduce_mean(kloldnew)<EOL>meanent = tf.reduce_mean(ent)<EOL>entbonus = ent_coef * meanent<EOL>vferr = tf.reduce_mean(tf.square(pi.vf - ret))<EOL>ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) <EOL>surrgain = tf.reduce_mean(ratio * atarg)<EOL>optimgain = surrgain + entbonus<EOL>losses = [optimgain, meankl, entbonus, surrgain, meanent]<EOL>loss_names = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>dist = meankl<EOL>all_var_list = get_trainable_variables("<STR_LIT>")<EOL>var_list = get_pi_trainable_variables("<STR_LIT>")<EOL>vf_var_list = get_vf_trainable_variables("<STR_LIT>")<EOL>vfadam = MpiAdam(vf_var_list)<EOL>get_flat = U.GetFlat(var_list)<EOL>set_from_flat = U.SetFromFlat(var_list)<EOL>klgrads = tf.gradients(dist, var_list)<EOL>flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="<STR_LIT>")<EOL>shapes = [var.get_shape().as_list() for var in var_list]<EOL>start = <NUM_LIT:0><EOL>tangents = []<EOL>for shape in shapes:<EOL><INDENT>sz = U.intprod(shape)<EOL>tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))<EOL>start += sz<EOL><DEDENT>gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) <EOL>fvp = U.flatgrad(gvp, var_list)<EOL>assign_old_eq_new = U.function([],[], updates=[tf.assign(oldv, newv)<EOL>for (oldv, newv) in zipsame(get_variables("<STR_LIT>"), get_variables("<STR_LIT>"))])<EOL>compute_losses = U.function([ob, ac, atarg], losses)<EOL>compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])<EOL>compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)<EOL>compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))<EOL>@contextmanager<EOL>def timed(msg):<EOL><INDENT>if rank == <NUM_LIT:0>:<EOL><INDENT>print(colorize(msg, color='<STR_LIT>'))<EOL>tstart = time.time()<EOL>yield<EOL>print(colorize("<STR_LIT>"%(time.time() - tstart), color='<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>yield<EOL><DEDENT><DEDENT>def allmean(x):<EOL><INDENT>assert isinstance(x, np.ndarray)<EOL>if MPI is not None:<EOL><INDENT>out = np.empty_like(x)<EOL>MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)<EOL>out /= nworkers<EOL><DEDENT>else:<EOL><INDENT>out = np.copy(x)<EOL><DEDENT>return out<EOL><DEDENT>U.initialize()<EOL>if load_path is not None:<EOL><INDENT>pi.load(load_path)<EOL><DEDENT>th_init = get_flat()<EOL>if MPI is not None:<EOL><INDENT>MPI.COMM_WORLD.Bcast(th_init, root=<NUM_LIT:0>)<EOL><DEDENT>set_from_flat(th_init)<EOL>vfadam.sync()<EOL>print("<STR_LIT>", th_init.sum(), flush=True)<EOL>seg_gen = traj_segment_generator(pi, env, timesteps_per_batch, stochastic=True)<EOL>episodes_so_far = <NUM_LIT:0><EOL>timesteps_so_far = <NUM_LIT:0><EOL>iters_so_far = <NUM_LIT:0><EOL>tstart = time.time()<EOL>lenbuffer = deque(maxlen=<NUM_LIT>) <EOL>rewbuffer = deque(maxlen=<NUM_LIT>) <EOL>if sum([max_iters><NUM_LIT:0>, total_timesteps><NUM_LIT:0>, max_episodes><NUM_LIT:0>])==<NUM_LIT:0>:<EOL><INDENT>return pi<EOL><DEDENT>assert sum([max_iters><NUM_LIT:0>, total_timesteps><NUM_LIT:0>, max_episodes><NUM_LIT:0>]) < <NUM_LIT:2>,'<STR_LIT>'<EOL>while True:<EOL><INDENT>if callback: callback(locals(), globals())<EOL>if total_timesteps and timesteps_so_far >= total_timesteps:<EOL><INDENT>break<EOL><DEDENT>elif max_episodes and episodes_so_far >= max_episodes:<EOL><INDENT>break<EOL><DEDENT>elif max_iters and iters_so_far >= max_iters:<EOL><INDENT>break<EOL><DEDENT>logger.log("<STR_LIT>"%iters_so_far)<EOL>with timed("<STR_LIT>"):<EOL><INDENT>seg = seg_gen.__next__()<EOL><DEDENT>add_vtarg_and_adv(seg, gamma, lam)<EOL>ob, ac, atarg, tdlamret = seg["<STR_LIT>"], seg["<STR_LIT>"], seg["<STR_LIT>"], seg["<STR_LIT>"]<EOL>vpredbefore = seg["<STR_LIT>"] <EOL>atarg = (atarg - atarg.mean()) / atarg.std() <EOL>if hasattr(pi, "<STR_LIT>"): pi.ret_rms.update(tdlamret)<EOL>if hasattr(pi, "<STR_LIT>"): pi.ob_rms.update(ob) <EOL>args = seg["<STR_LIT>"], seg["<STR_LIT>"], atarg<EOL>fvpargs = [arr[::<NUM_LIT:5>] for arr in args]<EOL>def fisher_vector_product(p):<EOL><INDENT>return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p<EOL><DEDENT>assign_old_eq_new() <EOL>with timed("<STR_LIT>"):<EOL><INDENT>*lossbefore, g = compute_lossandgrad(*args)<EOL><DEDENT>lossbefore = allmean(np.array(lossbefore))<EOL>g = allmean(g)<EOL>if np.allclose(g, <NUM_LIT:0>):<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>with timed("<STR_LIT>"):<EOL><INDENT>stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank==<NUM_LIT:0>)<EOL><DEDENT>assert np.isfinite(stepdir).all()<EOL>shs = <NUM_LIT>*stepdir.dot(fisher_vector_product(stepdir))<EOL>lm = np.sqrt(shs / max_kl)<EOL>fullstep = stepdir / lm<EOL>expectedimprove = g.dot(fullstep)<EOL>surrbefore = lossbefore[<NUM_LIT:0>]<EOL>stepsize = <NUM_LIT:1.0><EOL>thbefore = get_flat()<EOL>for _ in range(<NUM_LIT:10>):<EOL><INDENT>thnew = thbefore + fullstep * stepsize<EOL>set_from_flat(thnew)<EOL>meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))<EOL>improve = surr - surrbefore<EOL>logger.log("<STR_LIT>"%(expectedimprove, improve))<EOL>if not np.isfinite(meanlosses).all():<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>elif kl > max_kl * <NUM_LIT>:<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>elif improve < <NUM_LIT:0>:<EOL><INDENT>logger.log("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logger.log("<STR_LIT>")<EOL>break<EOL><DEDENT>stepsize *= <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>logger.log("<STR_LIT>")<EOL>set_from_flat(thbefore)<EOL><DEDENT>if nworkers > <NUM_LIT:1> and iters_so_far % <NUM_LIT:20> == <NUM_LIT:0>:<EOL><INDENT>paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) <EOL>assert all(np.allclose(ps, paramsums[<NUM_LIT:0>]) for ps in paramsums[<NUM_LIT:1>:])<EOL><DEDENT><DEDENT>for (lossname, lossval) in zip(loss_names, meanlosses):<EOL><INDENT>logger.record_tabular(lossname, lossval)<EOL><DEDENT>with timed("<STR_LIT>"):<EOL><INDENT>for _ in range(vf_iters):<EOL><INDENT>for (mbob, mbret) in dataset.iterbatches((seg["<STR_LIT>"], seg["<STR_LIT>"]),<EOL>include_final_partial_batch=False, batch_size=<NUM_LIT:64>):<EOL><INDENT>g = allmean(compute_vflossandgrad(mbob, mbret))<EOL>vfadam.update(g, vf_stepsize)<EOL><DEDENT><DEDENT><DEDENT>logger.record_tabular("<STR_LIT>", explained_variance(vpredbefore, tdlamret))<EOL>lrlocal = (seg["<STR_LIT>"], seg["<STR_LIT>"]) <EOL>if MPI is not None:<EOL><INDENT>listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) <EOL><DEDENT>else:<EOL><INDENT>listoflrpairs = [lrlocal]<EOL><DEDENT>lens, rews = map(flatten_lists, zip(*listoflrpairs))<EOL>lenbuffer.extend(lens)<EOL>rewbuffer.extend(rews)<EOL>logger.record_tabular("<STR_LIT>", np.mean(lenbuffer))<EOL>logger.record_tabular("<STR_LIT>", np.mean(rewbuffer))<EOL>logger.record_tabular("<STR_LIT>", len(lens))<EOL>episodes_so_far += len(lens)<EOL>timesteps_so_far += sum(lens)<EOL>iters_so_far += <NUM_LIT:1><EOL>logger.record_tabular("<STR_LIT>", episodes_so_far)<EOL>logger.record_tabular("<STR_LIT>", timesteps_so_far)<EOL>logger.record_tabular("<STR_LIT>", time.time() - tstart)<EOL>if rank==<NUM_LIT:0>:<EOL><INDENT>logger.dump_tabular()<EOL><DEDENT><DEDENT>return pi<EOL>
learn a policy function with TRPO algorithm

Parameters:
----------

network                 neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
                        or function that takes input placeholder and returns tuple (output, None) for feedforward nets
                        or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets

env                     environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class

timesteps_per_batch     timesteps per gradient estimation batch

max_kl                  max KL divergence between old policy and new policy ( KL(pi_old || pi) )

ent_coef                coefficient of policy entropy term in the optimization objective

cg_iters                number of iterations of conjugate gradient algorithm

cg_damping              conjugate gradient damping

vf_stepsize             learning rate for adam optimizer used to optimie value function loss

vf_iters                number of iterations of value function optimization iterations per each policy optimization step

total_timesteps           max number of timesteps

max_episodes            max number of episodes

max_iters               maximum number of policy optimization iterations

callback                function to be called with (locals(), globals()) each policy optimization step

load_path               str, path to load the model from (default: None, i.e. no model is loaded)

**network_kwargs        keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network

Returns:
-------

learnt model