# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import gym
import sys
import multiprocessing
import numpy as np
import tensorflow as tf
from baselines.ppo2 import ppo2
from baselines.ppo2.policies import CnnPolicy
from baselines.common.vec_env.vec_frame_stack import VecFrameStack
from baselines.common.cmd_util import make_atari_env

load_path = '/tmp/openai-2018-05-29-13-13-12-228152/checkpoints/00950'

env = VecFrameStack(make_atari_env('BreakoutNoFrameskip-v4',1,0),4)

obs = env.reset()
obs.shape
env.observation_space

ncpu = multiprocessing.cpu_count()
if sys.platform == 'darwin': ncpu //= 2
config = tf.ConfigProto(allow_soft_placement=True,
                        intra_op_parallelism_threads=ncpu,
                        inter_op_parallelism_threads=ncpu)
config.gpu_options.allow_growth = True #pylint: disable=E1101
tf.Session(config=config).__enter__()

policy = CnnPolicy
ob_space = env.observation_space
ac_space = env.action_space
nbatch_act = 1 * 1
nbatch_train = 32 * 1
nsteps = 128
ent_coef = 0.1
vf_coef = 0.5
max_grad_norm = 0.5
make_model = lambda : ppo2.Model(policy=policy, ob_space=ob_space, ac_space=ac_space, 
                            nbatch_act=nbatch_act, nbatch_train=nbatch_train, nsteps=nsteps, 
                            ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm)
model = make_model()

model.load(load_path)

actions, _, _, _, = model.step(obs)

actions

env.step(actions)

env.venv.render()

obs = env.reset()
while True:
    actions, _, _, _, = model.step(obs)
    obs[:], _, _, _, = env.step(actions)
    env.venv.render()
    

from baselines.ppo2.policies import nature_cnn
from baselines.a2c.utils import conv, fc

obs.shape

ob_shape=(128,84,84,4)

X = tf.placeholder(tf.uint8, ob_shape)
h = nature_cnn(X)
v_int = fc(h,'v',1)
v_int

v_int[:,0]

pdparam = fc(h, 'pi', 4, init_scale=1.0, init_bias=0.0)
pdparam

u = tf.random_uniform(tf.shape(pdparam))
u

h1 = tf.reshape(h, [2,128,-1])
h1

xs = [tf.squeeze(v, [1]) for v in tf.split(axis=1, num_or_size_splits=128, value=h1)]
xs

M = tf.placeholder(tf.float32, [256])
M

ms = [tf.squeeze(v, [1]) for v in tf.split(axis=1, num_or_size_splits=128, 
                                           value=tf.reshape(M, [2,128,-1]))]
ms

S = tf.placeholder(tf.float32, [2, 512])
S

c, h = tf.split(axis=1, num_or_size_splits=2, value=S)
h

xs[0].get_shape()

m = ms[0]
x = xs[0]
(1-m)

wx = tf.get_variable('wx', [256, 1024])
wx

tf.matmul(x, wx) + tf.matmul(x, wx)



StarcoderdataJupyterScriptsDedupFiltered