# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] pycharm={"is_executing": true}
# # Roboschool

# + pycharm={"is_executing": true, "name": "#%%\n"}
import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-08:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# -

os.makedirs(job_sub_dir)
os.makedirs(job_out_dir)

tasks_complement = ['RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1', 
                    'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']

# +
## 2L ss 

# + pycharm={"is_executing": true, "name": "#%%\n"}
tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size:
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --gres=gpu:{}        # request GPU "generic resource"\n'.format(GPU_NUM))
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --exp_name ddpg_n_step_2L_NoDelayTrain_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1']
n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size:
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --exp_name ddpg_n_step_2L_NoDelayTrain_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))


# ## 2L Start Step 100

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-10:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'


# +

tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1',
         'RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1',
         'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_cpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --data_dir spinup_data_ddpg_n_step_2L_ss100_Roboschool_new --exp_name ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))


# +

tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1',
         'RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1',
         'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_1L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_cpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 1 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --data_dir spinup_data_ddpg_n_step_1L_ss100_Roboschool_new --exp_name ddpg_n_step_1L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

# ### 3L ss100

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-08:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'


# +
# tasks = ['RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1',
#          'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']
tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_3L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 3 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --data_dir spinup_data_ddpg_n_step_Roboschool_3L_ss100 --exp_name ddpg_n_step_3L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -



# ## 1L Start Step 100

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-08:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# +

# tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1']
tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1',
         'RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1',
         'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 1 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --data_dir spinup_data_ddpg_n_step_Roboschool_1L_ss100 --exp_name ddpg_n_step_1L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))


# +

tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1']
# tasks = ['RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1',
#          'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 1 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --data_dir spinup_data_ddpg_n_step_Roboschool_1L_ss100 --exp_name ddpg_n_step_1L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

# ## 1L Start Step 10000

# +

# tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1']
tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1',
         'RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1',
         'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 1 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --data_dir spinup_data_ddpg_n_step_1L_ss10000 --exp_name ddpg_n_step_1L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

# ## 2L Start Step 10000

# +

# tasks = ['RoboschoolAnt-v1', 'RoboschoolHalfCheetah-v1', 'RoboschoolWalker2d-v1', 'RoboschoolHopper-v1']
tasks = ['RoboschoolInvertedPendulum-v1', 'RoboschoolInvertedPendulumSwingup-v1',
         'RoboschoolInvertedDoublePendulum-v1', 'RoboschoolReacher-v1', 'RoboschoolPong-v1']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --data_dir spinup_data_ddpg_n_step_2L_ss10000_complement --exp_name ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

import os
job_sub_dir = './job_scripts'
jobs = os.listdir('./job_scripts')
jobs.sort()
i=1
for job in jobs:
    code = os.system('sbatch {}'.format(os.path.join(job_sub_dir, job)))
    print('{} ---- {}: {}'.format(i, job, code))
    i += 1


# # PyBulletGym

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-06:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# ## Start Step 100

# +

tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0',
         'ReacherPyBulletEnv-v0', 'PusherPyBulletEnv-v0', 'ThrowerPyBulletEnv-v0', 'StrikerPyBulletEnv-v0']
n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --exp_name ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))


# +
tasks = ['InvertedPendulumPyBulletEnv-v0', 'InvertedPendulumSwingupPyBulletEnv-v0', 
         'InvertedDoublePendulumPyBulletEnv-v0', 'ReacherPyBulletEnv-v0']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_1L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_cpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 1 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --data_dir spinup_data_PyBulletEnv_1L_ss100_complement --exp_name ddpg_n_step_1L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

# ## 2L Start Step 10000



import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-06:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# +
tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0',
         'ReacherPyBulletEnv-v0', 'PusherPyBulletEnv-v0', 'ThrowerPyBulletEnv-v0', 'StrikerPyBulletEnv-v0']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --exp_name ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

# ## 2L Start Step 100

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-10:00'
JOB_MEMORY = '12000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# +
tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0',
         'InvertedPendulumPyBulletEnv-v0', 'InvertedPendulumSwingupPyBulletEnv-v0', 
         'InvertedDoublePendulumPyBulletEnv-v0', 'ReacherPyBulletEnv-v0']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_cpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --data_dir spinup_data_ddpg_n_step_2L_ss100_PyBulletEnv_new --exp_name ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -



# # Random Action
# ## Start Step 10000 

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-06:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# +
tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0',
         'ReacherPyBulletEnv-v0', 'PusherPyBulletEnv-v0', 'ThrowerPyBulletEnv-v0', 'StrikerPyBulletEnv-v0']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --random_action_baseline --exp_name ddpg_n_step_2L_NoDelayTrain_ss10000_random_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

# ## Start Step 100 1L

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-06:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# +
tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0',
         'ReacherPyBulletEnv-v0', 'PusherPyBulletEnv-v0', 'ThrowerPyBulletEnv-v0', 'StrikerPyBulletEnv-v0']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 1 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --exp_name ddpg_n_step_1L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

import os
job_sub_dir = './job_scripts'
jobs = os.listdir('./job_scripts')
jobs.sort()
i=1
for job in jobs:
    code = os.system('sbatch {}'.format(os.path.join(job_sub_dir, job)))
    print('{} ---- {}: {}'.format(i, job, code))
    i += 1


# ## Start Step 10000 1L

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-06:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# +
tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0',
         'ReacherPyBulletEnv-v0', 'PusherPyBulletEnv-v0', 'ThrowerPyBulletEnv-v0', 'StrikerPyBulletEnv-v0']

n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --seed {1} --l 1 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --exp_name ddpg_n_step_1L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

import os
job_sub_dir = './job_scripts'
jobs = os.listdir('./job_scripts')
jobs.sort()
i=1
for job in jobs:
    code = os.system('sbatch {}'.format(os.path.join(job_sub_dir, job)))
    print('{} ---- {}: {}'.format(i, job, code))
    i += 1


# ## Nonstationary Env Start Step 100

# +

tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0']
n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --nonstationary_env --gravity_cycle 1000 --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 100 --exp_name ddpg_n_step_Nonstationary1000_2L_NoDelayTrain_ss100_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))

# -

# ### Nonstationary Env Start Step 10000

import numpy as np
import os
CPU_NUM = 2
JOB_TIME = '0-06:00'
JOB_MEMORY = '8000M'
job_sub_dir = './job_scripts'
job_out_dir = './job_scripts_output'

# +

tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0']
n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --nonstationary_env --gravity_change_pattern gravity_averagely_harder --gravity_cycle 1000 --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --exp_name ddpg_n_step_NonstationaryHarder1000_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))


# +


tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0']
n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --nonstationary_env --gravity_change_pattern gravity_averagely_equal --gravity_cycle 1000 --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --exp_name ddpg_n_step_NonstationaryEqual1000_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))


# +


tasks = ['AntPyBulletEnv-v0', 'HalfCheetahPyBulletEnv-v0', 'Walker2DPyBulletEnv-v0', 'HopperPyBulletEnv-v0']
n_steps = ['1', '2', '3', '4', '5']
replay_size = ['1000000', '500000', '100000', '50000', '10000']
seeds = ['0', '1', '2', '3']
for s in seeds:
    for task in tasks:
        for n_s in n_steps:
            for b_s in replay_size: 
                job_filename = 'job_{0}_{1}_{2}_{3}.sh'.format(task, s, n_s, b_s)
                print(job_filename)
                with open(os.path.join(job_sub_dir, job_filename), 'w') as job_file:
                    job_file.write('#!/bin/bash\n')
                    job_file.write('#SBATCH --account=def-dkulic\n')
                    job_file.write('#SBATCH --cpus-per-task={}    #Maximum of CPU cores per GPU request: 6 on Cedar, 16 on Graham.\n'.format(CPU_NUM))
                    job_file.write('#SBATCH --mem={}               # memory per node\n'.format(JOB_MEMORY))
                    job_file.write('#SBATCH --time={}            # time (DD-HH:MM)\n'.format(JOB_TIME))
                    job_file.write('#SBATCH --output=./job_scripts_output/ddpg_n_step_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}_%N-%j.out        # %N for node name, %j for jobID\n'.format(task, s, n_s, b_s))
                    job_file.write('## Main processing command\n')
                    job_file.write('module load cuda cudnn \n')
                    job_file.write('source ~/tf_gpu/bin/activate\n')
                    job_file.write('python ./ddpg_n_step.py  --env {0} --nonstationary_env --gravity_change_pattern gravity_averagely_easier --gravity_cycle 1000 --seed {1} --l 2 --n_step {2} --replay_size {3} --without_delay_train --start_steps 10000 --exp_name ddpg_n_step_NonstationaryEasier1000_2L_NoDelayTrain_ss10000_{0}_{1}_{2}_{3}'.format(task, s, n_s, b_s))


# + pycharm={"is_executing": true, "name": "#%%\n"}
import os
job_sub_dir = './job_scripts'
jobs = os.listdir('./job_scripts')
jobs.sort()
i=1
for job in jobs:
    code = os.system('sbatch {}'.format(os.path.join(job_sub_dir, job)))
    print('{} ---- {}: {}'.format(i, job, code))
    i += 1


# + pycharm={"is_executing": true, "name": "#%%\n"}




# + pycharm={"name": "#%%\n"}



StarcoderdataJupyterScriptsDedupFiltered