code: import copy
  import torch.nn as nn
  from rlkit.launchers.launcher_util import setup_logger
  import rlkit.torch.pytorch_util as ptu
  from rlkit.core.ma_eval_util import get_generic_ma_path_information
  
  def experiment(variant):
      num_agent = variant['num_agent']
      from differential_game import DifferentialGame
      expl_env = DifferentialGame(game_name=args.exp_name)
      eval_env = DifferentialGame(game_name=args.exp_name)
      obs_dim = eval_env.observation_space.low.size
      action_dim = eval_env.action_space.low.size
  
      from rlkit.torch.networks.graph_builders import FullGraphBuilder
      graph_builder1 = FullGraphBuilder(
                  input_node_dim=obs_dim+action_dim,
                  num_node=num_agent,
                  contain_self_loop=False)
      from rlkit.torch.networks.gnn_networks import GNNNet
      gnn1 = GNNNet(
              graph_builder1,
              node_dim=variant['qf_kwargs']['hidden_dim'],
              conv_type=variant['qf_kwargs']['conv_type'],
              num_conv_layers=1,
              hidden_activation='relu',
              output_activation='relu',
          )
      qf1 = nn.Sequential(
              gnn1,
              nn.Linear(variant['qf_kwargs']['hidden_dim'],1)
          )
      target_qf1 = copy.deepcopy(qf1)
  
      from rlkit.torch.networks.graph_builders import FullGraphBuilder
      graph_builder2 = FullGraphBuilder(
                  input_node_dim=obs_dim+action_dim,
                  num_node=num_agent,
                  contain_self_loop=False)
      from rlkit.torch.networks.gnn_networks import GNNNet
      gnn2 = GNNNet(
              graph_builder2,
              node_dim=variant['qf_kwargs']['hidden_dim'],
              conv_type=variant['qf_kwargs']['conv_type'],
              num_conv_layers=1,
              hidden_activation='relu',
              output_activation='relu',
          )
      qf2 = nn.Sequential(
              gnn2,
              nn.Linear(variant['qf_kwargs']['hidden_dim'],1)
          )
      target_qf2 = copy.deepcopy(qf2)
  
      policy_n, eval_policy_n, expl_policy_n = [], [], []
      for i in range(num_agent):
          from rlkit.torch.networks.layers import SplitLayer
          policy = nn.Sequential(
              nn.Linear(obs_dim,variant['policy_kwargs']['hidden_dim']),
              nn.ReLU(),
              nn.Linear(variant['policy_kwargs']['hidden_dim'],variant['policy_kwargs']['hidden_dim']),
              nn.ReLU(),
              SplitLayer(layers=[nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim),
                                  nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim)])
              )
          from rlkit.torch.policies.tanh_gaussian_policy import TanhGaussianPolicy
          policy = TanhGaussianPolicy(module=policy)
          from rlkit.torch.policies.make_deterministic import MakeDeterministic
          eval_policy = MakeDeterministic(policy)
          from rlkit.exploration_strategies.base import PolicyWrappedWithExplorationStrategy
          if variant['random_exploration']:
              from rlkit.exploration_strategies.epsilon_greedy import EpsilonGreedy
              expl_policy = PolicyWrappedWithExplorationStrategy(
                  exploration_strategy=EpsilonGreedy(expl_env.action_space, prob_random_action=1.0),
                  policy=policy,
              )
          else:
              expl_policy = policy
  
          policy_n.append(policy)
          eval_policy_n.append(eval_policy)
          expl_policy_n.append(expl_policy)
  
      from rlkit.samplers.data_collector.ma_path_collector import MAMdpPathCollector
      eval_path_collector = MAMdpPathCollector(eval_env, eval_policy_n)
      expl_path_collector = MAMdpPathCollector(expl_env, expl_policy_n)
  
      from rlkit.data_management.ma_env_replay_buffer import MAEnvReplayBuffer
      replay_buffer = MAEnvReplayBuffer(variant['replay_buffer_size'], expl_env, num_agent=num_agent)
  
      from rlkit.torch.masac.masac_gnn import MASACGNNTrainer
      trainer = MASACGNNTrainer(
          env = expl_env,
          qf1=qf1,
          target_qf1=target_qf1,
          qf2=qf2,
          target_qf2=target_qf2,
          policy_n=policy_n,
          **variant['trainer_kwargs']
      )
  
      from rlkit.torch.torch_rl_algorithm import TorchBatchRLAlgorithm
      algorithm = TorchBatchRLAlgorithm(
          trainer=trainer,
          exploration_env=expl_env,
          evaluation_env=eval_env,
          exploration_data_collector=expl_path_collector,
          evaluation_data_collector=eval_path_collector,
          replay_buffer=replay_buffer,
          log_path_function=get_generic_ma_path_information,
          **variant['algorithm_kwargs']
      )
      algorithm.to(ptu.device)
      algorithm.train()
  
  
  if __name__ == "__main__":
      import argparse
      parser = argparse.ArgumentParser()
      parser.add_argument('--exp_name', type=str, default='zero_sum')
      parser.add_argument('--log_dir', type=str, default='MASACGNNGaussian')
      parser.add_argument('--conv', type=str, default='GSage')
      parser.add_argument('--hidden', type=int, default=16)
      parser.add_argument('--oa', action='store_true', default=False) # online action
      parser.add_argument('--snl', action='store_true', default=False) # sum n loss
      parser.add_argument('--re', action='store_true', default=False) # random exploration
      parser.add_argument('--alpha', type=float, default=None) # init alpha
      parser.add_argument('--fa', action='store_true', default=False) # fix alpha
      parser.add_argument('--lr', type=float, default=None)
      parser.add_argument('--bs', type=int, default=None)
      parser.add_argument('--epoch', type=int, default=None)
      parser.add_argument('--seed', type=int, default=0)
      parser.add_argument('--snapshot_mode', type=str, default="gap_and_last")
      parser.add_argument('--snapshot_gap', type=int, default=500)
      args = parser.parse_args()
      import os.path as osp
      pre_dir = './Data/'+args.exp_name
      main_dir = args.log_dir\
                  +args.conv\
                  +('hidden'+str(args.hidden))\
                  +('oa' if args.oa else '')\
                  +('snl' if args.snl else '')\
                  +('re' if args.re else '')\
                  +(('alpha'+str(args.alpha)) if args.alpha else '')\
                  +('fa' if args.fa else '')\
                  +(('lr'+str(args.lr)) if args.lr else '')\
                  +(('bs'+str(args.bs)) if args.bs else '')
      log_dir = osp.join(pre_dir,main_dir,'seed'+str(args.seed))
      # noinspection PyTypeChecker
      variant = dict(
          num_agent=2,
          random_exploration=args.re,
          algorithm_kwargs=dict(
              num_epochs=(args.epoch if args.epoch else 100),
              num_eval_steps_per_epoch=100,
              num_trains_per_train_loop=100,
              num_expl_steps_per_train_loop=100,
              min_num_steps_before_training=100,
              max_path_length=100,
              batch_size=(args.bs if args.bs else 256),
          ),
          trainer_kwargs=dict(
              use_soft_update=True,
              tau=1e-2,
              discount=0.99,
              qf_learning_rate=(args.lr if args.lr else 1e-3),
              policy_learning_rate=(args.lr if args.lr else 1e-4),
              online_action=args.oa,
              sum_n_loss=args.snl,
              init_alpha=(args.alpha if args.alpha else 1.),
              use_automatic_entropy_tuning=(not args.fa),
          ),
          qf_kwargs=dict(
              conv_type=args.conv,
              hidden_dim=args.hidden,
          ),
          policy_kwargs=dict(
              hidden_dim=args.hidden,
          ),
          replay_buffer_size=int(1E6),
      )
      import os
      if not os.path.isdir(log_dir):
          os.makedirs(log_dir)
      with open(osp.join(log_dir,'variant.json'),'w') as out_json:
          import json
          json.dump(variant,out_json,indent=2)
      import sys
      cmd_input = 'python ' + ' '.join(sys.argv) + '\n'
      with open(osp.join(log_dir, 'cmd_input.txt'), 'a') as f:
          f.write(cmd_input)
      setup_logger(args.exp_name+'/'+main_dir, variant=variant,
                  snapshot_mode=args.snapshot_mode, snapshot_gap=args.snapshot_gap,
                  log_dir=log_dir)
      import numpy as np
      import torch
      np.random.seed(args.seed)
      torch.manual_seed(args.seed)
      # ptu.set_gpu_mode(True)  # optionally set the GPU (default=False)
      experiment(variant)
   import torch
  import numpy as np
  import time
  import pdb
  from rlkit.torch.core import eval_np, np_ify
  
  import argparse
  parser = argparse.ArgumentParser()
  parser.add_argument('--exp_name', type=str, default='SimpleSupLSTM')
  parser.add_argument('--extra_name', type=str, default='obs1int10')
  parser.add_argument('--log_dir', type=str, default='SupLSTMlayer1hidden16')
  parser.add_argument('--file', type=str, default='params')
  parser.add_argument('--epoch', type=int, default=None)
  parser.add_argument('--seed', type=int, default=0)
  args = parser.parse_args()
  
  pre_dir = './Data/'+args.exp_name+args.extra_name
  import os
  data_path = '{}/{}/seed{}/{}.pkl'.format(pre_dir,args.log_dir,args.seed,args.file)
  data = torch.load(data_path,map_location='cpu')
  
  policy = data['trainer/policy']
  # from rlkit.torch.policies.make_deterministic import MakeDeterministic
  # policy = MakeDeterministic(policy)
  
  if 'trainer/sup_learner' in data.keys():
  	sup_learner = data['trainer/sup_learner']
  else:
  	sup_learner = None
  
  import sys
  import json
  with open('{}/{}/seed{}/variant.json'.format(pre_dir,args.log_dir,args.seed)) as f:
    variant = json.load(f)
  from simple_sup_lstm import SimpleSupLSTMEnv
  env = SimpleSupLSTMEnv(**variant['env_kwargs'])
  o = env.reset()
  policy.reset()
  
  max_path_length = 10
  path_length = 0
  done = False
  c_r = 0.
  while True:
  	path_length += 1
  	a, agent_info = policy.get_action(o)
  	o, r, done, env_info = env.step(a)
  
  	if sup_learner:
  		intentions = eval_np(sup_learner, o[None,:])
  	elif hasattr(policy, 'sup_prob'):
  		intentions = eval_np(policy.sup_prob, o[None,:])[0]
  	else:
  		intentions = None
  
  	c_r += r
  	print("step: ",path_length)
  	print("intentions: ",intentions)
  	print("a: ",a)
  	print("env_info: ",env_info)
  	print('r: ',r)
  	print(done)
  	# pdb.set_trace()
  	time.sleep(0.1)
  	if path_length > max_path_length or done:
  		print('c_r: ',c_r)
  		path_length = 0
  		done = False
  		c_r = 0.
  		pdb.set_trace()
  		o = env.reset()
  		policy.reset() import copy
  import torch.nn as nn
  from rlkit.launchers.launcher_util import setup_logger
  import rlkit.torch.pytorch_util as ptu
  from rlkit.core.ma_eval_util import get_generic_ma_path_information
  
  def experiment(variant):
      from multi_differential_game import MultiDifferentialGame
      expl_env = MultiDifferentialGame(**variant['env_kwargs'])
      eval_env = MultiDifferentialGame(**variant['env_kwargs'])
      num_agent = expl_env.agent_num
      obs_dim = eval_env.observation_space.low.size
      action_dim = eval_env.action_space.low.size
  
      from rlkit.torch.networks.graph_builders import FullGraphBuilder
      graph_builder_obs = FullGraphBuilder(
                  input_node_dim=obs_dim,
                  num_node=num_agent,
                  batch_size=variant['algorithm_kwargs']['batch_size'],
                  contain_self_loop=False)
      graph_builder_eval = FullGraphBuilder(
                  input_node_dim=graph_builder_obs.output_node_dim,
                  num_node=num_agent,
                  batch_size=variant['algorithm_kwargs']['batch_size'],
                  contain_self_loop=False)
      if variant['concat_emb']:
          gnn_out_dim = int(obs_dim + variant['graph_kwargs']['node_dim']*variant['graph_kwargs']['num_conv_layers'])
      else:
          gnn_out_dim = variant['graph_kwargs']['node_dim']
      from rlkit.torch.networks.networks import FlattenMlp
      post_mlp1 = FlattenMlp(input_size=gnn_out_dim,
                      output_size=1,
                      hidden_sizes=[variant['qf_kwargs']['hidden_dim']]*(variant['qf_kwargs']['num_layer']-1),
                      hidden_activation=nn.LeakyReLU(negative_slope=0.2),
                      )
      from rlkit.torch.networks.graph_r2g_qnet import R2GQNet
      qf1 = R2GQNet(
              obs_graph_builder=graph_builder_obs,
              eval_graph_builder=graph_builder_eval,
              obs_dim=obs_dim,
              action_dim=action_dim, 
              post_mlp=post_mlp1,
              normalize_emb=False,
              output_activation=None,
              concat_emb=variant['concat_emb'],
              **variant['graph_kwargs'],
          )
      target_qf1 = copy.deepcopy(qf1)
  
      post_mlp2 = FlattenMlp(input_size=gnn_out_dim,
                      output_size=1,
                      hidden_sizes=[variant['qf_kwargs']['hidden_dim']]*(variant['qf_kwargs']['num_layer']-1),
                      hidden_activation=nn.LeakyReLU(negative_slope=0.2),
                      )
      from rlkit.torch.networks.graph_r2g_qnet import R2GQNet
      qf2 = R2GQNet(
              obs_graph_builder=graph_builder_obs,
              eval_graph_builder=graph_builder_eval,
              obs_dim=obs_dim,
              action_dim=action_dim, 
              post_mlp=post_mlp2,
              normalize_emb=False,
              output_activation=None,
              concat_emb=variant['concat_emb'],
              **variant['graph_kwargs'],
          )
      target_qf2 = copy.deepcopy(qf2)
  
      graph_builder_ca = FullGraphBuilder(
                  input_node_dim=obs_dim+action_dim,
                  num_node=num_agent,
                  batch_size=variant['algorithm_kwargs']['batch_size'],
                  contain_self_loop=False)
      from rlkit.torch.networks.gnn_networks import GNNNet
      cgca = GNNNet(
              graph_builder_ca,
              hidden_activation='lrelu0.2',
              output_activation='lrelu0.2',
              **variant['graph_kwargs'],
          )
      from rlkit.torch.networks.networks import FlattenMlp
      from rlkit.torch.networks.layers import SplitLayer
      from rlkit.torch.policies.tanh_gaussian_policy import TanhGaussianPolicy
      cactor = nn.Sequential(
          cgca,
          FlattenMlp(input_size=variant['graph_kwargs']['node_dim'],
                      output_size=variant['cactor_kwargs']['hidden_dim'],
                      hidden_sizes=[variant['cactor_kwargs']['hidden_dim']]*(variant['cactor_kwargs']['num_layer']-1),
                      hidden_activation=nn.LeakyReLU(negative_slope=0.2),
                      output_activation=nn.LeakyReLU(negative_slope=0.2),
                      ),
          nn.LeakyReLU(negative_slope=0.2),
          SplitLayer(layers=[nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim),
                              nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim)])
          )
      cactor = TanhGaussianPolicy(module=cactor)
  
      graph_builder_policy = FullGraphBuilder(
                  input_node_dim=obs_dim,
                  num_node=num_agent,
                  batch_size=variant['algorithm_kwargs']['batch_size'],
                  contain_self_loop=False)
      shared_gnn = GNNNet(
              graph_builder_policy,
              hidden_activation='lrelu0.2',
              output_activation='lrelu0.2',
              **variant['graph_kwargs'],
          )
      policy_n, expl_policy_n, eval_policy_n = [], [], []
      for i in range(num_agent):
          policy = nn.Sequential(
              FlattenMlp(input_size=variant['graph_kwargs']['node_dim'],
                          output_size=variant['policy_kwargs']['hidden_dim'],
                          hidden_sizes=[variant['policy_kwargs']['hidden_dim']]*(variant['policy_kwargs']['num_layer']-1),
                          hidden_activation=nn.LeakyReLU(negative_slope=0.2),
                          output_activation=nn.LeakyReLU(negative_slope=0.2),
                          ),
              SplitLayer(layers=[nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim),
                                  nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim)])
              )
          policy = TanhGaussianPolicy(module=policy)
          from rlkit.torch.policies.make_deterministic import MakeDeterministic
          eval_policy = MakeDeterministic(policy)
          if variant['random_exploration']:
              from rlkit.exploration_strategies.base import PolicyWrappedWithExplorationStrategy
              from rlkit.exploration_strategies.epsilon_greedy import EpsilonGreedy
              expl_policy = PolicyWrappedWithExplorationStrategy(
                                      exploration_strategy=EpsilonGreedy(expl_env.action_space, prob_random_action=1.0),
                                      policy=policy,
                                  )
          else:
              expl_policy = policy
          
          policy_n.append(policy)
          expl_policy_n.append(expl_policy)
          eval_policy_n.append(eval_policy)
          
      from rlkit.samplers.data_collector.ma_path_collector import MAMdpPathCollector
      eval_path_collector = MAMdpPathCollector(eval_env, eval_policy_n, shared_encoder=shared_gnn)
      expl_path_collector = MAMdpPathCollector(expl_env, expl_policy_n, shared_encoder=shared_gnn)
  
      from rlkit.data_management.ma_env_replay_buffer import MAEnvReplayBuffer
      replay_buffer = MAEnvReplayBuffer(variant['replay_buffer_size'], expl_env, num_agent=num_agent)
  
      from rlkit.torch.r2g.r2g_gnn11 import R2GGNNTrainer
      trainer = R2GGNNTrainer(
          env=expl_env,
          qf1=qf1,
          target_qf1=target_qf1,
          qf2=qf2,
          target_qf2=target_qf2,
          cactor=cactor,
          policy_n=policy_n,
          shared_gnn=shared_gnn,
          **variant['trainer_kwargs']
      )
  
      from rlkit.torch.torch_rl_algorithm import TorchBatchRLAlgorithm
      algorithm = TorchBatchRLAlgorithm(
          trainer=trainer,
          exploration_env=expl_env,
          evaluation_env=eval_env,
          exploration_data_collector=expl_path_collector,
          evaluation_data_collector=eval_path_collector,
          replay_buffer=replay_buffer,
          log_path_function=get_generic_ma_path_information,
          **variant['algorithm_kwargs']
      )
      algorithm.to(ptu.device)
      # save init params
      from rlkit.core import logger
      snapshot = algorithm._get_snapshot()
      file_name = osp.join(logger._snapshot_dir, 'itr_-1.pkl')
      torch.save(snapshot, file_name)
  
      algorithm.train()
  
  
  if __name__ == "__main__":
      import argparse
      parser = argparse.ArgumentParser()
      parser.add_argument('--exp_name', type=str, default='zero_sum')
      parser.add_argument('--num_ag', type=int, default=2)
      parser.add_argument('--gpu', type=int, default=None)
      parser.add_argument('--log_dir', type=str, default='R2GGNN11ShareGaussian')
      parser.add_argument('--conv', type=str, default='GSage')
      parser.add_argument('--layer', type=int, default=2)
      parser.add_argument('--hidden', type=int, default=16)
      parser.add_argument('--glayer', type=int, default=2)
      parser.add_argument('--hnode', type=int, default=16)
      parser.add_argument('--sc', action='store_true', default=False) # skip connection between gnn layers
      parser.add_argument('--ceb', action='store_true', default=False) # concat gnn embeddings from each layer
      parser.add_argument('--ce', action='store_true', default=False) # cactor entropy
      parser.add_argument('--er', action='store_true', default=False) # entropy reward
      parser.add_argument('--re', action='store_true', default=False) # random exploration
      parser.add_argument('--alpha', type=float, default=None) # init alpha
      parser.add_argument('--fa', action='store_true', default=False) # fix alpha
      parser.add_argument('--dcig', action='store_true', default=False) # deterministic cactor in graph
      parser.add_argument('--dna', action='store_true', default=False) # deterministic next action
      parser.add_argument('--lr', type=float, default=None)
      parser.add_argument('--bs', type=int, default=None)
      parser.add_argument('--epoch', type=int, default=None)
      parser.add_argument('--seed', type=int, default=0)
      parser.add_argument('--snapshot_mode', type=str, default="gap_and_last")
      parser.add_argument('--snapshot_gap', type=int, default=100)
      args = parser.parse_args()
      import os.path as osp
      pre_dir = './Data/'+args.exp_name+'_p'+str(args.num_ag)
      main_dir = args.log_dir\
                  +args.conv\
                  +('layer'+str(args.layer))\
                  +('hidden'+str(args.hidden))\
                  +('glayer'+str(args.glayer))\
                  +('hnode'+str(args.hnode))\
                  +('sc' if args.sc else '')\
                  +('ceb' if args.ceb else '')\
                  +('ce' if args.ce else '')\
                  +('er' if args.er else '')\
                  +('re' if args.re else '')\
                  +(('alpha'+str(args.alpha)) if args.alpha else '')\
                  +('fa' if args.fa else '')\
                  +('dcig' if args.dcig else '')\
                  +('dna' if args.dna else '')\
                  +(('lr'+str(args.lr)) if args.lr else '')\
                  +(('bs'+str(args.bs)) if args.bs else '')
      log_dir = osp.join(pre_dir,main_dir,'seed'+str(args.seed))
      # noinspection PyTypeChecker
      variant = dict(
          random_exploration=args.re,
          env_kwargs=dict(
              game_name=args.exp_name,
              agent_num=args.num_ag,
          ),
          algorithm_kwargs=dict(
              num_epochs=(args.epoch+1 if args.epoch else 101),
              num_eval_steps_per_epoch=100,
              num_trains_per_train_loop=100*args.num_ag,
              num_expl_steps_per_train_loop=100*args.num_ag,
              min_num_steps_before_training=100*args.num_ag,
              max_path_length=100,
              batch_size=(args.bs if args.bs else 256),
          ),
          trainer_kwargs=dict(
              use_soft_update=True,
              tau=1e-2,
              discount=0.99,
              qf_learning_rate=(args.lr if args.lr else 1e-3),
              cactor_learning_rate=(args.lr if args.lr else 1e-4),
              policy_learning_rate=(args.lr if args.lr else 1e-4),
              use_entropy_loss=True,
              use_entropy_reward=args.er,
              use_cactor_entropy_loss=args.ce,
              init_alpha=(args.alpha if args.alpha else 1.),
              use_automatic_entropy_tuning=(not args.fa),
              deterministic_cactor_in_graph=args.dcig,
              deterministic_next_action=args.dna,
          ),
          graph_kwargs=dict(
              conv_type=args.conv,
              node_dim=args.hnode,
              num_conv_layers=args.glayer,
              skip_connect=args.sc,
          ),
          concat_emb=args.ceb,
          qf_kwargs=dict(
              hidden_dim=args.hidden,
              num_layer=args.layer,
          ),
          cactor_kwargs=dict(
              hidden_dim=args.hidden,
              num_layer=args.layer,
          ),
          policy_kwargs=dict(
              hidden_dim=args.hidden,
              num_layer=args.layer,
          ),
          replay_buffer_size=int(1E6),
      )
      import os
      if not os.path.isdir(log_dir):
          os.makedirs(log_dir)
      with open(osp.join(log_dir,'variant.json'),'w') as out_json:
          import json
          json.dump(variant,out_json,indent=2)
      import sys
      cmd_input = 'python ' + ' '.join(sys.argv) + '\n'
      with open(osp.join(log_dir, 'cmd_input.txt'), 'a') as f:
          f.write(cmd_input)
      setup_logger(args.exp_name+'/'+main_dir, variant=variant,
                  snapshot_mode=args.snapshot_mode, snapshot_gap=args.snapshot_gap,
                  log_dir=log_dir)
      import numpy as np
      import torch
      np.random.seed(args.seed)
      torch.manual_seed(args.seed)
      if isinstance(args.gpu, int):
          print('using gpu ',args.gpu)
          ptu.set_gpu_mode(True, gpu_id=args.gpu)
      experiment(variant)
   import copy
  import torch.nn as nn
  from rlkit.launchers.launcher_util import setup_logger
  import rlkit.torch.pytorch_util as ptu
  from rlkit.core.ma_eval_util import get_generic_ma_path_information
  
  def experiment(variant):
      import sys
      sys.path.append("./multiagent-particle-envs")
      from make_env import make_env
      from particle_env_wrapper import ParticleEnv
      expl_env = ParticleEnv(make_env(args.exp_name,discrete_action_space=False,world_args=variant['world_args']))
      eval_env = ParticleEnv(make_env(args.exp_name,discrete_action_space=False,world_args=variant['world_args']))
      num_agent = expl_env.num_agent
      obs_dim = eval_env.observation_space.low.size
      action_dim = eval_env.action_space.low.size
  
      policy_n, qf_n = [], []
      policy_optimizer_n, qf_optimizer_n = None, None
      for i in range(num_agent):
          from rlkit.torch.networks.networks import FlattenMlp
          from rlkit.torch.networks.layers import SplitLayer
          policy = nn.Sequential(
              FlattenMlp(input_size=obs_dim,
                          output_size=variant['policy_kwargs']['hidden_dim'],
                          hidden_sizes=[variant['policy_kwargs']['hidden_dim']]*(variant['policy_kwargs']['num_layer']-1),
                          ),
              SplitLayer(layers=[nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim),
                                  nn.Linear(variant['policy_kwargs']['hidden_dim'],action_dim)])
              )
          from rlkit.torch.policies.tanh_gaussian_policy import TanhGaussianPolicy
          policy = TanhGaussianPolicy(module=policy,return_raw_action=True)
          
          qf = FlattenMlp(
              input_size=(obs_dim*num_agent+action_dim*num_agent),
              output_size=1,
              hidden_sizes=[variant['qf_kwargs']['hidden_dim']]*variant['qf_kwargs']['num_layer'],
          )
  
          policy_n.append(policy)
          qf_n.append(qf)
  
      from rlkit.torch.policies.make_deterministic import MakeDeterministic
      eval_policy_n = [MakeDeterministic(policy) for policy in policy_n]
      expl_policy_n = policy_n
  
      from rlkit.samplers.data_collector.ma_path_collector import MAMdpPathCollector
      eval_path_collector = MAMdpPathCollector(eval_env, eval_policy_n)
      expl_path_collector = MAMdpPathCollector(expl_env, expl_policy_n, collect_raw_actions=True)
  
      from rlkit.torch.coma.coma import COMATrainer
      trainer = COMATrainer(
          env = expl_env,
          policy_n=policy_n,
          qf_n=qf_n,
          policy_optimizer_n=policy_optimizer_n,
          qf_optimizer_n=qf_optimizer_n,
          **variant['trainer_kwargs']
      )
  
      from rlkit.torch.torch_rl_algorithm import TorchOnlineRLAlgorithm
      algorithm = TorchOnlineRLAlgorithm(
          trainer=trainer,
          exploration_env=expl_env,
          evaluation_env=eval_env,
          exploration_data_collector=expl_path_collector,
          evaluation_data_collector=eval_path_collector,
          log_path_function=get_generic_ma_path_information,
          **variant['algorithm_kwargs']
      )
      algorithm.to(ptu.device)
      algorithm.train()
  
  
  if __name__ == "__main__":
      import argparse
      parser = argparse.ArgumentParser()
      parser.add_argument('--exp_name', type=str, default='simple')
      parser.add_argument('--boundary', action='store_true', default=False)
      parser.add_argument('--num_ag', type=int, default=None)
      parser.add_argument('--num_adv', type=int, default=None)
      parser.add_argument('--num_l', type=int, default=None)
      parser.add_argument('--mpl', type=int, default=25) # max path length
      parser.add_argument('--log_dir', type=str, default='COMAGaussian')
      parser.add_argument('--layer', type=int, default=2)
      parser.add_argument('--hidden', type=int, default=64)
      parser.add_argument('--mc', type=int, default=1)
      parser.add_argument('--em', type=str, default=None) # entropy method
      parser.add_argument('--ec', type=float, default=0.) # entropy coefficient
      parser.add_argument('--lr', type=float, default=None)
      parser.add_argument('--bs', type=int, default=None)
      parser.add_argument('--epoch', type=int, default=None)
      parser.add_argument('--seed', type=int, default=0)
      parser.add_argument('--snapshot_mode', type=str, default="gap_and_last")
      parser.add_argument('--snapshot_gap', type=int, default=500)
      args = parser.parse_args()
      import os.path as osp
      pre_dir = './Data/'+args.exp_name\
                  +('bd' if args.boundary else '')\
                  +(('ag'+str(args.num_ag)) if args.num_ag else '')\
                  +(('adv'+str(args.num_adv)) if args.num_adv else '')\
                  +(('l'+str(args.num_l)) if args.num_l else '')\
                  +'_mpl'+str(args.mpl)
      main_dir = args.log_dir\
                  +('layer'+str(args.layer))\
                  +('hidden'+str(args.hidden))\
                  +('mc'+str(args.mc))\
                  +(('em'+str(args.em)+'ec'+str(args.ec)) if args.em else '')\
                  +(('lr'+str(args.lr)) if args.lr else '')\
                  +(('bs'+str(args.bs)) if args.bs else '')
      log_dir = osp.join(pre_dir,main_dir,'seed'+str(args.seed))
      # noinspection PyTypeChecker
      variant = dict(
          world_args=dict(
              num_agents=args.num_ag,
              num_adversaries=args.num_adv,
              num_landmarks=args.num_l,
              boundary=([[-1.,-1.],[1.,1.]] if args.boundary else None)
          ),
          algorithm_kwargs=dict(
              num_epochs=(args.epoch+1 if args.epoch else 1001),
              num_eval_steps_per_epoch=1000,
              num_trains_per_train_loop=1,
              num_expl_steps_per_train_loop=1000,
              max_path_length=args.mpl,
          ),
          trainer_kwargs=dict(
              discount=0.99,
              qf_learning_rate=(args.lr if args.lr else 1e-3),
              policy_learning_rate=(args.lr if args.lr else 1e-4),
              max_path_length=args.mpl,
              entropy_method=args.em,
              policy_ent_coeff=args.ec,
              batch_size=(args.bs if args.bs else 256),
              mc_num=args.mc,
          ),
          qf_kwargs=dict(
              num_layer=args.layer,
              hidden_dim=args.hidden,
          ),
          policy_kwargs=dict(
              num_layer=args.layer,
              hidden_dim=args.hidden,
          ),
      )
      import os
      if not os.path.isdir(log_dir):
          os.makedirs(log_dir)
      with open(osp.join(log_dir,'variant.json'),'w') as out_json:
          import json
          json.dump(variant,out_json,indent=2)
      import sys
      cmd_input = 'python ' + ' '.join(sys.argv) + '\n'
      with open(osp.join(log_dir, 'cmd_input.txt'), 'a') as f:
          f.write(cmd_input)
      setup_logger(args.exp_name+'/'+main_dir, variant=variant,
                  snapshot_mode=args.snapshot_mode, snapshot_gap=args.snapshot_gap,
                  log_dir=log_dir,text_log_file=None)
      import numpy as np
      import torch
      np.random.seed(args.seed)
      torch.manual_seed(args.seed)
      # ptu.set_gpu_mode(True)  # optionally set the GPU (default=False)
      experiment(variant)
  
apis: torch.nn.Linear torch.nn.ReLU torch.manual_seed numpy.random.seed
apis: torch.load
apis: torch.nn.Linear torch.manual_seed torch.save numpy.random.seed torch.nn.LeakyReLU
apis: torch.nn.Linear torch.manual_seed numpy.random.seed