text: from __future__ import absolute_import, division, print_function
  
  import base64
  import imageio
  import numpy as np
  import PIL.Image
  
  import tensorflow as tf
  
  from tf_agents.agents.dqn import dqn_agent
  from tf_agents.drivers import dynamic_step_driver
  from tf_agents.environments import suite_gym
  from tf_agents.environments import tf_py_environment
  from tf_agents.eval import metric_utils
  from tf_agents.metrics import tf_metrics
  from tf_agents.networks import q_network
  from tf_agents.policies import random_tf_policy
  from tf_agents.replay_buffers import tf_uniform_replay_buffer
  from tf_agents.trajectories import trajectory
  from tf_agents.utils import common
  
  from data_collection import collect_data, collect_step
  from metrics import compute_avg_return
  
  tf.compat.v1.enable_v2_behavior()
  
  #### Hyperparameters
  
  num_iterations = 20000
  initial_collect_steps = 1000
  collect_steps_per_iteration = 1
  replay_buffer_max_length = 100000
  
  batch_size = 64
  learning_rate = 1e-3
  log_interval = 200
  
  num_eval_episodes = 10
  eval_interval = 1000
  
  #### Environment
  
  env_name = 'MountainCar-v0'
  env = suite_gym.load(env_name)
  
  train_py_env = suite_gym.load(env_name)
  eval_py_env = suite_gym.load(env_name)
  
  train_env = tf_py_environment.TFPyEnvironment(train_py_env)
  eval_env = tf_py_environment.TFPyEnvironment(eval_py_env)
  
  #### Agent
  
  q_net = q_network.QNetwork(
      train_env.observation_spec(),
      train_env.action_spec(),
      fc_layer_params=(75,40))
  
  optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)
  
  train_step_counter = tf.Variable(0)
  
  agent = dqn_agent.DdqnAgent(
      train_env.time_step_spec(),
      train_env.action_spec(),
      q_network=q_net,
      optimizer=optimizer,
      td_errors_loss_fn=common.element_wise_squared_loss,
      train_step_counter=train_step_counter)
  
  agent.initialize()
  
  #### Policies
  
  random_policy = random_tf_policy.RandomTFPolicy(train_env.time_step_spec(),
                                                  train_env.action_spec())
  
  #### Replay Buffer
  
  replay_buffer = tf_uniform_replay_buffer.TFUniformReplayBuffer(
      data_spec=agent.collect_data_spec,
      batch_size=train_env.batch_size,
      max_length=replay_buffer_max_length)
  
  collect_data(train_env, random_policy, replay_buffer, steps=initial_collect_steps)
  
  dataset = replay_buffer.as_dataset(
      num_parallel_calls=3,
      sample_batch_size=batch_size,
      num_steps=2).prefetch(3)
  
  iterator = iter(dataset)
  
  #### Training the Agent
  
  # (Optional) Optimize by wrapping some of the code in a graph using TF function.
  agent.train = common.function(agent.train)
  
  # Reset the train step
  agent.train_step_counter.assign(0)
  
  # Evaluate the agent's policy once before training.
  avg_return = compute_avg_return(eval_env, agent.policy, num_eval_episodes)
  returns = [avg_return]
  
  for _ in range(num_iterations):
  
      for _ in range(collect_steps_per_iteration):
          collect_step(train_env, agent.collect_policy, replay_buffer)
  
      experience, unused_info = next(iterator)
      train_loss = agent.train(experience).loss
      step = agent.train_step_counter.numpy()
  
      if step % log_interval == 0:
          print('step = {0}: loss = {1}'.format(step, train_loss))
  
      if step % eval_interval == 0:
          avg_return = compute_avg_return(eval_env, agent.policy, num_eval_episodes)
          print('step = {0}: Average Return = {1}'.format(step, avg_return))
          returns.append(avg_return)
  
  returnsArray = np.asarray(returns)
  np.save("returns", returnsArray)
  