
        return actors
    
    def get_target_actors(self):
        """get target_actors of all the agents in the MADDPG object"""
        target_actors = [ddpg_agent.target_actors for ddpg_agent in self.maddpg_agent]
        return target_actors
    
    def act(self, obs_all_agents, noise=0.0001):
        """get actions from all agents in the MADDPG object"""
        actions = [agent.act(obs, noise) for agent, obs in zip(self.maddpg_agent, obs_all_agents)]
        return actions
    
    def target_act(self, obs_all_agents, noise=0.0001):
        """Get target network actions from all the agent in the MADDPG object"""
        
        #target_actions = [ddpg_agent.target_act(obs, noise) for ddpg_agent,obs in zip(self.maddpg_agent, obs_all_agents)]
        target_actions = []
        for ii in range(2):
            target_actions.append(self.maddpg_agent[ii].target_act(obs_all_agents[:,ii,:],noise))
            
        return target_actions  
    
    def step(self, state, action, reward, next_state, done):
        """Save experience in replay memory, and use random sample from buffer to learn."""
        # Save experience / reward 
        self.memory.add(state, action, reward, next_state, done)
        self.counter+=1
        
        # Learn, if enough samples are available in memory
        if len(self.memory) > BATCH_SIZE:
            for ii in range(2):
                experience = self.memory.sample()
                self.learn(experience,ii)
                
    def learn(self, experience, agent_number):
        state,action,reward,next_state,done = experience
        agent = self.maddpg_agent[agent_number]
        critic = self.maddpg_agent[agent_number]
        
        ## ============================== ##
        #       Critic Training            #
        ## ============================== ##        
        
        critic.critic_optimizer.zero_grad()
        state_full = torch.cat((state[:,0,:],state[:,1,:]),dim=1)
        state_next_full = torch.cat((next_state[:,0,:],next_state[:,1,:]),dim=1)
        
        
        target_actions = self.target_act(next_state)
        target_actions = torch.cat(target_actions,dim=1)
        
        target_critic_input = torch.cat((state_next_full,target_actions),dim=1).to(device)
        #print(target_critic_input.size(),"Sdasdasd")
        with torch.no_grad():
            q_next = critic.target_critic(target_critic_input)
            
        y= reward[:,agent_number].reshape(-1,1) +self.discount_factor*q_next * (1 - done[:,agent_number].reshape(-1,1))
        #print(y.shape,"Asdas")
        action = action.view(-1,4)
        critic_input = torch.cat((state_full,action),dim=1).to(device)
        q = critic.critic(critic_input)
        
        huber_loss = torch.nn.MSELoss()
        critic_loss = huber_loss(q,y.detach())
        critic_loss.backward()
        agent.critic_optimizer.step()
        
        
        ## ============================== ##
        #         Agent Training           #
        ## ============================== ##
        
        agent.actor_optimizer.zero_grad()
        #print(state.shape,"Sdas")
        
        #q_input = [self.maddpg_agent[i].actor(ob[i]) if i == agent_number  else  self.maddpg_agent[i].actor(ob).detach() for i, ob in enumerate(state)]
        q_input = []
        for ii in range(2):
            if ii == agent_number:
                q_input.append(self.maddpg_agent[ii].actor(state[:,ii,:]))
            else:
                q_input.append(self.maddpg_agent[ii].actor(state[:,ii,:]).detach())
        
        q_input = torch.cat(q_input,dim=1)
        
        
        q_input2 = torch.cat((state_full,q_input),dim=1)
        
        actor_loss = -agent.critic(q_input2).mean()
        actor_loss.backward()
        agent.actor_optimizer.step()
        
        self.update_targets()
        
    def update_targets(self):
        """soft update targets"""
        for ddpg_agent in self.maddpg_agent:
            ddpg_agent.soft_update(ddpg_agent.target_actor, ddpg_agent.actor, self.tau)
            ddpg_agent.soft_update(ddpg_agent.target_critic, ddpg_agent.critic, self.tau)    
        
        
        
        

    

class ReplayBuffer:
    """Fixed-size buffer to store experience tuples."""

    def __init__(self, action_size, buffer_size, batch_size, seed):
        """Initialize a ReplayBuffer object.
        Params
        ======
            buffer_size (int): maximum size of buffer
            batch_size (int): size of each training batch
        """
        self.action_size = action_size
        self.memory = deque(maxlen=buffer_size)  # internal memory (deque)
        self.batch_size = batch_size
        self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])
        self.seed = random.seed(seed)
    
    def add(self, state, action, reward, next_state, done):
        """Add a new experience to memory."""
        e = self.experience(state, action, reward, next_state, done)
        self.memory.append(e)
    
    def sample(self):
        """Randomly sample a batch of experiences from memory."""
        experiences = random.sample(self.memory, k=self.batch_size)
        states = torch.from_numpy(np.array([e.state for e in experiences if e is not None])).float().to(device)
        actions = torch.from_numpy(np.array([e.action for e in experiences if e is not None])).float().to(device)
        rewards = torch.from_numpy(np.array([e.reward for e in experiences if e is not None])).float().to(device)
        next_states = torch.from_numpy(np.array([e.next_state for e in experiences if e is not None])).float().to(device)
        dones = torch.from_numpy(np.array([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)

        return (states, actions, rewards, next_states, dones)

    def __len__(self):
        """Return the current size of internal memory."""
        return len(self.memory)        
        
        
        
        
# -

maddpg = MADDPG()


def ddpg(n_episode=1500, max_t=320, print_every=100):
    scores_deque = deque(maxlen=print_every)
    scoress= []
    average_Score = []
    #logger = Logger('./logs')
    for i_episodes in range(1, n_episode+1):
        env_info = env.reset(train_mode=True)[brain_name]      # reset the environment    
        states = env_info.vector_observations # get the current state (for each agent)
        scores = np.zeros(num_agents)                          # initialize the score (for each agent)
        while True:
            actions=maddpg.act(states, noise =0.00009)
            actions = [a.detach().numpy() for a in actions]
            actions = np.array(actions).reshape(2,2)
            actions = np