docstr: Initializes a AlphaStarLeagueBuilder instance.
  
  The following match types are possible:
  LE: A learning (not snapshot) league_exploiter vs any snapshot policy.
  ME: A learning (not snapshot) main exploiter vs any main.
  M: Main self-play (main vs main).
  
  Args:
      algo: The Algorithm object by which this league builder is used.
          Algorithm calls `build_league()` after each training step to reconfigure
          the league structure (e.g. to add/remove policies).
      algo_config: The (not yet validated) config to be
          used on the Algorithm. Child classes of `LeagueBuilder`
          should preprocess this to add e.g. multiagent settings
          to this config.
      num_random_policies: The number of random policies to add to the
          league. This must be an even number (including 0) as these
          will be evenly distributed amongst league- and main- exploiters.
      num_learning_league_exploiters: The number of initially learning
          league-exploiters to create.
      num_learning_main_exploiters: The number of initially learning
          main-exploiters to create.
      win_rate_threshold_for_new_snapshot: The win-rate to be achieved
          for a learning policy to get snapshot'd (forked into `self` +
          a new learning or non-learning copy of `self`).
      keep_new_snapshot_training_prob: The probability with which a new
          snapshot should keep training. Note that the policy from which
          this snapshot is taken will continue to train regardless.
      prob_league_exploiter_match: Probability of an episode to become a
          league-exploiter vs snapshot match.
      prob_main_exploiter_match: Probability of an episode to become a
          main-exploiter vs main match.
      prob_main_exploiter_playing_against_learning_main: Probability of
          a main-exploiter vs (training!) main match.
docstr_tok: ["Initializes","a","AlphaStarLeagueBuilder","instance",".","The","following","match","types","are","possible",":","LE",":","A","learning","(","not","snapshot",")","league_exploiter","vs","any","snapshot","policy",".","ME",":","A","learning","(","not","snapshot",")","main","exploiter","vs","any","main",".","M",":","Main","self-play","(","main","vs","main",")",".","Args",":","algo",":","The","Algorithm","object","by","which","this","league","builder","is","used",".","Algorithm","calls","`","build_league","(",")","`","after","each","training","step","to","reconfigure","the","league","structure","(","e.g",".","to","add\/remove","policies",")",".","algo_config",":","The","(","not","yet","validated",")","config","to","be","used","on","the","Algorithm",".","Child","classes","of","`","LeagueBuilder","`","should","preprocess","this","to","add","e.g",".","multiagent","settings","to","this","config",".","num_random_policies",":","The","number","of","random","policies","to","add","to","the","league",".","This","must","be","an","even","number","(","including","0",")","as","these","will","be","evenly","distributed","amongst","league-","and","main-","exploiters",".","num_learning_league_exploiters",":","The","number","of","initially","learning","league-exploiters","to","create",".","num_learning_main_exploiters",":","The","number","of","initially","learning","main-exploiters","to","create",".","win_rate_threshold_for_new_snapshot",":","The","win-rate","to","be","achieved","for","a","learning","policy","to","get","snapshot","'d","(","forked","into","`","self","`","+","a","new","learning","or","non-learning","copy","of","`","self","`",")",".","keep_new_snapshot_training_prob",":","The","probability","with","which","a","new","snapshot","should","keep","training",".","Note","that","the","policy","from","which","this","snapshot","is","taken","will","continue","to","train","regardless",".","prob_league_exploiter_match",":","Probability","of","an","episode","to","become","a","league-exploiter","vs","snapshot","match",".","prob_main_exploiter_match",":","Probability","of","an","episode","to","become","a","main-exploiter","vs","main","match",".","prob_main_exploiter_playing_against_learning_main",":","Probability","of","a","main-exploiter","vs","(","training","!",")","main","match","."]
code: def __init__(
      self,
      algo: Algorithm,
      algo_config: AlgorithmConfig,
      num_random_policies: int = 2,
      num_learning_league_exploiters: int = 4,
      num_learning_main_exploiters: int = 4,
      win_rate_threshold_for_new_snapshot: float = 0.8,
      keep_new_snapshot_training_prob: float = 0.0,
      prob_league_exploiter_match: float = 0.33,
      prob_main_exploiter_match: float = 0.33,
      prob_main_exploiter_playing_against_learning_main: float = 0.5,
  ):
      """Initializes a AlphaStarLeagueBuilder instance.
  
      The following match types are possible:
      LE: A learning (not snapshot) league_exploiter vs any snapshot policy.
      ME: A learning (not snapshot) main exploiter vs any main.
      M: Main self-play (main vs main).
  
      Args:
          algo: The Algorithm object by which this league builder is used.
              Algorithm calls `build_league()` after each training step to reconfigure
              the league structure (e.g. to add/remove policies).
          algo_config: The (not yet validated) config to be
              used on the Algorithm. Child classes of `LeagueBuilder`
              should preprocess this to add e.g. multiagent settings
              to this config.
          num_random_policies: The number of random policies to add to the
              league. This must be an even number (including 0) as these
              will be evenly distributed amongst league- and main- exploiters.
          num_learning_league_exploiters: The number of initially learning
              league-exploiters to create.
          num_learning_main_exploiters: The number of initially learning
              main-exploiters to create.
          win_rate_threshold_for_new_snapshot: The win-rate to be achieved
              for a learning policy to get snapshot'd (forked into `self` +
              a new learning or non-learning copy of `self`).
          keep_new_snapshot_training_prob: The probability with which a new
              snapshot should keep training. Note that the policy from which
              this snapshot is taken will continue to train regardless.
          prob_league_exploiter_match: Probability of an episode to become a
              league-exploiter vs snapshot match.
          prob_main_exploiter_match: Probability of an episode to become a
              main-exploiter vs main match.
          prob_main_exploiter_playing_against_learning_main: Probability of
              a main-exploiter vs (training!) main match.
      """
      super().__init__(algo, algo_config)
  
      self.win_rate_threshold_for_new_snapshot = (
          win_rate_threshold_for_new_snapshot
      )
      self.keep_new_snapshot_training_prob = (
          keep_new_snapshot_training_prob
      )
      self.prob_league_exploiter_match = prob_league_exploiter_match
      self.prob_main_exploiter_match = prob_main_exploiter_match
      self.prob_main_exploiter_playing_against_learning_main = (
          prob_main_exploiter_playing_against_learning_main
      )
      # Store the win rates for league overview printouts.
      self.win_rates: DefaultDict[PolicyID, float] = defaultdict(float)
  
      assert num_random_policies % 2 == 0, (
          "ERROR: `num_random_policies` must be even number (we'll distribute "
          "these evenly amongst league- and main-exploiters)!"
      )
  
      # Build trainer's multiagent config.
      self.config._is_frozen = False
      # Make sure the multiagent config dict has no policies defined:
      assert self.config.policies is None, (
          "ERROR: `config.policies` should be None (not pre-defined by user)! "
          "AlphaStarLeagueBuilder will construct this itself."
      )
      policies = {}
  
      self.main_policies = 1
      self.league_exploiters = (
          num_learning_league_exploiters + num_random_policies / 2
      )
      self.main_exploiters = (
          num_learning_main_exploiters + num_random_policies / 2
      )
  
      # Add 1 initial (learning) main policy.
      policies["main_0"] = PolicySpec()
  
      # Train all non-random policies that exist at beginning.
      policies_to_train = ["main_0"]
  
      # Add random policies.
      i = -1
      for i in range(num_random_policies // 2):
          policies[f"league_exploiter_{i}"] = PolicySpec(
              policy_class=RandomPolicy
          )
          policies[f"main_exploiter_{i}"] = PolicySpec(
              policy_class=RandomPolicy
          )
      # Add initial (learning) league-exploiters.
      for j in range(num_learning_league_exploiters):
          pid = f"league_exploiter_{j + i + 1}"
          policies[pid] = PolicySpec()
          policies_to_train.append(pid)
      # Add initial (learning) main-exploiters.
      for j in range(num_learning_league_exploiters):
          pid = f"main_exploiter_{j + i + 1}"
          policies[pid] = PolicySpec()
          policies_to_train.append(pid)
  
      # Build initial policy mapping function: main_0 vs main_exploiter_0.
      self.config.policy_mapping_fn = (
          lambda agent_id, episode, worker, **kw: "main_0"
          if episode.episode_id % 2 == agent_id
          else "main_exploiter_0"
      )
      self.config.policies = policies
      self.config.policies_to_train = policies_to_train
      self.config.freeze()
  
imports: import logging
  import re
  from abc import ABCMeta
  from collections import defaultdict
  from typing import Any, DefaultDict, Dict
  import numpy
  from ray.rllib.algorithms.algorithm import Algorithm
  from ray.rllib.algorithms.algorithm_config import AlgorithmConfig
  from ray.rllib.examples.policy.random_policy import RandomPolicy
  from ray.rllib.policy.policy import PolicySpec
  from ray.rllib.utils.annotations import ExperimentalAPI, override
  from ray.rllib.utils.numpy import softmax
  from ray.rllib.utils.typing import PolicyID, ResultDict
label_desc: Use image node_id 1 to create a new AlphaStarLeagueBuilder object from inherited base classes: LeagueBuilder with example: obj = AlphaStarLeagueBuilder(algo, algo_config, num_random_policies, num_learning_league_exploiters, num_learning_main_exploiters, win_rate_threshold_for_new_snapshot, keep_new_snapshot_training_prob, prob_league_exploiter_match, prob_main_exploiter_match, prob_main_exploiter_playing_against_learning_main)