docstr: Args:   tp_engine : The tp engine that dynamic batch manager hold, defined before dynamic batch manager
          max_total_token_num : max_total_token_num for memory manager, default to: max batch size * (max input len + max output len)
          batch_max_tokens : max tokens of one batch, default to (max input + output len) * num_requests
          running_max_req_size : max request size of running batch, equals to MAX_BATCH_SIZE of tp engine
          eos_id : The end token of a seq
          model: the model weight dir path, the app will load config, weights and tokenizer from this dir
          log_stats : whether to log stats
          log_stats_interval : log stats interval
          running_batch : running batch
          waiting_req_list : list of waiting requests, initialized before dynamic batch manager
docstr_tok: ["Args",":","tp_engine",":","The","tp","engine","that","dynamic","batch","manager","hold",",","defined","before","dynamic","batch","manager","max_total_token_num",":","max_total_token_num","for","memory","manager",",","default","to",":","max","batch","size","*","(","max","input","len","+","max","output","len",")","batch_max_tokens",":","max","tokens","of","one","batch",",","default","to","(","max","input","+","output","len",")","*","num_requests","running_max_req_size",":","max","request","size","of","running","batch",",","equals","to","MAX_BATCH_SIZE","of","tp","engine","eos_id",":","The","end","token","of","a","seq","model",":","the","model","weight","dir","path",",","the","app","will","load","config",",","weights","and","tokenizer","from","this","dir","log_stats",":","whether","to","log","stats","log_stats_interval",":","log","stats","interval","running_batch",":","running","batch","waiting_req_list",":","list","of","waiting","requests",",","initialized","before","dynamic","batch","manager"]
code: def __init__(
      self,
      tp_engine: TPInferEngine,
      max_total_token_num: int,
      batch_max_tokens: int,
      model: str,
      tokenizer=None,
      eos_id=None,
      log_stats=True,
      log_stats_interval=10,
      running_batch: Batch = None,
      waiting_req_list: List = [],
  ):
      """
      Args:   tp_engine : The tp engine that dynamic batch manager hold, defined before dynamic batch manager
              max_total_token_num : max_total_token_num for memory manager, default to: max batch size * (max input len + max output len)
              batch_max_tokens : max tokens of one batch, default to (max input + output len) * num_requests
              running_max_req_size : max request size of running batch, equals to MAX_BATCH_SIZE of tp engine
              eos_id : The end token of a seq
              model: the model weight dir path, the app will load config, weights and tokenizer from this dir
              log_stats : whether to log stats
              log_stats_interval : log stats interval
              running_batch : running batch
              waiting_req_list : list of waiting requests, initialized before dynamic batch manager
      """
      super().__init__(
          tp_engine,
          max_total_token_num,
          batch_max_tokens,
          model,
          tokenizer,
          eos_id,
          log_stats,
          log_stats_interval,
          running_batch,
          waiting_req_list,
      )
  
file_path: ColossalAI/colossalai/legacy/inference/async_manager.py
imports: from typing import List
  from .dynamic_batching.io_struct import Batch, Req, RequestOutput
  from .manager import DynamicBatchManager
  from .tensor_parallel import TPInferEngine
label_desc: Use image node_id 1 to create a new Async_DynamicBatchManager object from inherited base classes: DynamicBatchManager with example: obj = Async_DynamicBatchManager(tp_engine, max_total_token_num, batch_max_tokens, model, tokenizer, eos_id, log_stats, log_stats_interval, running_batch, waiting_req_list)