class_docstr: The LoadBalancer is a LightningWork component that collects the requests and sends them to the prediciton API
  asynchronously using RoundRobin scheduling. It also performs auto batching of the incoming requests.
  
  After enabling you will require to send username and password from the request header for the private endpoints.
  
  Args:
      input_type: Input type.
      output_type: Output type.
      endpoint: The REST API path.
      max_batch_size: The number of requests processed at once.
      timeout_batching: The number of seconds to wait before sending the requests to process in order to allow for
          requests to be batched. In any case, requests are processed as soon as `max_batch_size` is reached.
      timeout_keep_alive: The number of seconds until it closes Keep-Alive connections if no new data is received.
      timeout_inference_request: The number of seconds to wait for inference.
      api_name: The name to be displayed on the UI. Normally, it is the name of the work class
      cold_start_proxy: The proxy service to use while the work is cold starting.
      **kwargs: Arguments passed to :func:`LightningWork.init` like ``CloudCompute``, ``BuildConfig``, etc.
class_docstr_tok: ["The","LoadBalancer","is","a","LightningWork","component","that","collects","the","requests","and","sends","them","to","the","prediciton","API","asynchronously","using","RoundRobin","scheduling",".","It","also","performs","auto","batching","of","the","incoming","requests",".","After","enabling","you","will","require","to","send","username","and","password","from","the","request","header","for","the","private","endpoints",".","Args",":","input_type",":","Input","type",".","output_type",":","Output","type",".","endpoint",":","The","REST","API","path",".","max_batch_size",":","The","number","of","requests","processed","at","once",".","timeout_batching",":","The","number","of","seconds","to","wait","before","sending","the","requests","to","process","in","order","to","allow","for","requests","to","be","batched",".","In","any","case",",","requests","are","processed","as","soon","as","`","max_batch_size","`","is","reached",".","timeout_keep_alive",":","The","number","of","seconds","until","it","closes","Keep-Alive","connections","if","no","new","data","is","received",".","timeout_inference_request",":","The","number","of","seconds","to","wait","for","inference",".","api_name",":","The","name","to","be","displayed","on","the","UI",".","Normally",",","it","is","the","name","of","the","work","class","cold_start_proxy",":","The","proxy","service","to","use","while","the","work","is","cold","starting",".","*","*","kwargs",":","Arguments","passed","to",":","func",":","`","LightningWork.init","`","like","``","CloudCompute","``",",","``","BuildConfig","``",",","etc","."]
code: def run(self):
      logger.info(f"servers: {self.servers}")
  
      self._iter = cycle(self.servers)
  
      fastapi_app = _create_fastapi("Load Balancer")
      fastapi_app.SEND_TASK = None
      self._fastapi_app = fastapi_app
  
      input_type = self._input_type
  
      @fastapi_app.middleware("http")
      async def current_request_counter(request: Request, call_next):
          if request.scope["path"] != self.endpoint:
              return await call_next(request)
          fastapi_app.global_request_count += 1
          fastapi_app.num_current_requests += 1
          start_time = time.time()
          response = await call_next(request)
          processing_time = time.time() - start_time
          fastapi_app.last_processing_time = processing_time
          fastapi_app.num_current_requests -= 1
          return response
  
      @fastapi_app.on_event("startup")
      async def startup_event():
          fastapi_app.SEND_TASK = asyncio.create_task(self.consumer())
  
      @fastapi_app.on_event("shutdown")
      def shutdown_event():
          fastapi_app.SEND_TASK.cancel()
  
      @fastapi_app.get("/system/info", response_model=_SysInfo)
      async def sys_info():
          return _SysInfo(
              num_workers=len(self.servers),
              servers=self.servers,
              num_requests=fastapi_app.num_current_requests,
              processing_time=fastapi_app.last_processing_time,
              global_request_count=fastapi_app.global_request_count,
          )
  
      @fastapi_app.put("/system/update-servers")
      async def update_servers(servers: List[str]):
          self.servers = servers
          self._iter = cycle(self.servers)
          updated_servers = set()
          # do not try to loop over the dict keys as the dict might change from other places
          existing_servers = list(self._server_status.keys())
          for server in servers:
              updated_servers.add(server)
              if server not in existing_servers:
                  self._server_status[server] = True
                  logger.info(
                      f"Registering server {server}",
                      self._server_status,
                  )
          for existing in existing_servers:
              if existing not in updated_servers:
                  logger.info(
                      f"De-Registering server {existing}",
                      self._server_status,
                  )
                  del self._server_status[existing]
  
      @fastapi_app.post(self.endpoint, response_model=self._output_type)
      async def balance_api(inputs: input_type):
          return await self.process_request(inputs)
  
      endpoint_info_page = self._get_endpoint_info_page()
      if endpoint_info_page:
          fastapi_app.mount(
              "/endpoint-info",
              StaticFiles(
                  directory=endpoint_info_page.serve_dir, html=True
              ),
              name="static",
          )
  
      logger.info(
          f"Your load balancer has started. The endpoint is 'http://{self.host}:{self.port}{self.endpoint}'"
      )
      self.ready = True
      uvicorn.run(
          fastapi_app,
          host=self.host,
          port=self.port,
          loop="uvloop",
          timeout_keep_alive=self._timeout_keep_alive,
          access_log=False,
      )
  
imports: import asyncio
  import logging
  import time
  import uuid
  from itertools import cycle
  from typing import Any, Dict, List, Optional, Tuple, Type, Union
  from typing import SupportsFloat
  import requests
  import uvicorn
  from fastapi import FastAPI, HTTPException, Request
  from fastapi.middleware.cors import CORSMiddleware
  from fastapi.responses import RedirectResponse
  from pydantic import BaseModel
  from starlette.staticfiles import StaticFiles
  from lightning.app.components.serve.cold_start_proxy import ColdStartProxy
  from lightning.app.core.flow import LightningFlow
  from lightning.app.core.work import LightningWork
  from lightning.app.utilities.app_helpers import Logger
  from lightning.app.utilities.cloud import is_running_in_cloud
  from lightning.app.utilities.imports import _is_aiohttp_available, requires
  from lightning.app.utilities.packaging.cloud_compute import CloudCompute
label_desc: Use image node_id 5 for calling the _LoadBalancer obj's underlying member method code with example usage: obj.run() and returns: response, _SysInfo, unknown, unknown