code: def completion(
      model: str,
      # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create
      messages: List = [],
      timeout: Optional[Union[float, int]] = None,
      temperature: Optional[float] = None,
      top_p: Optional[float] = None,
      n: Optional[int] = None,
      stream: Optional[bool] = None,
      stop=None,
      max_tokens: Optional[float] = None,
      presence_penalty: Optional[float] = None,
      frequency_penalty: Optional[float] = None,
      logit_bias: Optional[dict] = None,
      user: Optional[str] = None,
      # openai v1.0+ new params
      response_format: Optional[dict] = None,
      seed: Optional[int] = None,
      tools: Optional[List] = None,
      tool_choice: Optional[str] = None,
      logprobs: Optional[bool] = None,
      top_logprobs: Optional[int] = None,
      deployment_id=None,
      # soon to be deprecated params by OpenAI
      functions: Optional[List] = None,
      function_call: Optional[str] = None,
      # set api_base, api_version, api_key
      base_url: Optional[str] = None,
      api_version: Optional[str] = None,
      api_key: Optional[str] = None,
      model_list: Optional[
          list
      ] = None,  # pass in a list of api_base,keys, etc.
      # Optional liteLLM function params
      **kwargs,
  ) -> Union[ModelResponse, CustomStreamWrapper]:
      """
      Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly)
      Parameters:
          model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/
          messages (List): A list of message objects representing the conversation context (default is an empty list).
  
          OPTIONAL PARAMS
          functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list).
          function_call (str, optional): The name of the function to call within the conversation (default is an empty string).
          temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0).
          top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0).
          n (int, optional): The number of completions to generate (default is 1).
          stream (bool, optional): If True, return a streaming response (default is False).
          stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens.
          max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity).
          presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far.
          frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
          logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
          user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
          logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message
          top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used.
          metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc.
          api_base (str, optional): Base URL for the API (default is None).
          api_version (str, optional): API version (default is None).
          api_key (str, optional): API key (default is None).
          model_list (list, optional): List of api base, version, keys
  
          LITELLM Specific Params
          mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
          custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock"
          max_retries (int, optional): The number of retries to attempt (default is 0).
      Returns:
          ModelResponse: A response object containing the generated completion and associated metadata.
  
      Note:
          - This function is used to perform completions() using the specified language model.
          - It supports various optional parameters for customizing the completion behavior.
          - If 'mock_response' is provided, a mock completion response is returned for testing or debugging.
      """
      ######### unpacking kwargs #####################
      args = locals()
      api_base = kwargs.get("api_base", None)
      mock_response = kwargs.get("mock_response", None)
      force_timeout = kwargs.get("force_timeout", 600)  ## deprecated
      logger_fn = kwargs.get("logger_fn", None)
      verbose = kwargs.get("verbose", False)
      custom_llm_provider = kwargs.get("custom_llm_provider", None)
      litellm_logging_obj = kwargs.get("litellm_logging_obj", None)
      id = kwargs.get("id", None)
      metadata = kwargs.get("metadata", None)
      model_info = kwargs.get("model_info", None)
      proxy_server_request = kwargs.get("proxy_server_request", None)
      fallbacks = kwargs.get("fallbacks", None)
      headers = kwargs.get("headers", None)
      num_retries = kwargs.get("num_retries", None)  ## deprecated
      max_retries = kwargs.get("max_retries", None)
      context_window_fallback_dict = kwargs.get(
          "context_window_fallback_dict", None
      )
      ### CUSTOM MODEL COST ###
      input_cost_per_token = kwargs.get("input_cost_per_token", None)
      output_cost_per_token = kwargs.get("output_cost_per_token", None)
      ### CUSTOM PROMPT TEMPLATE ###
      initial_prompt_value = kwargs.get("initial_prompt_value", None)
      roles = kwargs.get("roles", None)
      final_prompt_value = kwargs.get("final_prompt_value", None)
      bos_token = kwargs.get("bos_token", None)
      eos_token = kwargs.get("eos_token", None)
      preset_cache_key = kwargs.get("preset_cache_key", None)
      hf_model_name = kwargs.get("hf_model_name", None)
      ### ASYNC CALLS ###
      acompletion = kwargs.get("acompletion", False)
      client = kwargs.get("client", None)
      ######## end of unpacking kwargs ###########
      openai_params = [
          "functions",
          "function_call",
          "temperature",
          "temperature",
          "top_p",
          "n",
          "stream",
          "stop",
          "max_tokens",
          "presence_penalty",
          "frequency_penalty",
          "logit_bias",
          "user",
          "request_timeout",
          "api_base",
          "api_version",
          "api_key",
          "deployment_id",
          "organization",
          "base_url",
          "default_headers",
          "timeout",
          "response_format",
          "seed",
          "tools",
          "tool_choice",
          "max_retries",
          "logprobs",
          "top_logprobs",
      ]
      litellm_params = [
          "metadata",
          "acompletion",
          "caching",
          "mock_response",
          "api_key",
          "api_version",
          "api_base",
          "force_timeout",
          "logger_fn",
          "verbose",
          "custom_llm_provider",
          "litellm_logging_obj",
          "litellm_call_id",
          "use_client",
          "id",
          "fallbacks",
          "azure",
          "headers",
          "model_list",
          "num_retries",
          "context_window_fallback_dict",
          "roles",
          "final_prompt_value",
          "bos_token",
          "eos_token",
          "request_timeout",
          "complete_response",
          "self",
          "client",
          "rpm",
          "tpm",
          "input_cost_per_token",
          "output_cost_per_token",
          "hf_model_name",
          "model_info",
          "proxy_server_request",
          "preset_cache_key",
          "caching_groups",
          "ttl",
          "cache",
      ]
      default_params = openai_params + litellm_params
      non_default_params = {
          k: v for k, v in kwargs.items() if k not in default_params
      }  # model-specific params - pass them straight to the model/provider
      if mock_response:
          return mock_completion(
              model,
              messages,
              stream=stream,
              mock_response=mock_response,
          )
      if timeout is None:
          timeout = (
              kwargs.get("request_timeout", None) or 600
          )  # set timeout for 10 minutes by default
      timeout = float(timeout)
      try:
          if base_url is not None:
              api_base = base_url
          if (
              max_retries is not None
          ):  # openai allows openai.OpenAI(max_retries=3)
              num_retries = max_retries
          logging = litellm_logging_obj
          fallbacks = fallbacks or litellm.model_fallbacks
          if fallbacks is not None:
              return completion_with_fallbacks(**args)
          if model_list is not None:
              deployments = [
                  m["litellm_params"]
                  for m in model_list
                  if m["model_name"] == model
              ]
              return batch_completion_models(
                  deployments=deployments, **args
              )
          if (
              litellm.model_alias_map
              and model in litellm.model_alias_map
          ):
              model = litellm.model_alias_map[
                  model
              ]  # update the model to the actual value if an alias has been passed in
          model_response = ModelResponse()
          if (
              kwargs.get("azure", False) == True
          ):  # don't remove flag check, to remain backwards compatible for repos like Codium
              custom_llm_provider = "azure"
          if deployment_id != None:  # azure llms
              model = deployment_id
              custom_llm_provider = "azure"
          (
              model,
              custom_llm_provider,
              dynamic_api_key,
              api_base,
          ) = get_llm_provider(
              model=model,
              custom_llm_provider=custom_llm_provider,
              api_base=api_base,
              api_key=api_key,
          )
          if model_response is not None and hasattr(
              model_response, "_hidden_params"
          ):
              model_response._hidden_params[
                  "custom_llm_provider"
              ] = custom_llm_provider
          ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
          if (
              input_cost_per_token is not None
              and output_cost_per_token is not None
          ):
              litellm.register_model(
                  {
                      model: {
                          "input_cost_per_token": input_cost_per_token,
                          "output_cost_per_token": output_cost_per_token,
                          "litellm_provider": custom_llm_provider,
                      }
                  }
              )
          ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
          custom_prompt_dict = {}  # type: ignore
          if (
              initial_prompt_value
              or roles
              or final_prompt_value
              or bos_token
              or eos_token
          ):
              custom_prompt_dict = {model: {}}
              if initial_prompt_value:
                  custom_prompt_dict[model][
                      "initial_prompt_value"
                  ] = initial_prompt_value
              if roles:
                  custom_prompt_dict[model]["roles"] = roles
              if final_prompt_value:
                  custom_prompt_dict[model][
                      "final_prompt_value"
                  ] = final_prompt_value
              if bos_token:
                  custom_prompt_dict[model]["bos_token"] = bos_token
              if eos_token:
                  custom_prompt_dict[model]["eos_token"] = eos_token
          model_api_key = get_api_key(
              llm_provider=custom_llm_provider, dynamic_api_key=api_key
          )  # get the api key from the environment if required for the model
  
          if dynamic_api_key is not None:
              api_key = dynamic_api_key
          # check if user passed in any of the OpenAI optional params
          optional_params = get_optional_params(
              functions=functions,
              function_call=function_call,
              temperature=temperature,
              top_p=top_p,
              n=n,
              stream=stream,
              stop=stop,
              max_tokens=max_tokens,
              presence_penalty=presence_penalty,
              frequency_penalty=frequency_penalty,
              logit_bias=logit_bias,
              user=user,
              # params to identify the model
              model=model,
              custom_llm_provider=custom_llm_provider,
              response_format=response_format,
              seed=seed,
              tools=tools,
              tool_choice=tool_choice,
              max_retries=max_retries,
              logprobs=logprobs,
              top_logprobs=top_logprobs,
              **non_default_params,
          )
  
          if litellm.add_function_to_prompt and optional_params.get(
              "functions_unsupported_model", None
          ):  # if user opts to add it to prompt, when API doesn't support function calling
              functions_unsupported_model = optional_params.pop(
                  "functions_unsupported_model"
              )
              messages = function_call_prompt(
                  messages=messages,
                  functions=functions_unsupported_model,
              )
  
          # For logging - save the values of the litellm-specific params passed in
          litellm_params = get_litellm_params(
              acompletion=acompletion,
              api_key=api_key,
              force_timeout=force_timeout,
              logger_fn=logger_fn,
              verbose=verbose,
              custom_llm_provider=custom_llm_provider,
              api_base=api_base,
              litellm_call_id=kwargs.get("litellm_call_id", None),
              model_alias_map=litellm.model_alias_map,
              completion_call_id=id,
              metadata=metadata,
              model_info=model_info,
              proxy_server_request=proxy_server_request,
              preset_cache_key=preset_cache_key,
          )
          logging.update_environment_variables(
              model=model,
              user=user,
              optional_params=optional_params,
              litellm_params=litellm_params,
          )
          if custom_llm_provider == "azure":
              # azure configs
              api_type = get_secret("AZURE_API_TYPE") or "azure"
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("AZURE_API_BASE")
              )
  
              api_version = (
                  api_version
                  or litellm.api_version
                  or get_secret("AZURE_API_VERSION")
              )
  
              api_key = (
                  api_key
                  or litellm.api_key
                  or litellm.azure_key
                  or get_secret("AZURE_OPENAI_API_KEY")
                  or get_secret("AZURE_API_KEY")
              )
  
              azure_ad_token = optional_params.pop(
                  "azure_ad_token", None
              ) or get_secret("AZURE_AD_TOKEN")
  
              headers = headers or litellm.headers
  
              ## LOAD CONFIG - if set
              config = litellm.AzureOpenAIConfig.get_config()
              for k, v in config.items():
                  if (
                      k not in optional_params
                  ):  # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
                      optional_params[k] = v
  
              ## COMPLETION CALL
              response = azure_chat_completions.completion(
                  model=model,
                  messages=messages,
                  headers=headers,
                  api_key=api_key,
                  api_base=api_base,
                  api_version=api_version,
                  api_type=api_type,
                  azure_ad_token=azure_ad_token,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  logging_obj=logging,
                  acompletion=acompletion,
                  timeout=timeout,
                  client=client,  # pass AsyncAzureOpenAI, AzureOpenAI client
              )
  
              if (
                  optional_params.get("stream", False)
                  or acompletion == True
              ):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=api_key,
                      original_response=response,
                      additional_args={
                          "headers": headers,
                          "api_version": api_version,
                          "api_base": api_base,
                      },
                  )
          elif (
              model in litellm.open_ai_chat_completion_models
              or custom_llm_provider == "custom_openai"
              or custom_llm_provider == "deepinfra"
              or custom_llm_provider == "perplexity"
              or custom_llm_provider == "anyscale"
              or custom_llm_provider == "mistral"
              or custom_llm_provider == "openai"
              or "ft:gpt-3.5-turbo" in model  # finetune gpt-3.5-turbo
          ):  # allow user to make an openai call with a custom base
              # note: if a user sets a custom base - we should ensure this works
              # allow for the setting of dynamic and stateful api-bases
              api_base = (
                  api_base  # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api base from there
                  or litellm.api_base
                  or get_secret("OPENAI_API_BASE")
                  or "https://api.openai.com/v1"
              )
              openai.organization = (
                  litellm.organization
                  or get_secret("OPENAI_ORGANIZATION")
                  or None  # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
              )
              # set API KEY
              api_key = (
                  api_key
                  or litellm.api_key  # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
                  or litellm.openai_key
                  or get_secret("OPENAI_API_KEY")
              )
  
              headers = headers or litellm.headers
  
              ## LOAD CONFIG - if set
              config = litellm.OpenAIConfig.get_config()
              for k, v in config.items():
                  if (
                      k not in optional_params
                  ):  # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
                      optional_params[k] = v
  
              ## COMPLETION CALL
              try:
                  response = openai_chat_completions.completion(
                      model=model,
                      messages=messages,
                      headers=headers,
                      model_response=model_response,
                      print_verbose=print_verbose,
                      api_key=api_key,
                      api_base=api_base,
                      acompletion=acompletion,
                      logging_obj=logging,
                      optional_params=optional_params,
                      litellm_params=litellm_params,
                      logger_fn=logger_fn,
                      timeout=timeout,
                      custom_prompt_dict=custom_prompt_dict,
                      client=client,  # pass AsyncOpenAI, OpenAI client
                  )
              except Exception as e:
                  ## LOGGING - log the original exception returned
                  logging.post_call(
                      input=messages,
                      api_key=api_key,
                      original_response=str(e),
                      additional_args={"headers": headers},
                  )
                  raise e
  
              if optional_params.get("stream", False):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=api_key,
                      original_response=response,
                      additional_args={"headers": headers},
                  )
          elif (
              custom_llm_provider == "text-completion-openai"
              or "ft:babbage-002" in model
              or "ft:davinci-002"
              in model  # support for finetuned completion models
          ):
              openai.api_type = "openai"
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("OPENAI_API_BASE")
                  or "https://api.openai.com/v1"
              )
  
              openai.api_version = None
              # set API KEY
  
              api_key = (
                  api_key
                  or litellm.api_key
                  or litellm.openai_key
                  or get_secret("OPENAI_API_KEY")
              )
  
              headers = headers or litellm.headers
  
              ## LOAD CONFIG - if set
              config = litellm.OpenAITextCompletionConfig.get_config()
              for k, v in config.items():
                  if (
                      k not in optional_params
                  ):  # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in
                      optional_params[k] = v
              if litellm.organization:
                  openai.organization = litellm.organization
  
              if (
                  len(messages) > 0
                  and "content" in messages[0]
                  and type(messages[0]["content"]) == list
              ):
                  # text-davinci-003 can accept a string or array, if it's an array, assume the array is set in messages[0]['content']
                  # https://platform.openai.com/docs/api-reference/completions/create
                  prompt = messages[0]["content"]
              else:
                  prompt = " ".join([message["content"] for message in messages])  # type: ignore
              ## COMPLETION CALL
              model_response = openai_text_completions.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  api_key=api_key,
                  api_base=api_base,
                  acompletion=acompletion,
                  logging_obj=logging,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  timeout=timeout,
              )
  
              if (
                  optional_params.get("stream", False)
                  or acompletion == True
              ):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=api_key,
                      original_response=model_response,
                      additional_args={"headers": headers},
                  )
              response = model_response
          elif (
              "replicate" in model
              or custom_llm_provider == "replicate"
              or model in litellm.replicate_models
          ):
              # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN")
              replicate_key = None
              replicate_key = (
                  api_key
                  or litellm.replicate_key
                  or litellm.api_key
                  or get_secret("REPLICATE_API_KEY")
                  or get_secret("REPLICATE_API_TOKEN")
              )
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("REPLICATE_API_BASE")
                  or "https://api.replicate.com/v1"
              )
  
              custom_prompt_dict = (
                  custom_prompt_dict or litellm.custom_prompt_dict
              )
  
              model_response = replicate.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,  # for calculating input/output tokens
                  api_key=replicate_key,
                  logging_obj=logging,
                  custom_prompt_dict=custom_prompt_dict,
              )
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  model_response = CustomStreamWrapper(model_response, model, logging_obj=logging, custom_llm_provider="replicate")  # type: ignore
  
              if (
                  optional_params.get("stream", False)
                  or acompletion == True
              ):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=replicate_key,
                      original_response=model_response,
                  )
  
              response = model_response
  
          elif custom_llm_provider == "anthropic":
              api_key = (
                  api_key
                  or litellm.anthropic_key
                  or litellm.api_key
                  or os.environ.get("ANTHROPIC_API_KEY")
              )
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("ANTHROPIC_API_BASE")
                  or "https://api.anthropic.com/v1/complete"
              )
              custom_prompt_dict = (
                  custom_prompt_dict or litellm.custom_prompt_dict
              )
              response = anthropic.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  custom_prompt_dict=litellm.custom_prompt_dict,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,  # for calculating input/output tokens
                  api_key=api_key,
                  logging_obj=logging,
              )
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      response,
                      model,
                      custom_llm_provider="anthropic",
                      logging_obj=logging,
                  )
  
              if (
                  optional_params.get("stream", False)
                  or acompletion == True
              ):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=api_key,
                      original_response=response,
                  )
              response = response
          elif custom_llm_provider == "nlp_cloud":
              nlp_cloud_key = (
                  api_key
                  or litellm.nlp_cloud_key
                  or get_secret("NLP_CLOUD_API_KEY")
                  or litellm.api_key
              )
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("NLP_CLOUD_API_BASE")
                  or "https://api.nlpcloud.io/v1/gpu/"
              )
  
              response = nlp_cloud.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=nlp_cloud_key,
                  logging_obj=logging,
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      response,
                      model,
                      custom_llm_provider="nlp_cloud",
                      logging_obj=logging,
                  )
  
              if (
                  optional_params.get("stream", False)
                  or acompletion == True
              ):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=api_key,
                      original_response=response,
                  )
  
              response = response
          elif custom_llm_provider == "aleph_alpha":
              aleph_alpha_key = (
                  api_key
                  or litellm.aleph_alpha_key
                  or get_secret("ALEPH_ALPHA_API_KEY")
                  or get_secret("ALEPHALPHA_API_KEY")
                  or litellm.api_key
              )
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("ALEPH_ALPHA_API_BASE")
                  or "https://api.aleph-alpha.com/complete"
              )
  
              model_response = aleph_alpha.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  default_max_tokens_to_sample=litellm.max_tokens,
                  api_key=aleph_alpha_key,
                  logging_obj=logging,  # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="aleph_alpha",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "cohere":
              cohere_key = (
                  api_key
                  or litellm.cohere_key
                  or get_secret("COHERE_API_KEY")
                  or get_secret("CO_API_KEY")
                  or litellm.api_key
              )
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("COHERE_API_BASE")
                  or "https://api.cohere.ai/v1/generate"
              )
  
              model_response = cohere.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=cohere_key,
                  logging_obj=logging,  # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="cohere",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "maritalk":
              maritalk_key = (
                  api_key
                  or litellm.maritalk_key
                  or get_secret("MARITALK_API_KEY")
                  or litellm.api_key
              )
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("MARITALK_API_BASE")
                  or "https://chat.maritaca.ai/api/chat/inference"
              )
  
              model_response = maritalk.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=maritalk_key,
                  logging_obj=logging,
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="maritalk",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "huggingface":
              custom_llm_provider = "huggingface"
              huggingface_key = (
                  api_key
                  or litellm.huggingface_key
                  or os.environ.get("HF_TOKEN")
                  or os.environ.get("HUGGINGFACE_API_KEY")
                  or litellm.api_key
              )
              hf_headers = headers or litellm.headers
  
              custom_prompt_dict = (
                  custom_prompt_dict or litellm.custom_prompt_dict
              )
              model_response = huggingface.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,  # type: ignore
                  headers=hf_headers,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=huggingface_key,
                  acompletion=acompletion,
                  logging_obj=logging,
                  custom_prompt_dict=custom_prompt_dict,
                  timeout=timeout,
              )
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
                  and acompletion is False
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="huggingface",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "oobabooga":
              custom_llm_provider = "oobabooga"
              model_response = oobabooga.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  api_base=api_base,  # type: ignore
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  api_key=None,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  logging_obj=logging,
              )
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="oobabooga",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "openrouter":
              api_base = (
                  api_base
                  or litellm.api_base
                  or "https://openrouter.ai/api/v1"
              )
  
              api_key = (
                  api_key
                  or litellm.api_key
                  or litellm.openrouter_key
                  or get_secret("OPENROUTER_API_KEY")
                  or get_secret("OR_API_KEY")
              )
  
              openrouter_site_url = (
                  get_secret("OR_SITE_URL") or "https://litellm.ai"
              )
  
              openrouter_app_name = (
                  get_secret("OR_APP_NAME") or "liteLLM"
              )
  
              headers = (
                  headers
                  or litellm.headers
                  or {
                      "HTTP-Referer": openrouter_site_url,
                      "X-Title": openrouter_app_name,
                  }
              )
  
              ## Load Config
              config = openrouter.OpenrouterConfig.get_config()
              for k, v in config.items():
                  if k == "extra_body":
                      # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models
                      if "extra_body" in optional_params:
                          optional_params[k].update(v)
                      else:
                          optional_params[k] = v
                  elif k not in optional_params:
                      optional_params[k] = v
  
              data = {
                  "model": model,
                  "messages": messages,
                  **optional_params,
              }
  
              ## COMPLETION CALL
              response = openai_chat_completions.completion(
                  model=model,
                  messages=messages,
                  headers=headers,
                  api_key=api_key,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  logging_obj=logging,
                  acompletion=acompletion,
                  timeout=timeout,
              )
              ## LOGGING
              logging.post_call(
                  input=messages,
                  api_key=openai.api_key,
                  original_response=response,
              )
          elif (
              custom_llm_provider == "together_ai"
              or ("togethercomputer" in model)
              or (model in litellm.together_ai_models)
          ):
              custom_llm_provider = "together_ai"
              together_ai_key = (
                  api_key
                  or litellm.togetherai_api_key
                  or get_secret("TOGETHER_AI_TOKEN")
                  or get_secret("TOGETHERAI_API_KEY")
                  or litellm.api_key
              )
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("TOGETHERAI_API_BASE")
                  or "https://api.together.xyz/inference"
              )
  
              custom_prompt_dict = (
                  custom_prompt_dict or litellm.custom_prompt_dict
              )
  
              model_response = together_ai.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=together_ai_key,
                  logging_obj=logging,
                  custom_prompt_dict=custom_prompt_dict,
              )
              if (
                  "stream_tokens" in optional_params
                  and optional_params["stream_tokens"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="together_ai",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "palm":
              palm_api_key = (
                  api_key
                  or get_secret("PALM_API_KEY")
                  or litellm.api_key
              )
  
              # palm does not support streaming as yet :(
              model_response = palm.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=palm_api_key,
                  logging_obj=logging,
              )
              # fake palm streaming
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # fake streaming for palm
                  resp_string = model_response["choices"][0]["message"][
                      "content"
                  ]
                  response = CustomStreamWrapper(
                      resp_string,
                      model,
                      custom_llm_provider="palm",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "gemini":
              gemini_api_key = (
                  api_key
                  or get_secret("GEMINI_API_KEY")
                  or get_secret(
                      "PALM_API_KEY"
                  )  # older palm api key should also work
                  or litellm.api_key
              )
  
              # palm does not support streaming as yet :(
              model_response = gemini.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=gemini_api_key,
                  logging_obj=logging,
                  acompletion=acompletion,
                  custom_prompt_dict=custom_prompt_dict,
              )
              response = model_response
          elif custom_llm_provider == "vertex_ai":
              vertex_ai_project = litellm.vertex_project or get_secret(
                  "VERTEXAI_PROJECT"
              )
              vertex_ai_location = (
                  litellm.vertex_location
                  or get_secret("VERTEXAI_LOCATION")
              )
  
              model_response = vertex_ai.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  vertex_location=vertex_ai_location,
                  vertex_project=vertex_ai_project,
                  logging_obj=logging,
                  acompletion=acompletion,
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
                  and acompletion == False
              ):
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="vertex_ai",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "ai21":
              custom_llm_provider = "ai21"
              ai21_key = (
                  api_key
                  or litellm.ai21_key
                  or os.environ.get("AI21_API_KEY")
                  or litellm.api_key
              )
  
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("AI21_API_BASE")
                  or "https://api.ai21.com/studio/v1/"
              )
  
              model_response = ai21.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=ai21_key,
                  logging_obj=logging,
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="ai21",
                      logging_obj=logging,
                  )
                  return response
  
              ## RESPONSE OBJECT
              response = model_response
          elif custom_llm_provider == "sagemaker":
              # boto3 reads keys from .env
              model_response = sagemaker.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  custom_prompt_dict=custom_prompt_dict,
                  hf_model_name=hf_model_name,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  logging_obj=logging,
              )
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):  ## [BETA]
                  # sagemaker does not support streaming as of now so we're faking streaming:
                  # https://discuss.huggingface.co/t/streaming-output-text-when-deploying-on-sagemaker/39611
                  # "SageMaker is currently not supporting streaming responses."
  
                  # fake streaming for sagemaker
                  print_verbose(f"ENTERS SAGEMAKER CUSTOMSTREAMWRAPPER")
                  resp_string = model_response["choices"][0]["message"][
                      "content"
                  ]
                  response = CustomStreamWrapper(
                      resp_string,
                      model,
                      custom_llm_provider="sagemaker",
                      logging_obj=logging,
                  )
                  return response
  
              ## RESPONSE OBJECT
              response = model_response
          elif custom_llm_provider == "bedrock":
              # boto3 reads keys from .env
              custom_prompt_dict = (
                  custom_prompt_dict or litellm.custom_prompt_dict
              )
              response = bedrock.completion(
                  model=model,
                  messages=messages,
                  custom_prompt_dict=litellm.custom_prompt_dict,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  logging_obj=logging,
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  if "ai21" in model:
                      response = CustomStreamWrapper(
                          response,
                          model,
                          custom_llm_provider="bedrock",
                          logging_obj=logging,
                      )
                  else:
                      response = CustomStreamWrapper(
                          iter(response),
                          model,
                          custom_llm_provider="bedrock",
                          logging_obj=logging,
                      )
  
              if optional_params.get("stream", False):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=None,
                      original_response=response,
                  )
  
              ## RESPONSE OBJECT
              response = response
          elif custom_llm_provider == "vllm":
              model_response = vllm.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  logging_obj=logging,
              )
  
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):  ## [BETA]
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="vllm",
                      logging_obj=logging,
                  )
                  return response
  
              ## RESPONSE OBJECT
              response = model_response
          elif custom_llm_provider == "ollama":
              api_base = (
                  litellm.api_base
                  or api_base
                  or get_secret("OLLAMA_API_BASE")
                  or "http://localhost:11434"
              )
              custom_prompt_dict = (
                  custom_prompt_dict or litellm.custom_prompt_dict
              )
              if model in custom_prompt_dict:
                  # check if the model has a registered custom prompt
                  model_prompt_details = custom_prompt_dict[model]
                  prompt = custom_prompt(
                      role_dict=model_prompt_details["roles"],
                      initial_prompt_value=model_prompt_details[
                          "initial_prompt_value"
                      ],
                      final_prompt_value=model_prompt_details[
                          "final_prompt_value"
                      ],
                      messages=messages,
                  )
              else:
                  prompt = prompt_factory(
                      model=model,
                      messages=messages,
                      custom_llm_provider=custom_llm_provider,
                  )
                  if isinstance(prompt, dict):
                      # for multimode models - ollama/llava prompt_factory returns a dict {
                      #     "prompt": prompt,
                      #     "images": images
                      # }
                      prompt, images = (
                          prompt["prompt"],
                          prompt["images"],
                      )
                      optional_params["images"] = images
  
              ## LOGGING
              generator = ollama.get_ollama_response(
                  api_base,
                  model,
                  prompt,
                  optional_params,
                  logging_obj=logging,
                  acompletion=acompletion,
                  model_response=model_response,
                  encoding=encoding,
              )
              if (
                  acompletion is True
                  or optional_params.get("stream", False) == True
              ):
                  return generator
  
              response = generator
          elif custom_llm_provider == "ollama_chat":
              api_base = (
                  litellm.api_base
                  or api_base
                  or get_secret("OLLAMA_API_BASE")
                  or "http://localhost:11434"
              )
  
              ## LOGGING
              generator = ollama_chat.get_ollama_response(
                  api_base,
                  model,
                  messages,
                  optional_params,
                  logging_obj=logging,
                  acompletion=acompletion,
                  model_response=model_response,
                  encoding=encoding,
              )
              if (
                  acompletion is True
                  or optional_params.get("stream", False) == True
              ):
                  return generator
  
              response = generator
          elif custom_llm_provider == "cloudflare":
              api_key = (
                  api_key
                  or litellm.cloudflare_api_key
                  or litellm.api_key
                  or get_secret("CLOUDFLARE_API_KEY")
              )
              account_id = get_secret("CLOUDFLARE_ACCOUNT_ID")
              api_base = (
                  api_base
                  or litellm.api_base
                  or get_secret("CLOUDFLARE_API_BASE")
                  or f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/"
              )
  
              custom_prompt_dict = (
                  custom_prompt_dict or litellm.custom_prompt_dict
              )
              response = cloudflare.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  custom_prompt_dict=litellm.custom_prompt_dict,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,  # for calculating input/output tokens
                  api_key=api_key,
                  logging_obj=logging,
              )
              if (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      response,
                      model,
                      custom_llm_provider="cloudflare",
                      logging_obj=logging,
                  )
  
              if (
                  optional_params.get("stream", False)
                  or acompletion == True
              ):
                  ## LOGGING
                  logging.post_call(
                      input=messages,
                      api_key=api_key,
                      original_response=response,
                  )
              response = response
          elif (
              custom_llm_provider == "baseten"
              or litellm.api_base == "https://app.baseten.co"
          ):
              custom_llm_provider = "baseten"
              baseten_key = (
                  api_key
                  or litellm.baseten_key
                  or os.environ.get("BASETEN_API_KEY")
                  or litellm.api_key
              )
  
              model_response = baseten.completion(
                  model=model,
                  messages=messages,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  api_key=baseten_key,
                  logging_obj=logging,
              )
              if inspect.isgenerator(model_response) or (
                  "stream" in optional_params
                  and optional_params["stream"] == True
              ):
                  # don't try to access stream object,
                  response = CustomStreamWrapper(
                      model_response,
                      model,
                      custom_llm_provider="baseten",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif (
              custom_llm_provider == "petals"
              or model in litellm.petals_models
          ):
              api_base = api_base or litellm.api_base
  
              custom_llm_provider = "petals"
              stream = optional_params.pop("stream", False)
              model_response = petals.completion(
                  model=model,
                  messages=messages,
                  api_base=api_base,
                  model_response=model_response,
                  print_verbose=print_verbose,
                  optional_params=optional_params,
                  litellm_params=litellm_params,
                  logger_fn=logger_fn,
                  encoding=encoding,
                  logging_obj=logging,
              )
              if stream == True:  ## [BETA]
                  # Fake streaming for petals
                  resp_string = model_response["choices"][0]["message"][
                      "content"
                  ]
                  response = CustomStreamWrapper(
                      resp_string,
                      model,
                      custom_llm_provider="petals",
                      logging_obj=logging,
                  )
                  return response
              response = model_response
          elif custom_llm_provider == "custom":
              import requests
  
              url = litellm.api_base or api_base or ""
              if url == None or url == "":
                  raise ValueError(
                      "api_base not set. Set api_base or litellm.api_base for custom endpoints"
                  )
  
              """
              assume input to custom LLM api bases follow this format:
              resp = requests.post(
                  api_base, 
                  json={
                      'model': 'meta-llama/Llama-2-13b-hf', # model name
                      'params': {
                          'prompt': ["The capital of France is P"],
                          'max_tokens': 32,
                          'temperature': 0.7,
                          'top_p': 1.0,
                          'top_k': 40,
                      }
                  }
              )
  
              """
              prompt = " ".join([message["content"] for message in messages])  # type: ignore
              resp = requests.post(
                  url,
                  json={
                      "model": model,
                      "params": {
                          "prompt": [prompt],
                          "max_tokens": max_tokens,
                          "temperature": temperature,
                          "top_p": top_p,
                          "top_k": kwargs.get("top_k", 40),
                      },
                  },
              )
              response_json = resp.json()
              """
              assume all responses from custom api_bases of this format:
              {
                  'data': [
                      {
                          'prompt': 'The capital of France is P',
                          'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'],
                          'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}],
                          'message': 'ok'
                      }
                  ]
              }
              """
              string_response = response_json["data"][0]["output"][0]
              ## RESPONSE OBJECT
              model_response["choices"][0]["message"][
                  "content"
              ] = string_response
              model_response["created"] = int(time.time())
              model_response["model"] = model
              response = model_response
          else:
              raise ValueError(
                  f"Unable to map your input to a model. Check your input - {args}"
              )
          return response
      except Exception as e:
          ## Map to OpenAI Exception
          raise exception_type(
              model=model,
              custom_llm_provider=custom_llm_provider,
              original_exception=e,
              completion_kwargs=args,
          )
  
imports: import os, openai, sys, json, inspect, uuid, datetime, threading
  from typing import Any, Literal, Union
  from functools import partial
  import dotenv, traceback, random, asyncio, time, contextvars
  from copy import deepcopy
  import httpx
  import litellm
  from litellm import client, exception_type, get_optional_params, get_litellm_params, Logging
  from litellm.utils import get_secret, CustomStreamWrapper, read_config_args, completion_with_fallbacks, get_llm_provider, get_api_key, mock_completion_streaming_obj, convert_to_model_response_object, token_counter, Usage, get_optional_params_embeddings, get_optional_params_image_gen
  from .llms import anthropic, together_ai, ai21, sagemaker, bedrock, huggingface_restapi, replicate, aleph_alpha, nlp_cloud, baseten, vllm, ollama, ollama_chat, cloudflare, cohere, petals, oobabooga, openrouter, palm, gemini, vertex_ai, maritalk
  from .llms.openai import OpenAIChatCompletion, OpenAITextCompletion
  from .llms.azure import AzureChatCompletion
  from .llms.huggingface_restapi import Huggingface
  from .llms.prompt_templates.factory import prompt_factory, custom_prompt, function_call_prompt
  import tiktoken
  from concurrent.futures import ThreadPoolExecutor
  from typing import Callable, List, Optional, Dict, Union, Mapping
  from .caching import enable_cache, disable_cache, update_cache
  from litellm.utils import get_secret, CustomStreamWrapper, TextCompletionStreamWrapper, ModelResponse, TextCompletionResponse, TextChoices, EmbeddingResponse, read_config_args, Choices, Message
label_desc: Use image node_id 2 for calling a global function with example usage: completion(model, messages, timeout, temperature, top_p, n, stream, stop, max_tokens, presence_penalty, frequency_penalty, logit_bias, user, response_format, seed, tools, tool_choice, logprobs, top_logprobs, deployment_id, functions, function_call, base_url, api_version, api_key, model_list) and returns: mock_completion, response, completion_with_fallbacks, batch_completion_models, response, response, response, response, response, response, response, response, response, response, response, generator, generator, response, response