code: def submit_ml_job(
      job_mode: conf.JobMode,
      docker_args: Dict[str, Any],
      region: ct.Region,
      project_id: str,
      credentials_path: Optional[str] = None,
      dry_run: bool = False,
      job_name: Optional[str] = None,
      machine_type: Optional[ct.MachineType] = None,
      gpu_spec: Optional[ct.GPUSpec] = None,
      tpu_spec: Optional[ct.TPUSpec] = None,
      image_tag: Optional[str] = None,
      labels: Optional[Dict[str, str]] = None,
      experiment_config: Optional[ce.ExpConf] = None,
      script_args: Optional[List[str]] = None,
      request_retries: Optional[int] = None,
      xgroup: Optional[str] = None,
  ) -> None:
      """Top level function in the module. This function:
  
      - builds an image using the supplied docker_args, in either CPU or GPU mode
      - pushes that image to the Cloud Container Repository of the supplied
        project_id
      - generates a sequence of 'JobSpec' instances, one for every combination in
        the supplied experiment_config, and
      - batch-submits all jobs to AI Platform
  
      Keyword args:
  
      - job_mode: caliban.config.JobMode.
      - docker_args: these arguments are passed through to
        caliban.docker.build.build_image.
      - region: the region to use for AI Platform job submission. Different regions
        support different GPUs.
      - project_id: GCloud project ID for container storage and job submission.
      - credentials_path: explicit path to a service account JSON file, if it exists.
      - dry_run: if True, no actual jobs will be submitted and docker won't
        actually build; logging side effects will show the user what will happen
        without dry_run=True.
      - job_name: optional custom name. This is applied as a label to every job,
        and used as a prefix for all jobIds submitted to Cloud.
      - machine_type: the machine type to allocate for each job. Must be one
        supported by Cloud.
      - gpu_spec: if None and job_mode is GPU, defaults to a standard single GPU.
        Else, configures the count and type of GPUs to attach to the machine that
        runs each job.
      - tpu_spec: if None, defaults to no TPU attached. Else, configures the count
        and type of TPUs to attach to the machine that runs each job.
      - image_tag: optional explicit tag of a Container-Registry-available Docker
        container. If supplied, submit_ml_job will skip the docker build and push
        phases and use this image_tag directly.
      - labels: dictionary of KV pairs to apply to each job. User args will also be
        applied as labels, plus a few default labels supplied by Caliban.
      - experiment_config: dict of string to list, boolean, string or int. Any
        lists will trigger a cartesian product out with the rest of the config. A
        job will be submitted for every combination of parameters in the experiment
        config.
      - script_args: these are extra arguments that will be passed to every job
        executed, in addition to the arguments created by expanding out the
        experiment config.
      - request_retries: the number of times to retry each request if it fails for
        a timeout or a rate limiting request.
      - xgroup: experiment group for this submission, if None a new group will
        be created
      """
      if script_args is None:
          script_args = []
  
      if job_name is None:
          job_name = "caliban_{}".format(u.current_user())
  
      if job_mode == conf.JobMode.GPU and gpu_spec is None:
          gpu_spec = ct.GPUSpec(ct.GPU.P100, 1)
  
      if machine_type is None:
          machine_type = conf.DEFAULT_MACHINE_TYPE[job_mode]
  
      if experiment_config is None:
          experiment_config = {}
  
      if labels is None:
          labels = {}
  
      if request_retries is None:
          request_retries = 10
  
      caliban_config = docker_args.get("caliban_config", {})
  
      engine = get_mem_engine() if dry_run else get_sql_engine()
  
      with session_scope(engine) as session:
          container_spec = generate_container_spec(
              session, docker_args, image_tag
          )
  
          if image_tag is None:
              image_tag = generate_image_tag(
                  project_id, docker_args, dry_run=dry_run
              )
  
          experiments = create_experiments(
              session=session,
              container_spec=container_spec,
              script_args=script_args,
              experiment_config=experiment_config,
              xgroup=xgroup,
          )
  
          specs = build_job_specs(
              job_name=job_name,
              image_tag=image_tag,
              region=region,
              machine_type=machine_type,
              experiments=experiments,
              user_labels=labels,
              gpu_spec=gpu_spec,
              tpu_spec=tpu_spec,
              caliban_config=caliban_config,
          )
  
          if dry_run:
              return execute_dry_run(specs)
  
          try:
              submit_job_specs(
                  specs=specs,
                  project_id=project_id,
                  credentials_path=credentials_path,
                  num_specs=len(experiments),
                  request_retries=request_retries,
              )
          except Exception as e:
              logging.error(f"exception: {e}")
              logging.error(f"{traceback.format_exc()}")
              session.commit()  # commit here, otherwise will be rolled back
  
          logging.info("")
          logging.info(
              t.green(
                  "Visit {} to see the status of all jobs.".format(
                      job_url(project_id, "")
                  )
              )
          )
          logging.info("")
  
imports: from __future__ import absolute_import, division, print_function
  import datetime
  import traceback
  from copy import deepcopy
  from pprint import pformat
  from typing import Any, Dict, Iterable, List, Optional, Tuple
  import tqdm
  from absl import logging
  from blessings import Terminal
  from googleapiclient import discovery
  from googleapiclient.errors import HttpError
  import caliban.config
  import caliban.config.experiment
  import caliban.docker.build
  import caliban.docker.push
  import caliban.history.types
  import caliban.platform.cloud.types
  import caliban.platform.cloud.util
  import caliban.util
  import caliban.util.auth
  import caliban.util.metrics
  import caliban.util.tqdm
  from caliban.history.util import create_experiments, generate_container_spec, get_mem_engine, get_sql_engine, session_scope
label_desc: Use image node_id 23 for calling a global function with example usage: submit_ml_job(job_mode, docker_args, region, project_id, credentials_path, dry_run, job_name, machine_type, gpu_spec, tpu_spec, image_tag, labels, experiment_config, script_args, request_retries, xgroup) and returns: execute_dry_run