class_docstr: Helper class for running an MLflow project as a Databricks Job.
  :param databricks_profile: Optional Databricks CLI profile to use to fetch hostname &
         authentication information when making Databricks API requests.
class_docstr_tok: ["Helper","class","for","running","an","MLflow","project","as","a","Databricks","Job",".",":","param","databricks_profile",":","Optional","Databricks","CLI","profile","to","use","to","fetch","hostname","&","authentication","information","when","making","Databricks","API","requests","."]
docstr: Run the specified shell command on a Databricks cluster.
  
  :param project_uri: URI of the project from which the shell command originates.
  :param command: Shell command to run.
  :param env_vars: Environment variables to set in the process running ``command``.
  :param cluster_spec: Dictionary containing a `Databricks cluster specification
                       <https://docs.databricks.com/dev-tools/api/latest/jobs.html#clusterspec>`_
                       or a `Databricks new cluster specification
                       <https://docs.databricks.com/dev-tools/api/latest/jobs.html#jobsclusterspecnewcluster>`_
                       to use when launching a run. If you specify libraries, this function
                       will add MLflow to the library list. This function does not support
                       installation of conda environment libraries on the workers.
  :return: ID of the Databricks job run. Can be used to query the run's status via the
           Databricks
           `Runs Get <https://docs.databricks.com/api/latest/jobs.html#runs-get>`_ API.
docstr_tok: ["Run","the","specified","shell","command","on","a","Databricks","cluster",".",":","param","project_uri",":","URI","of","the","project","from","which","the","shell","command","originates",".",":","param","command",":","Shell","command","to","run",".",":","param","env_vars",":","Environment","variables","to","set","in","the","process","running","``","command","``",".",":","param","cluster_spec",":","Dictionary","containing","a","`","Databricks","cluster","specification","<","https",":","\/\/docs.databricks.com\/dev-tools\/api\/latest\/jobs.html","#","clusterspec",">","`","_","or","a","`","Databricks","new","cluster","specification","<","https",":","\/\/docs.databricks.com\/dev-tools\/api\/latest\/jobs.html","#","jobsclusterspecnewcluster",">","`","_","to","use","when","launching","a","run",".","If","you","specify","libraries",",","this","function","will","add","MLflow","to","the","library","list",".","This","function","does","not","support","installation","of","conda","environment","libraries","on","the","workers",".",":","return",":","ID","of","the","Databricks","job","run",".","Can","be","used","to","query","the","run","'s","status","via","the","Databricks","`","Runs","Get","<","https",":","\/\/docs.databricks.com\/api\/latest\/jobs.html","#","runs-get",">","`","_","API","."]
code: def _run_shell_command_job(
      self, project_uri, command, env_vars, cluster_spec
  ):
      """
      Run the specified shell command on a Databricks cluster.
  
      :param project_uri: URI of the project from which the shell command originates.
      :param command: Shell command to run.
      :param env_vars: Environment variables to set in the process running ``command``.
      :param cluster_spec: Dictionary containing a `Databricks cluster specification
                           <https://docs.databricks.com/dev-tools/api/latest/jobs.html#clusterspec>`_
                           or a `Databricks new cluster specification
                           <https://docs.databricks.com/dev-tools/api/latest/jobs.html#jobsclusterspecnewcluster>`_
                           to use when launching a run. If you specify libraries, this function
                           will add MLflow to the library list. This function does not support
                           installation of conda environment libraries on the workers.
      :return: ID of the Databricks job run. Can be used to query the run's status via the
               Databricks
               `Runs Get <https://docs.databricks.com/api/latest/jobs.html#runs-get>`_ API.
      """
      if is_release_version():
          mlflow_lib = {"pypi": {"package": f"mlflow=={VERSION}"}}
      else:
          # When running a non-release version as the client the same version will not be
          # available within Databricks.
          _logger.warning(
              "Your client is running a non-release version of MLflow. "
              "This version is not available on the databricks runtime. "
              "MLflow will fallback the MLflow version provided by the runtime. "
              "This might lead to unforeseen issues. "
          )
          mlflow_lib = {"pypi": {"package": f"'mlflow<={VERSION}'"}}
  
      # Check syntax of JSON - if it contains libraries and new_cluster, pull those out
      if "new_cluster" in cluster_spec:
          # Libraries are optional, so we don't require that this be specified
          cluster_spec_libraries = cluster_spec.get("libraries", [])
          libraries = (
              # This is for development purposes only. If the cluster spec already includes
              # an MLflow Git URI, then we don't append `mlflow_lib` to avoid having
              # two different pip requirements for mlflow.
              cluster_spec_libraries
              if _contains_mlflow_git_uri(cluster_spec_libraries)
              else cluster_spec_libraries + [mlflow_lib]
          )
          cluster_spec = cluster_spec["new_cluster"]
      else:
          libraries = [mlflow_lib]
  
      # Make jobs API request to launch run.
      req_body_json = {
          "run_name": f"MLflow Run for {project_uri}",
          "new_cluster": cluster_spec,
          "shell_command_task": {
              "command": command,
              "env_vars": env_vars,
          },
          "libraries": libraries,
      }
      _logger.info(
          "=== Submitting a run to execute the MLflow project... ==="
      )
      run_submit_res = self._jobs_runs_submit(req_body_json)
      return run_submit_res["run_id"]
  
imports: import hashlib
  import json
  import logging
  import os
  import posixpath
  import re
  import tempfile
  import textwrap
  import time
  from shlex import quote
  from mlflow import tracking
  from mlflow.entities import RunStatus
  from mlflow.environment_variables import MLFLOW_EXPERIMENT_ID, MLFLOW_TRACKING_URI
  from mlflow.exceptions import ExecutionException, MlflowException
  from mlflow.projects.submitted_run import SubmittedRun
  from mlflow.projects.utils import MLFLOW_LOCAL_BACKEND_RUN_ID_CONFIG
  from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
  from mlflow.utils import databricks_utils, file_utils, rest_utils
  from mlflow.utils.mlflow_tags import MLFLOW_DATABRICKS_RUN_URL, MLFLOW_DATABRICKS_SHELL_JOB_ID, MLFLOW_DATABRICKS_SHELL_JOB_RUN_ID, MLFLOW_DATABRICKS_WEBAPP_URL
  from mlflow.utils.uri import is_databricks_uri, is_http_uri
  from mlflow.version import VERSION, is_release_version
label_desc: Use image node_id 7 for calling the DatabricksJobRunner obj's underlying member method code with example usage: obj._run_shell_command_job(project_uri, command, env_vars, cluster_spec) and returns: run_submit_res