text: import six
  import time
  from typing import List, Dict
  
  from prefect import Task
  from prefect.utilities.tasks import defaults_from_attrs
  from prefect.exceptions import PrefectException
  
  from prefect.tasks.databricks.databricks_hook import DatabricksHook
  
  
  def _deep_string_coerce(content, json_path="json"):
      """
      Coerces content or all values of content if it is a dict to a string. The
      function will throw if content contains non-string or non-numeric types.
  
      The reason why we have this function is because the `self.json` field must be a
      dict with only string values. This is because `render_template` will fail
      for numerical values.
      """
      c = _deep_string_coerce
      if isinstance(content, six.string_types):
          return content
      elif isinstance(content, six.integer_types + (float,)):
          # Databricks can tolerate either numeric or string types in the API backend.
          return str(content)
      elif isinstance(content, (list, tuple)):
          return [c(e, "{0}[{1}]".format(json_path, i)) for i, e in enumerate(content)]
      elif isinstance(content, dict):
          return {
              k: c(v, "{0}[{1}]".format(json_path, k)) for k, v in list(content.items())
          }
      else:
          param_type = type(content)
          msg = "Type {0} used for parameter {1} is not a number or a string".format(
              param_type, json_path
          )
          raise ValueError(msg)
  
  
  def _handle_databricks_task_execution(task, hook, log):
      """
      Handles the Databricks + Prefect lifecycle logic for a Databricks task
  
      Args:
          - task (prefect.Task) : Prefect task being handled
          - hook (prefect.tasks.databricks.databricks_hook.DatabricksHook): Databricks Hook
          - log (logger): Prefect logging instance
      """
  
      log.info("Run submitted with run_id: %s", task.run_id)
      run_page_url = hook.get_run_page_url(task.run_id)
  
      log.info("Run submitted with config : %s", task.json)
  
      log.info("View run status, Spark UI, and logs at %s", run_page_url)
      while True:
          run_state = hook.get_run_state(task.run_id)
          if run_state.is_terminal:
              if run_state.is_successful:
                  log.info("%s completed successfully.", task.name)
                  log.info("View run status, Spark UI, and logs at %s", run_page_url)
                  return
              else:
                  error_message = "{t} failed with terminal state: {s}".format(
                      t=task.name, s=run_state
                  )
                  raise PrefectException(error_message)
          else:
              log.info("%s in run state: %s", task.name, run_state)
              log.info("View run status, Spark UI, and logs at %s", run_page_url)
              log.info("Sleeping for %s seconds.", task.polling_period_seconds)
              time.sleep(task.polling_period_seconds)
  
  
  class DatabricksSubmitRun(Task):
      """
      Submits a Spark job run to Databricks using the
      `api/2.0/jobs/runs/submit
      <https://docs.databricks.com/api/latest/jobs.html#runs-submit>`_
      API endpoint.
  
      There are two ways to instantiate this task.
  
      In the first way, you can take the JSON payload that you typically use
      to call the `api/2.0/jobs/runs/submit` endpoint and pass it directly
      to our `DatabricksSubmitRun` task through the `json` parameter.
      For example:
  
      ```
      json = {
          'new_cluster': {
          'spark_version': '2.1.0-db3-scala2.11',
          'num_workers': 2
          },
          'notebook_task': {
          'notebook_path': '/Users/prefect@example.com/PrepareData',
          },
      }
  
      conn = PrefectSecret('DATABRICKS_CONNECTION_STRING')
      notebook_run = DatabricksSubmitRun(json=json)
      notebook_run(databricks_conn_secret=conn)
      ```
  
      Another way to accomplish the same thing is to use the named parameters
      of the `DatabricksSubmitRun` directly. Note that there is exactly
      one named parameter for each top level parameter in the `runs/submit`
      endpoint. In this method, your code would look like this:
  
      ```
      new_cluster = {
          'spark_version': '2.1.0-db3-scala2.11',
          'num_workers': 2
      }
      notebook_task = {
          'notebook_path': '/Users/prefect@example.com/PrepareData',
      }
  
      conn = PrefectSecret('DATABRICKS_CONNECTION_STRING')
      notebook_run = DatabricksSubmitRun(
          new_cluster=new_cluster,
          notebook_task=notebook_task)
      notebook_run(databricks_conn_secret=conn)
      ```
  
      In the case where both the json parameter **AND** the named parameters
      are provided, they will be merged together. If there are conflicts during the merge,
      the named parameters will take precedence and override the top level `json` keys.
  
      This task requires a Databricks connection to be specified as a Prefect secret and can
      be passed to the task like so:
  
      ```
      from prefect.tasks.secrets import PrefectSecret
      from prefect.tasks.databricks import DatabricksSubmitRun
  
      with Flow('my flow') as flow:
          conn = PrefectSecret('DATABRICKS_CONNECTION_STRING')
          notebook_run = DatabricksSubmitRun(json=...)
          notebook_run(databricks_conn_secret=conn)
      ```
  
      Currently the named parameters that `DatabricksSubmitRun` task supports are
  
      - `spark_jar_task`
      - `notebook_task`
      - `new_cluster`
      - `existing_cluster_id`
      - `libraries`
      - `run_name`
      - `timeout_seconds`
  
      Args:
          - databricks_conn_secret (dict, optional): Dictionary representation of the Databricks Connection
              String. Structure must be a string of valid JSON. To use token based authentication, provide
              the key `token` in the string for the connection and create the key `host`.
              `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
              '{"host": "abcdef.xyz", "login": "ghijklmn", "password": "opqrst"}'`
              OR
              `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
              '{"host": "abcdef.xyz", "token": "ghijklmn"}'`
              See documentation of the `DatabricksSubmitRun` Task to see how to pass in the connection
              string using `PrefectSecret`.
          - json (dict, optional): A JSON object containing API parameters which will be passed
              directly to the `api/2.0/jobs/runs/submit` endpoint. The other named parameters
              (i.e. `spark_jar_task`, `notebook_task`..) to this task will
              be merged with this json dictionary if they are provided.
              If there are conflicts during the merge, the named parameters will
              take precedence and override the top level json keys. (templated)
              For more information about templating see :ref:`jinja-templating`.
              https://docs.databricks.com/api/latest/jobs.html#runs-submit
          - spark_jar_task (dict, optional): The main class and parameters for the JAR task. Note that
              the actual JAR is specified in the `libraries`.
              *EITHER* `spark_jar_task` *OR* `notebook_task` should be specified.
              This field will be templated.
              https://docs.databricks.com/api/latest/jobs.html#jobssparkjartask
          - notebook_task (dict, optional): The notebook path and parameters for the notebook task.
              *EITHER* `spark_jar_task` *OR* `notebook_task` should be specified.
              This field will be templated.
              https://docs.databricks.com/api/latest/jobs.html#jobsnotebooktask
          - new_cluster (dict, optional): Specs for a new cluster on which this task will be run.
              *EITHER* `new_cluster` *OR* `existing_cluster_id` should be specified.
              This field will be templated.
              https://docs.databricks.com/api/latest/jobs.html#jobsclusterspecnewcluster
          - existing_cluster_id (str, optional): ID for existing cluster on which to run this task.
              *EITHER* `new_cluster` *OR* `existing_cluster_id` should be specified.
              This field will be templated.
          - libraries (list of dicts, optional): Libraries which this run will use.
              This field will be templated.
              https://docs.databricks.com/api/latest/libraries.html#managedlibrarieslibrary
          - run_name (str, optional): The run name used for this task.
              By default this will be set to the Prefect `task_id`. This `task_id` is a
              required parameter of the superclass `Task`.
              This field will be templated.
          - timeout_seconds (int, optional): The timeout for this run. By default a value of 0 is used
              which means to have no timeout.
              This field will be templated.
          - polling_period_seconds (int, optional): Controls the rate which we poll for the result of
              this run. By default the task will poll every 30 seconds.
          - databricks_retry_limit (int, optional): Amount of times retry if the Databricks backend is
              unreachable. Its value must be greater than or equal to 1.
          - databricks_retry_delay (float, optional): Number of seconds to wait between retries (it
              might be a floating point number).
          - **kwargs (dict, optional): additional keyword arguments to pass to the
              Task constructor
      """
  
      def __init__(
          self,
          databricks_conn_secret: dict = None,
          json: dict = None,
          spark_jar_task: dict = None,
          notebook_task: dict = None,
          new_cluster: dict = None,
          existing_cluster_id: str = None,
          libraries: List[Dict] = None,
          run_name: str = None,
          timeout_seconds: int = None,
          polling_period_seconds: int = 30,
          databricks_retry_limit: int = 3,
          databricks_retry_delay: float = 1,
          **kwargs
      ) -> None:
  
          self.databricks_conn_secret = databricks_conn_secret
          self.json = json or {}
          self.spark_jar_task = spark_jar_task
          self.notebook_task = notebook_task
          self.new_cluster = new_cluster
          self.existing_cluster_id = existing_cluster_id
          self.libraries = libraries
          self.run_name = run_name
          self.timeout_seconds = timeout_seconds
          self.polling_period_seconds = polling_period_seconds
          self.databricks_retry_limit = databricks_retry_limit
          self.databricks_retry_delay = databricks_retry_delay
  
          self.run_id = None
  
          super().__init__(**kwargs)
  
      def get_hook(self):
          return DatabricksHook(
              self.databricks_conn_secret,
              retry_limit=self.databricks_retry_limit,
              retry_delay=self.databricks_retry_delay,
          )
  
      @defaults_from_attrs(
          "databricks_conn_secret",
          "json",
          "spark_jar_task",
          "notebook_task",
          "new_cluster",
          "existing_cluster_id",
          "libraries",
          "run_name",
          "timeout_seconds",
          "polling_period_seconds",
          "databricks_retry_limit",
          "databricks_retry_delay",
      )
      def run(
          self,
          databricks_conn_secret: dict = None,
          json: dict = None,
          spark_jar_task: dict = None,
          notebook_task: dict = None,
          new_cluster: dict = None,
          existing_cluster_id: str = None,
          libraries: List[Dict] = None,
          run_name: str = None,
          timeout_seconds: int = None,
          polling_period_seconds: int = 30,
          databricks_retry_limit: int = 3,
          databricks_retry_delay: float = 1,
      ) -> str:
          """
          Task run method.
  
          Args:
  
          - databricks_conn_secret (dict, optional): Dictionary representation of the Databricks Connection
              String. Structure must be a string of valid JSON. To use token based authentication, provide
              the key `token` in the string for the connection and create the key `host`.
              `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
              '{"host": "abcdef.xyz", "login": "ghijklmn", "password": "opqrst"}'`
              OR
              `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
              '{"host": "abcdef.xyz", "token": "ghijklmn"}'`
              See documentation of the `DatabricksSubmitRun` Task to see how to pass in the connection
              string using `PrefectSecret`.
          - json (dict, optional): A JSON object containing API parameters which will be passed
              directly to the `api/2.0/jobs/runs/submit` endpoint. The other named parameters
              (i.e. `spark_jar_task`, `notebook_task`..) to this task will
              be merged with this json dictionary if they are provided.
              If there are conflicts during the merge, the named parameters will
              take precedence and override the top level json keys. (templated)
              For more information about templating see :ref:`jinja-templating`.
              https://docs.databricks.com/api/latest/jobs.html#runs-submit
          - spark_jar_task (dict, optional): The main class and parameters for the JAR task. Note that
              the actual JAR is specified in the `libraries`.
              *EITHER* `spark_jar_task` *OR* `notebook_task` should be specified.
              This field will be templated.
              https://docs.databricks.com/api/latest/jobs.html#jobssparkjartask
          - notebook_task (dict, optional): The notebook path and parameters for the notebook task.
              *EITHER* `spark_jar_task` *OR* `notebook_task` should be specified.
              This field will be templated.
              https://docs.databricks.com/api/latest/jobs.html#jobsnotebooktask
          - new_cluster (dict, optional): Specs for a new cluster on which this task will be run.
              *EITHER* `new_cluster` *OR* `existing_cluster_id` should be specified.
              This field will be templated.
              https://docs.databricks.com/api/latest/jobs.html#jobsclusterspecnewcluster
          - existing_cluster_id (str, optional): ID for existing cluster on which to run this task.
              *EITHER* `new_cluster` *OR* `existing_cluster_id` should be specified.
              This field will be templated.
          - libraries (list of dicts, optional): Libraries which this run will use.
              This field will be templated.
              https://docs.databricks.com/api/latest/libraries.html#managedlibrarieslibrary
          - run_name (str, optional): The run name used for this task.
              By default this will be set to the Prefect `task_id`. This `task_id` is a
              required parameter of the superclass `Task`.
              This field will be templated.
          - timeout_seconds (int, optional): The timeout for this run. By default a value of 0 is used
              which means to have no timeout.
              This field will be templated.
          - polling_period_seconds (int, optional): Controls the rate which we poll for the result of
              this run. By default the task will poll every 30 seconds.
          - databricks_retry_limit (int, optional): Amount of times retry if the Databricks backend is
              unreachable. Its value must be greater than or equal to 1.
          - databricks_retry_delay (float, optional): Number of seconds to wait between retries (it
              might be a floating point number).
  
          Returns:
              - run_id (str): Run id of the submitted run
          """
  
          assert (
              databricks_conn_secret
          ), "A databricks connection string must be supplied as a dictionary or through Prefect Secrets"
          assert isinstance(
              databricks_conn_secret, dict
          ), "`databricks_conn_secret` must be supplied as a valid dictionary."
          self.databricks_conn_secret = databricks_conn_secret
  
          if json:
              self.json = json
  
          # Initialize Databricks Connections
          hook = self.get_hook()
  
          if spark_jar_task is not None:
              self.json["spark_jar_task"] = spark_jar_task
          if notebook_task is not None:
              self.json["notebook_task"] = notebook_task
          if new_cluster is not None:
              self.json["new_cluster"] = new_cluster
          if existing_cluster_id is not None:
              self.json["existing_cluster_id"] = existing_cluster_id
          if libraries is not None:
              self.json["libraries"] = libraries
          if run_name is not None:
              self.json["run_name"] = run_name
          if timeout_seconds is not None:
              self.json["timeout_seconds"] = timeout_seconds
          if "run_name" not in self.json:
              self.json["run_name"] = run_name or "Run Submitted by Prefect"
  
          # Validate the dictionary to a valid JSON object
          self.json = _deep_string_coerce(self.json)
  
          # Submit the job
          self.run_id = hook.submit_run(self.json)
          _handle_databricks_task_execution(self, hook, self.logger)
  
          return self.run_id
  
  
  class DatabricksRunNow(Task):
      """
      Runs an existing Spark job run to Databricks using the
      `api/2.0/jobs/run-now
      <https://docs.databricks.com/api/latest/jobs.html#run-now>`_
      API endpoint.
  
      There are two ways to instantiate this task.
  
      In the first way, you can take the JSON payload that you typically use
      to call the `api/2.0/jobs/run-now` endpoint and pass it directly
      to our `DatabricksRunNow` task through the `json` parameter.
      For example:
  
      ```
      json = {
            "job_id": 42,
            "notebook_params": {
              "dry-run": "true",
              "oldest-time-to-consider": "1457570074236"
            }
          }
  
      conn = PrefectSecret('DATABRICKS_CONNECTION_STRING')
      notebook_run = DatabricksRunNow(json=json)
      notebook_run(databricks_conn_secret=conn)
      ```
  
      Another way to accomplish the same thing is to use the named parameters
      of the `DatabricksRunNow` task directly. Note that there is exactly
      one named parameter for each top level parameter in the `run-now`
      endpoint. In this method, your code would look like this:
  
      ```
      job_id=42
  
      notebook_params = {
          "dry-run": "true",
          "oldest-time-to-consider": "1457570074236"
      }
  
      python_params = ["douglas adams", "42"]
  
      spark_submit_params = ["--class", "org.apache.spark.examples.SparkPi"]
  
      jar_params = ["john doe","35"]
  
      conn = PrefectSecret('DATABRICKS_CONNECTION_STRING')
      notebook_run = DatabricksRunNow(
          notebook_params=notebook_params,
          python_params=python_params,
          spark_submit_params=spark_submit_params,
          jar_params=jar_params
      )
      notebook_run(databricks_conn_secret=conn)
      ```
  
      In the case where both the json parameter **AND** the named parameters
      are provided, they will be merged together. If there are conflicts during the merge,
      the named parameters will take precedence and override the top level `json` keys.
  
      This task requires a Databricks connection to be specified as a Prefect secret and can
      be passed to the task like so:
  
      ```
      from prefect.tasks.secrets import PrefectSecret
      from prefect.tasks.databricks import DatabricksRunNow
  
      with Flow('my flow') as flow:
          conn = PrefectSecret('DATABRICKS_CONNECTION_STRING')
          notebook_run = DatabricksRunNow(json=...)
          notebook_run(databricks_conn_secret=conn)
      ```
  
      Currently the named parameters that `DatabricksRunNow` task supports are
  
      - `job_id`
      - `json`
      - `notebook_params`
      - `python_params`
      - `spark_submit_params`
      - `jar_params`
  
      Args:
          - databricks_conn_secret (dict, optional): Dictionary representation of the Databricks Connection
              String. Structure must be a string of valid JSON. To use token based authentication, provide
              the key `token` in the string for the connection and create the key `host`.
              `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
              '{"host": "abcdef.xyz", "login": "ghijklmn", "password": "opqrst"}'`
              OR
              `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
              '{"host": "abcdef.xyz", "token": "ghijklmn"}'`
              See documentation of the `DatabricksSubmitRun` Task to see how to pass in the connection
              string using `PrefectSecret`.
          - job_id (str, optional): The job_id of the existing Databricks job.
              https://docs.databricks.com/api/latest/jobs.html#run-now
          - json (dict, optional): A JSON object containing API parameters which will be passed
              directly to the `api/2.0/jobs/run-now` endpoint. The other named parameters
              (i.e. `notebook_params`, `spark_submit_params`..) to this operator will
              be merged with this json dictionary if they are provided.
              If there are conflicts during the merge, the named parameters will
              take precedence and override the top level json keys. (templated)
              https://docs.databricks.com/api/latest/jobs.html#run-now
          - notebook_params (dict, optional): A dict from keys to values for jobs with notebook task,
              e.g. "notebook_params": {"name": "john doe", "age":  "35"}.
              The map is passed to the notebook and will be accessible through the
              dbutils.widgets.get function. See Widgets for more information.
              If not specified upon run-now, the triggered run will use the
              job’s base parameters. notebook_params cannot be
              specified in conjunction with jar_params. The json representation
              of this field (i.e. {"notebook_params":{"name":"john doe","age":"35"}})
              cannot exceed 10,000 bytes.
              https://docs.databricks.com/user-guide/notebooks/widgets.html
          - python_params (list[str], optional): A list of parameters for jobs with python tasks,
              e.g. "python_params": ["john doe", "35"].
              The parameters will be passed to python file as command line parameters.
              If specified upon run-now, it would overwrite the parameters specified in
              job setting.
              The json representation of this field (i.e. {"python_params":["john doe","35"]})
              cannot exceed 10,000 bytes.
              https://docs.databricks.com/api/latest/jobs.html#run-now
          - spark_submit_params (list[str], optional): A list of parameters for jobs with spark submit
              task, e.g. "spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"].
              The parameters will be passed to spark-submit script as command line parameters.
              If specified upon run-now, it would overwrite the parameters specified
              in job setting.
              The json representation of this field cannot exceed 10,000 bytes.
              https://docs.databricks.com/api/latest/jobs.html#run-now
          - jar_params (list[str], optional): A list of parameters for jobs with JAR tasks,
              e.g. "jar_params": ["john doe", "35"]. The parameters will be used to invoke the main
              function of the main class specified in the Spark JAR task. If not specified upon
              run-now, it will default to an empty list. jar_params cannot be specified in conjunction
              with notebook_params. The JSON representation of this field (i.e.
              {"jar_params":["john doe","35"]}) cannot exceed 10,000 bytes.
              https://docs.databricks.com/api/latest/jobs.html#run-now
          - timeout_seconds (int, optional): The timeout for this run. By default a value of 0 is used
              which means to have no timeout.
              This field will be templated.
          - polling_period_seconds (int, optional): Controls the rate which we poll for the result of
              this run. By default the task will poll every 30 seconds.
          - databricks_retry_limit (int, optional): Amount of times retry if the Databricks backend is
              unreachable. Its value must be greater than or equal to 1.
          - databricks_retry_delay (float, optional): Number of seconds to wait between retries (it
              might be a floating point number).
          - **kwargs (dict, optional): additional keyword arguments to pass to the
              Task constructor
      """
  
      def __init__(
          self,
          databricks_conn_secret: dict = None,
          job_id: str = None,
          json: dict = None,
          notebook_params: dict = None,
          python_params: List[str] = None,
          spark_submit_params: List[str] = None,
          jar_params: List[str] = None,
          polling_period_seconds: int = 30,
          databricks_retry_limit: int = 3,
          databricks_retry_delay: float = 1,
          **kwargs
      ) -> None:
  
          self.databricks_conn_secret = databricks_conn_secret
          self.json = json or {}
          self.job_id = job_id
          self.notebook_params = notebook_params
          self.python_params = python_params
          self.spark_submit_params = spark_submit_params
          self.jar_params = jar_params
          self.polling_period_seconds = polling_period_seconds
          self.databricks_retry_limit = databricks_retry_limit
          self.databricks_retry_delay = databricks_retry_delay
  
          self.run_id = None
  
          super().__init__(**kwargs)
  
      def get_hook(self):
          return DatabricksHook(
              self.databricks_conn_secret,
              retry_limit=self.databricks_retry_limit,
              retry_delay=self.databricks_retry_delay,
          )
  
      @defaults_from_attrs(
          "databricks_conn_secret",
          "job_id",
          "json",
          "notebook_params",
          "python_params",
          "spark_submit_params",
          "jar_params",
          "polling_period_seconds",
          "databricks_retry_limit",
          "databricks_retry_delay",
      )
      def run(
          self,
          databricks_conn_secret: dict = None,
          job_id: str = None,
          json: dict = None,
          notebook_params: dict = None,
          python_params: List[str] = None,
          spark_submit_params: List[str] = None,
          jar_params: List[str] = None,
          polling_period_seconds: int = 30,
          databricks_retry_limit: int = 3,
          databricks_retry_delay: float = 1,
      ) -> str:
          """
          Task run method.
  
          Args:
  
              - databricks_conn_secret (dict, optional): Dictionary representation of the Databricks
                  Connection String. Structure must be a string of valid JSON. To use token based
                  authentication, provide the key `token` in the string for the connection and create the
                  key `host`.
                  `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
                  '{"host": "abcdef.xyz", "login": "ghijklmn", "password": "opqrst"}'`
                  OR
                  `PREFECT__CONTEXT__SECRETS__DATABRICKS_CONNECTION_STRING=
                  '{"host": "abcdef.xyz", "token": "ghijklmn"}'`
                  See documentation of the `DatabricksSubmitRun` Task to see how to pass in the connection
                  string using `PrefectSecret`.
              - job_id (str, optional): The job_id of the existing Databricks job.
                  https://docs.databricks.com/api/latest/jobs.html#run-now
              - json (dict, optional): A JSON object containing API parameters which will be passed
                  directly to the `api/2.0/jobs/run-now` endpoint. The other named parameters
                  (i.e. `notebook_params`, `spark_submit_params`..) to this operator will
                  be merged with this json dictionary if they are provided.
                  If there are conflicts during the merge, the named parameters will
                  take precedence and override the top level json keys. (templated)
                  https://docs.databricks.com/api/latest/jobs.html#run-now
              - notebook_params (dict, optional): A dict from keys to values for jobs with notebook task,
                  e.g. "notebook_params": {"name": "john doe", "age":  "35"}.
                  The map is passed to the notebook and will be accessible through the
                  dbutils.widgets.get function. See Widgets for more information.
                  If not specified upon run-now, the triggered run will use the
                  job’s base parameters. notebook_params cannot be
                  specified in conjunction with jar_params. The json representation
                  of this field (i.e. {"notebook_params":{"name":"john doe","age":"35"}})
                  cannot exceed 10,000 bytes.
                  https://docs.databricks.com/user-guide/notebooks/widgets.html
              - python_params (list[str], optional): A list of parameters for jobs with python tasks,
                  e.g. "python_params": ["john doe", "35"].
                  The parameters will be passed to python file as command line parameters.
                  If specified upon run-now, it would overwrite the parameters specified in
                  job setting.
                  The json representation of this field (i.e. {"python_params":["john doe","35"]})
                  cannot exceed 10,000 bytes.
                  https://docs.databricks.com/api/latest/jobs.html#run-now
              - spark_submit_params (list[str], optional): A list of parameters for jobs with spark submit
                  task, e.g. "spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"].
                  The parameters will be passed to spark-submit script as command line parameters.
                  If specified upon run-now, it would overwrite the parameters specified
                  in job setting.
                  The json representation of this field cannot exceed 10,000 bytes.
                  https://docs.databricks.com/api/latest/jobs.html#run-now
              - jar_params (list[str], optional): A list of parameters for jobs with JAR tasks,
                  e.g. "jar_params": ["john doe", "35"]. The parameters will be used to invoke the main
                  function of the main class specified in the Spark JAR task. If not specified upon
                  run-now, it will default to an empty list. jar_params cannot be specified in conjunction
                  with notebook_params. The JSON representation of this field (i.e.
                  {"jar_params":["john doe","35"]}) cannot exceed 10,000 bytes.
                  https://docs.databricks.com/api/latest/jobs.html#run-now
              - polling_period_seconds (int, optional): Controls the rate which we poll for the result of
                  this run. By default the task will poll every 30 seconds.
              - databricks_retry_limit (int, optional): Amount of times retry if the Databricks backend is
                  unreachable. Its value must be greater than or equal to 1.
              - databricks_retry_delay (float, optional): Number of seconds to wait between retries (it
                  might be a floating point number).
  
          Returns:
              - run_id (str): Run id of the submitted run
          """
  
          assert (
              databricks_conn_secret
          ), "A databricks connection string must be supplied as a dictionary or through Prefect Secrets"
          assert isinstance(
              databricks_conn_secret, dict
          ), "`databricks_conn_secret` must be supplied as a valid dictionary."
          self.databricks_conn_secret = databricks_conn_secret
  
          # Initialize Databricks Connections
          hook = self.get_hook()
  
          run_now_json = json or {}
  
          if job_id is not None:
              run_now_json["job_id"] = job_id
          if notebook_params is not None:
              merged = run_now_json.setdefault("notebook_params", {})
              merged.update(notebook_params)
              run_now_json["notebook_params"] = merged
          if python_params is not None:
              run_now_json["python_params"] = python_params
          if spark_submit_params is not None:
              run_now_json["spark_submit_params"] = spark_submit_params
          if jar_params is not None:
              run_now_json["jar_params"] = jar_params
  
          # Validate the dictionary to a valid JSON object
          self.json = _deep_string_coerce(run_now_json)
  
          # Submit the job
          self.run_id = hook.run_now(self.json)
          _handle_databricks_task_execution(self, hook, self.logger)
  
          return self.run_id
  