code: def main():
      opts = parse_opts()
      # Ensure the user home directory (HOME) is valid and is an absolute directory
      if not USER_HOME or not os.path.isabs(USER_HOME):
          print(
              "[error] Cannot determine your home directory as an absolute path;",
              " ensure the $HOME environment variable is set properly.",
          )
          sys.exit(1)
  
      os.chdir(SPARK_HOME)
  
      rm_r(os.path.join(SPARK_HOME, "work"))
      rm_r(
          os.path.join(USER_HOME, ".ivy2", "local", "org.apache.spark")
      )
      rm_r(
          os.path.join(USER_HOME, ".ivy2", "cache", "org.apache.spark")
      )
  
      os.environ["CURRENT_BLOCK"] = str(ERROR_CODES["BLOCK_GENERAL"])
  
      java_exe = determine_java_executable()
  
      if not java_exe:
          print(
              "[error] Cannot find a version of `java` on the system; please",
              " install one and retry.",
          )
          sys.exit(2)
  
      # Install SparkR
      should_only_test_modules = opts.modules is not None
      test_modules = []
      if should_only_test_modules:
          str_test_modules = [
              m.strip() for m in opts.modules.split(",")
          ]
          test_modules = [
              m
              for m in modules.all_modules
              if m.name in str_test_modules
          ]
  
      if not should_only_test_modules or modules.sparkr in test_modules:
          # If tests modules are specified, we will not run R linter.
          # SparkR needs the manual SparkR installation.
          if which("R"):
              run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
          else:
              print("Cannot install SparkR as R was not found in PATH")
  
      if os.environ.get("SPARK_JENKINS"):
          # if we're on the Amplab Jenkins build servers setup variables
          # to reflect the environment settings
          build_tool = os.environ.get("SPARK_JENKINS_BUILD_TOOL", "sbt")
          scala_version = os.environ.get(
              "SPARK_JENKINS_BUILD_SCALA_PROFILE"
          )
          hadoop_version = os.environ.get(
              "SPARK_JENKINS_BUILD_PROFILE", "hadoop3"
          )
          test_env = "spark_jenkins"
      else:
          # else we're running locally or GitHub Actions.
          build_tool = "sbt"
          scala_version = os.environ.get("SCALA_PROFILE")
          hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop3")
          if "GITHUB_ACTIONS" in os.environ:
              test_env = "github_actions"
          else:
              test_env = "local"
  
      extra_profiles = get_hadoop_profiles(
          hadoop_version
      ) + get_scala_profiles(scala_version)
  
      print(
          "[info] Using build tool",
          build_tool,
          "with profiles",
          *(extra_profiles + ["under environment", test_env]),
      )
  
      changed_modules = []
      changed_files = []
      included_tags = []
      excluded_tags = []
      if should_only_test_modules:
          # We're likely in the forked repository
          is_apache_spark_ref = (
              os.environ.get("APACHE_SPARK_REF", "") != ""
          )
          # We're likely in the main repo build.
          is_github_prev_sha = (
              os.environ.get("GITHUB_PREV_SHA", "") != ""
          )
          # Otherwise, we're in either periodic job in Github Actions or somewhere else.
  
          # If we're running the tests in GitHub Actions, attempt to detect and test
          # only the affected modules.
          if test_env == "github_actions" and (
              is_apache_spark_ref or is_github_prev_sha
          ):
              if is_apache_spark_ref:
                  changed_files = (
                      identify_changed_files_from_git_commits(
                          "HEAD",
                          target_ref=os.environ["APACHE_SPARK_REF"],
                      )
                  )
              elif is_github_prev_sha:
                  changed_files = (
                      identify_changed_files_from_git_commits(
                          os.environ["GITHUB_SHA"],
                          target_ref=os.environ["GITHUB_PREV_SHA"],
                      )
                  )
  
              modules_to_test = determine_modules_to_test(
                  determine_modules_for_files(changed_files),
                  deduplicated=False,
              )
  
              if modules.root not in modules_to_test:
                  # If root module is not found, only test the intersected modules.
                  # If root module is found, just run the modules as specified initially.
                  test_modules = list(
                      set(modules_to_test).intersection(test_modules)
                  )
  
          changed_modules = test_modules
          if len(changed_modules) == 0:
              print(
                  "[info] There are no modules to test, exiting without testing."
              )
              return
  
      # If we're running the tests in Jenkins, calculate the diff from the targeted branch, and
      # detect modules to test.
      elif os.environ.get("SPARK_JENKINS_PRB"):
          target_branch = os.environ["ghprbTargetBranch"]
          changed_files = identify_changed_files_from_git_commits(
              "HEAD", target_branch=target_branch
          )
          changed_modules = determine_modules_for_files(changed_files)
          test_modules = determine_modules_to_test(changed_modules)
          excluded_tags = determine_tags_to_exclude(changed_modules)
  
      # If there is no changed module found, tests all.
      if not changed_modules:
          changed_modules = [modules.root]
      if not test_modules:
          test_modules = determine_modules_to_test(changed_modules)
  
      if opts.excluded_tags:
          excluded_tags.extend(
              [t.strip() for t in opts.excluded_tags.split(",")]
          )
      if opts.included_tags:
          included_tags.extend(
              [t.strip() for t in opts.included_tags.split(",")]
          )
  
      print(
          "[info] Found the following changed modules:",
          ", ".join(x.name for x in changed_modules),
      )
  
      # setup environment variables
      # note - the 'root' module doesn't collect environment variables for all modules. Because the
      # environment variables should not be set if a module is not changed, even if running the 'root'
      # module. So here we should use changed_modules rather than test_modules.
      test_environ = {}
      for m in changed_modules:
          test_environ.update(m.environ)
      setup_test_environ(test_environ)
  
      if scala_version is not None:
          # If not set, assume this is default and doesn't need to change.
          switch_scala_version(scala_version)
  
      should_run_java_style_checks = False
      if not should_only_test_modules:
          # license checks
          run_apache_rat_checks()
  
          # style checks
          if not changed_files or any(
              f.endswith(".scala")
              or f.endswith("scalastyle-config.xml")
              for f in changed_files
          ):
              run_scala_style_checks(extra_profiles)
          if not changed_files or any(
              f.endswith(".java")
              or f.endswith("checkstyle.xml")
              or f.endswith("checkstyle-suppressions.xml")
              for f in changed_files
          ):
              # Run SBT Checkstyle after the build to prevent a side-effect to the build.
              should_run_java_style_checks = True
          if not changed_files or any(
              f.endswith("lint-python")
              or f.endswith("tox.ini")
              or f.endswith(".py")
              for f in changed_files
          ):
              run_python_style_checks()
          if not changed_files or any(
              f.endswith(".R")
              or f.endswith("lint-r")
              or f.endswith(".lintr")
              for f in changed_files
          ):
              run_sparkr_style_checks()
  
      # determine if docs were changed and if we're inside the jenkins environment
      # note - the below commented out until *all* Jenkins workers can get the Bundler gem installed
      # if "DOCS" in changed_modules and test_env == "spark_jenkins":
      #    build_spark_documentation()
  
      if (
          any(m.should_run_build_tests for m in test_modules)
          and test_env != "spark_jenkins"
      ):
          run_build_tests()
  
      # spark build
      build_apache_spark(build_tool, extra_profiles)
  
      # backwards compatibility checks
      if build_tool == "sbt":
          # Note: compatibility tests only supported in sbt for now
          if not os.environ.get("SKIP_MIMA"):
              detect_binary_inop_with_mima(extra_profiles)
          # Since we did not build assembly/package before running dev/mima, we need to
          # do it here because the tests still rely on it; see SPARK-13294 for details.
          build_spark_assembly_sbt(
              extra_profiles, should_run_java_style_checks
          )
  
      # run the test suites
      run_scala_tests(
          build_tool,
          extra_profiles,
          test_modules,
          excluded_tags,
          included_tags,
      )
  
      modules_with_python_tests = [
          m for m in test_modules if m.python_test_goals
      ]
      if modules_with_python_tests and not os.environ.get(
          "SKIP_PYTHON"
      ):
          run_python_tests(
              modules_with_python_tests,
              opts.python_executables,
              opts.parallelism,
              with_coverage=os.environ.get("PYSPARK_CODECOV", "false")
              == "true",
          )
          run_python_packaging_tests()
      if any(
          m.should_run_r_tests for m in test_modules
      ) and not os.environ.get("SKIP_R"):
          run_sparkr_tests()
  
imports: import itertools
  from argparse import ArgumentParser
  import os
  import re
  import sys
  import subprocess
  from sparktestsupport import SPARK_HOME, USER_HOME, ERROR_CODES
  from sparktestsupport.shellutils import exit_from_command_with_retcode, run_cmd, rm_r, which
  from sparktestsupport.utils import determine_modules_for_files, determine_modules_to_test, determine_tags_to_exclude, identify_changed_files_from_git_commits
  import sparktestsupport.modules
label_desc: Use image node_id 29 for calling a global function with example usage: main() without return types