text: """
  This module contains platform specific methods and classes for deployment
  on vSphere platform
  """
  import glob
  import json
  import logging
  import os
  from shutil import rmtree
  import time
  
  import tempfile
  import hcl2
  import yaml
  
  from ocs_ci.deployment.helpers.vsphere_helpers import VSPHEREHELPERS
  from ocs_ci.deployment.helpers.prechecks import VSpherePreChecks
  from ocs_ci.deployment.install_ocp_on_rhel import OCPINSTALLRHEL
  from ocs_ci.deployment.ocp import OCPDeployment as BaseOCPDeployment
  from ocs_ci.deployment.terraform import Terraform
  from ocs_ci.framework import config
  from ocs_ci.ocs import constants, exceptions
  from ocs_ci.ocs.exceptions import CommandFailed, RDMDiskNotFound
  from ocs_ci.ocs.node import (
      get_node_ips,
      get_typed_worker_nodes,
      remove_nodes,
      wait_for_nodes_status,
  )
  from ocs_ci.utility import templating, version
  from ocs_ci.ocs.openshift_ops import OCP
  from ocs_ci.ocs.resources import pod
  from ocs_ci.utility.aws import AWS
  from ocs_ci.utility.bootstrap import gather_bootstrap
  from ocs_ci.utility.csr import approve_pending_csr, wait_for_all_nodes_csr_and_approve
  from ocs_ci.utility.ipam import IPAM
  from ocs_ci.utility.load_balancer import LoadBalancer
  from ocs_ci.utility.retry import retry
  from ocs_ci.utility.templating import (
      dump_data_to_json,
      Templating,
      json_to_dict,
  )
  from ocs_ci.utility.utils import (
      clone_repo,
      convert_yaml2tfvars,
      create_directory_path,
      read_file_as_str,
      replace_content_in_file,
      run_cmd,
      upload_file,
      wait_for_co,
      get_infra_id,
      get_ocp_version,
      get_openshift_installer,
      get_terraform,
      set_aws_region,
      get_terraform_ignition_provider,
      get_ocp_upgrade_history,
      add_chrony_to_ocp_deployment,
  )
  from ocs_ci.utility.vsphere import VSPHERE as VSPHEREUtil
  from semantic_version import Version
  from .deployment import Deployment
  from .flexy import FlexyVSPHEREUPI
  
  logger = logging.getLogger(__name__)
  
  
  # As of now only UPI
  __all__ = ["VSPHEREUPI", "VSPHEREIPI"]
  
  
  class VSPHEREBASE(Deployment):
  
      # default storage class for StorageCluster CRD on VmWare platform
      DEFAULT_STORAGECLASS = "thin"
  
      def __init__(self):
          """
          This would be base for both IPI and UPI deployment
          """
          super(VSPHEREBASE, self).__init__()
          self.region = config.ENV_DATA["region"]
          self.server = config.ENV_DATA["vsphere_server"]
          self.user = config.ENV_DATA["vsphere_user"]
          self.password = config.ENV_DATA["vsphere_password"]
          self.cluster = config.ENV_DATA["vsphere_cluster"]
          self.datacenter = config.ENV_DATA["vsphere_datacenter"]
          self.datastore = config.ENV_DATA["vsphere_datastore"]
          self.vsphere = VSPHEREUtil(self.server, self.user, self.password)
          self.upi_repo_path = os.path.join(constants.EXTERNAL_DIR, "installer")
          self.upi_scale_up_repo_path = os.path.join(
              constants.EXTERNAL_DIR, "openshift-misc"
          )
          self.cluster_launcer_repo_path = os.path.join(
              constants.EXTERNAL_DIR, "v4-scaleup"
          )
          os.environ["TF_LOG"] = config.ENV_DATA.get("TF_LOG_LEVEL", "TRACE")
          os.environ["TF_LOG_PATH"] = os.path.join(
              config.ENV_DATA.get("cluster_path"), config.ENV_DATA.get("TF_LOG_FILE")
          )
  
          # pre-checks for the vSphere environment
          # skip pre-checks for destroying cluster
          teardown = config.RUN["cli_params"].get("teardown")
          if not teardown:
              vsphere_prechecks = VSpherePreChecks()
              vsphere_prechecks.get_all_checks()
  
          self.ocp_version = get_ocp_version()
          config.ENV_DATA["ocp_version"] = self.ocp_version
          config.ENV_DATA[
              "ocp_version_object"
          ] = version.get_semantic_ocp_version_from_config()
          config.ENV_DATA["version_4_9_object"] = version.VERSION_4_9
  
          self.wait_time = 90
  
      def attach_disk(self, size=100, disk_type=constants.VM_DISK_TYPE):
          """
          Add a new disk to all the workers nodes
  
          Args:
              size (int): Size of disk in GB (default: 100)
  
          """
          vms = self.vsphere.get_all_vms_in_pool(
              config.ENV_DATA.get("cluster_name"), self.datacenter, self.cluster
          )
          # Add disks to all worker nodes
          for vm in vms:
              if "compute" in vm.name:
                  self.vsphere.add_disks(
                      config.ENV_DATA.get("extra_disks", 1), vm, size, disk_type
                  )
  
      def add_nodes(self):
          """
          Add new nodes to the cluster
          """
          # create separate directory for scale-up terraform data
          scaleup_terraform_data_dir = os.path.join(
              self.cluster_path,
              constants.TERRAFORM_DATA_DIR,
              constants.SCALEUP_TERRAFORM_DATA_DIR,
          )
          create_directory_path(scaleup_terraform_data_dir)
          logger.info(f"scale-up terraform data directory: {scaleup_terraform_data_dir}")
  
          # git clone repo from openshift-misc
          clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path)
  
          # git clone repo from v4-scaleup
          clone_repo(constants.VSPHERE_CLUSTER_LAUNCHER, self.cluster_launcer_repo_path)
  
          helpers = VSPHEREHELPERS()
          helpers.modify_scaleup_repo()
  
          config.ENV_DATA["vsphere_resource_pool"] = config.ENV_DATA.get("cluster_name")
  
          # sync guest time with host
          sync_time_with_host_file = constants.SCALEUP_VSPHERE_MACHINE_CONF
          if config.ENV_DATA["folder_structure"]:
              sync_time_with_host_file = os.path.join(
                  constants.CLUSTER_LAUNCHER_VSPHERE_DIR,
                  f"aos-{get_ocp_version(seperator='_')}",
                  constants.CLUSTER_LAUNCHER_MACHINE_CONF,
              )
          if config.ENV_DATA.get("sync_time_with_host"):
              sync_time_with_host(sync_time_with_host_file, True)
  
          # get the RHCOS worker list
          rhcos_ips = get_node_ips()
          logger.info(f"RHCOS IP's: {json.dumps(rhcos_ips)}")
  
          # generate terraform variable for scaling nodes
          self.scale_up_terraform_var = helpers.generate_terraform_vars_for_scaleup(
              rhcos_ips
          )
  
          # choose the vsphere_dir based on OCP version
          # generate cluster_info and config yaml files
          # for OCP version greater than 4.4
          vsphere_dir = constants.SCALEUP_VSPHERE_DIR
          rhel_module = "rhel-worker"
          if Version.coerce(self.ocp_version) >= Version.coerce("4.5"):
              vsphere_dir = os.path.join(
                  constants.CLUSTER_LAUNCHER_VSPHERE_DIR,
                  f"aos-{get_ocp_version('_')}",
                  "vsphere",
              )
              helpers.generate_cluster_info()
              helpers.generate_config_yaml()
              rhel_module = "RHEL_WORKER_LIST"
  
          # Add nodes using terraform
          scaleup_terraform = Terraform(vsphere_dir)
          previous_dir = os.getcwd()
          os.chdir(scaleup_terraform_data_dir)
          scaleup_terraform.initialize()
          scaleup_terraform.apply(self.scale_up_terraform_var)
          scaleup_terraform_tfstate = os.path.join(
              scaleup_terraform_data_dir, "terraform.tfstate"
          )
          out = scaleup_terraform.output(scaleup_terraform_tfstate, rhel_module)
          if config.ENV_DATA["folder_structure"]:
              rhel_worker_nodes = out.strip().replace('"', "").split(",")
          else:
              rhel_worker_nodes = json.loads(out)["value"]
  
          logger.info(f"RHEL worker nodes: {rhel_worker_nodes}")
          os.chdir(previous_dir)
  
          # Install OCP on rhel nodes
          rhel_install = OCPINSTALLRHEL(rhel_worker_nodes)
          rhel_install.prepare_rhel_nodes()
          rhel_install.execute_ansible_playbook()
  
          # Giving some time to settle down the new nodes
          time.sleep(self.wait_time)
  
          # wait for nodes to be in READY state
          wait_for_nodes_status(timeout=300)
  
      def delete_disks(self):
          """
          Delete the extra disks from all the worker nodes
          """
          vms = self.get_compute_vms(self.datacenter, self.cluster)
          if vms:
              for vm in vms:
                  self.vsphere.remove_disks(vm)
          else:
              logger.debug("NO Resource Pool or VMs exists")
  
      def get_compute_vms(self, dc, cluster):
          """
          Gets the compute VM's from resource pool
  
          Args:
              dc (str): Datacenter name
              cluster (str): Cluster name
  
          Returns:
              list: VM instance
  
          """
          if self.vsphere.is_resource_pool_exist(
              config.ENV_DATA["cluster_name"], self.datacenter, self.cluster
          ):
              vms = self.vsphere.get_all_vms_in_pool(
                  config.ENV_DATA.get("cluster_name"), dc, cluster
              )
              return [vm for vm in vms if "compute" in vm.name or "rhel" in vm.name]
  
      def add_rdm_disks(self):
          """
          Attaches RDM disk to the compute nodes
  
          Raises:
              RDMDiskNotFound: In case there is no disks found on host
  
          """
          logger.info("Adding RDM disk to all compute nodes")
          datastore_type = self.vsphere.get_datastore_type_by_name(
              self.datastore, self.datacenter
          )
  
          compute_vms = self.get_compute_vms(self.datacenter, self.cluster)
          for vm in compute_vms:
              host = self.vsphere.get_host(vm)
              logger.info(f"{vm.name} belongs to host {host.name}")
              devices_available = self.vsphere.available_storage_devices(
                  host, datastore_type=datastore_type
              )
              if not devices_available:
                  raise RDMDiskNotFound
  
              # Erase the partition on the disk before adding to node
              device = devices_available[0]
              self.vsphere.erase_partition(host, device)
  
              # Attach RDM disk to node
              self.attach_rdm_disk(vm, device)
  
      def attach_rdm_disk(self, vm, device_name):
          """
          Attaches RDM disk to host
  
          Args:
              vm (vim.VirtualMachine): VM instance
              device_name (str): Device name to add to VM.
                  e.g:"/vmfs/devices/disks/naa.600304801b540c0125ef160f3048faba"
  
          """
          self.vsphere.add_rdm_disk(vm, device_name)
  
      def post_destroy_checks(self):
          """
          Post destroy checks on cluster
          """
          pool = config.ENV_DATA["cluster_name"]
          if self.vsphere.is_resource_pool_exist(pool, self.datacenter, self.cluster):
              logger.warning(f"Resource pool {pool} exists even after destroying cluster")
              self.vsphere.destroy_pool(pool, self.datacenter, self.cluster)
          else:
              logger.info(
                  f"Resource pool {pool} does not exist in " f"cluster {self.cluster}"
              )
  
          # destroy the folder in templates
          template_folder = get_infra_id(self.cluster_path)
          self.vsphere.destroy_folder(template_folder, self.cluster, self.datacenter)
  
          # remove .terraform directory ( this is only to reclaim space )
          terraform_plugins_dir = os.path.join(
              config.ENV_DATA["cluster_path"],
              constants.TERRAFORM_DATA_DIR,
              constants.TERRAFORM_PLUGINS_DIR,
          )
          rmtree(terraform_plugins_dir, ignore_errors=True)
  
      def check_cluster_existence(self, cluster_name_prefix):
          """
          Check cluster existence according to cluster name prefix
  
          Args:
              cluster_name_prefix (str): The cluster name prefix to look for
  
          Returns:
              bool: True if a cluster with the same name prefix already exists,
                  False otherwise
  
          """
          cluster_name_pattern = cluster_name_prefix
          rp_exist = self.vsphere.is_resource_pool_prefix_exist(
              cluster_name_pattern, self.datacenter, self.cluster
          )
          if rp_exist:
              logger.error(
                  f"Resource pool with the prefix of {cluster_name_prefix} was found"
              )
              return True
          else:
              return False
  
  
  class VSPHEREUPI(VSPHEREBASE):
      """
      A class to handle vSphere UPI specific deployment
      """
  
      def __init__(self):
          super(VSPHEREUPI, self).__init__()
          self.ipam = config.ENV_DATA.get("ipam")
          self.token = config.ENV_DATA.get("ipam_token")
          self.cidr = config.ENV_DATA.get("machine_cidr")
          self.vm_network = config.ENV_DATA.get("vm_network")
  
      class OCPDeployment(BaseOCPDeployment):
          def __init__(self):
              super(VSPHEREUPI.OCPDeployment, self).__init__()
              self.public_key = {}
              self.upi_repo_path = os.path.join(constants.EXTERNAL_DIR, "installer")
              self.previous_dir = os.getcwd()
  
              # get OCP version
              ocp_version = get_ocp_version()
  
              # create terraform_data directory
              self.terraform_data_dir = os.path.join(
                  self.cluster_path, constants.TERRAFORM_DATA_DIR
              )
              create_directory_path(self.terraform_data_dir)
  
              # Download terraform binary based on ocp version and
              # update the installer path in ENV_DATA
              # use "0.11.14" for releases below OCP 4.5
              terraform_version = config.DEPLOYMENT["terraform_version"]
              terraform_installer = get_terraform(version=terraform_version)
              config.ENV_DATA["terraform_installer"] = terraform_installer
  
              # Download terraform ignition provider
              # ignition provider dependancy from OCP 4.6
              if Version.coerce(ocp_version) >= Version.coerce("4.6"):
                  get_terraform_ignition_provider(
                      self.terraform_data_dir, version=get_ignition_provider_version()
                  )
  
              # Initialize Terraform
              self.terraform_work_dir = constants.VSPHERE_DIR
              self.terraform = Terraform(self.terraform_work_dir)
  
              self.folder_structure = False
              if Version.coerce(ocp_version) >= Version.coerce("4.5"):
                  self.folder_structure = True
                  config.ENV_DATA["folder_structure"] = self.folder_structure
  
          def deploy_prereq(self):
              """
              Pre-Requisites for vSphere UPI Deployment
              """
              super(VSPHEREUPI.OCPDeployment, self).deploy_prereq()
              # generate manifests
              self.generate_manifests()
  
              # create chrony resource
              if Version.coerce(get_ocp_version()) >= Version.coerce("4.5"):
                  add_chrony_to_ocp_deployment()
  
              # create ignitions
              self.create_ignitions()
              self.kubeconfig = os.path.join(
                  self.cluster_path, config.RUN.get("kubeconfig_location")
              )
              self.terraform_var = os.path.join(
                  config.ENV_DATA["cluster_path"],
                  constants.TERRAFORM_DATA_DIR,
                  "terraform.tfvars",
              )
  
              # git clone repo from openshift installer
              clone_openshift_installer()
  
              # comment sensitive variable as current terraform version doesn't support
              if version.get_semantic_ocp_version_from_config() >= version.VERSION_4_11:
                  comment_sensitive_var()
  
              # generate terraform variable file
              generate_terraform_vars_and_update_machine_conf()
  
              # Add shutdown_wait_timeout to VM's
              add_shutdown_wait_timeout()
  
              # sync guest time with host
              vm_file = (
                  constants.VM_MAIN
                  if self.folder_structure
                  else constants.INSTALLER_MACHINE_CONF
              )
              if config.ENV_DATA.get("sync_time_with_host"):
                  sync_time_with_host(vm_file, True)
  
          def create_config(self):
              """
              Creates the OCP deploy config for the vSphere
              """
              # Generate install-config from template
              _templating = Templating()
              ocp_install_template = (
                  f"install-config-{self.deployment_platform}-"
                  f"{self.deployment_type}.yaml.j2"
              )
              ocp_install_template_path = os.path.join(
                  "ocp-deployment", ocp_install_template
              )
              install_config_str = _templating.render_template(
                  ocp_install_template_path, config.ENV_DATA
              )
  
              # Parse the rendered YAML so that we can manipulate the object directly
              install_config_obj = yaml.safe_load(install_config_str)
              if version.get_semantic_ocp_version_from_config() >= version.VERSION_4_10:
                  install_config_obj["platform"]["vsphere"]["network"] = config.ENV_DATA[
                      "vm_network"
                  ]
              install_config_obj["pullSecret"] = self.get_pull_secret()
              install_config_obj["sshKey"] = self.get_ssh_key()
              install_config_str = yaml.safe_dump(install_config_obj)
              install_config = os.path.join(self.cluster_path, "install-config.yaml")
              with open(install_config, "w") as f:
                  f.write(install_config_str)
  
          def generate_manifests(self):
              """
              Generates manifest files
              """
              logger.info("creating manifest files for the cluster")
              run_cmd(f"{self.installer} create manifests --dir {self.cluster_path}")
  
              # remove machines and machinesets
              # Some of the manifests produced are for creating machinesets
              # and machine objects. We should remove these, because we don't
              # want to involve the machine-API operator and
              # machine-api-operator during install.
              manifest_files_path = os.path.join(self.cluster_path, "openshift")
              files_to_remove = glob.glob(
                  f"{manifest_files_path}/99_openshift-cluster-api_"
                  f"master-machines-*.yaml"
              )
              files_to_remove.extend(
                  glob.glob(
                      f"{manifest_files_path}/99_openshift-cluster-api_"
                      f"worker-machineset-*.yaml"
                  )
              )
              logger.debug(f"Removing machines and machineset files: {files_to_remove}")
              for each_file in files_to_remove:
                  os.remove(each_file)
  
          def create_ignitions(self):
              """
              Creates the ignition files
              """
              logger.info("creating ignition files for the cluster")
              run_cmd(
                  f"{self.installer} create ignition-configs "
                  f"--dir {self.cluster_path} "
              )
  
          @retry(exceptions.CommandFailed, tries=10, delay=30, backoff=1)
          def configure_storage_for_image_registry(self, kubeconfig):
              """
              Configures storage for the image registry
              """
              logger.info("configuring storage for image registry")
              patch = ' \'{"spec":{"storage":{"emptyDir":{}}}}\' '
              run_cmd(
                  f"oc --kubeconfig {kubeconfig} patch "
                  f"configs.imageregistry.operator.openshift.io "
                  f"cluster --type merge --patch {patch}"
              )
  
          def deploy(self, log_cli_level="DEBUG"):
              """
              Deployment specific to OCP cluster on this platform
  
              Args:
                  log_cli_level (str): openshift installer's log level
                      (default: "DEBUG")
  
              """
              logger.info("Deploying OCP cluster for vSphere platform")
              logger.info(f"Openshift-installer will be using loglevel:{log_cli_level}")
              os.chdir(self.terraform_data_dir)
              self.terraform.initialize()
              self.terraform.apply(self.terraform_var)
              os.chdir(self.previous_dir)
              logger.info("waiting for bootstrap to complete")
              try:
                  run_cmd(
                      f"{self.installer} wait-for bootstrap-complete "
                      f"--dir {self.cluster_path} "
                      f"--log-level {log_cli_level}",
                      timeout=3600,
                  )
              except CommandFailed as e:
                  if constants.GATHER_BOOTSTRAP_PATTERN in str(e):
                      try:
                          gather_bootstrap()
                      except Exception as ex:
                          logger.error(ex)
                  raise e
  
              if self.folder_structure:
                  # comment bootstrap module
                  comment_bootstrap_in_lb_module()
  
                  # remove bootstrap IP in load balancer and
                  # restart haproxy
                  lb = LoadBalancer()
                  lb.rename_haproxy_conf_and_reload()
                  lb.remove_boostrap_in_proxy()
                  lb.restart_haproxy()
  
              # remove bootstrap node
              if not config.DEPLOYMENT["preserve_bootstrap_node"]:
                  logger.info("removing bootstrap node")
                  os.chdir(self.terraform_data_dir)
                  if self.folder_structure:
                      self.terraform.destroy_module(
                          self.terraform_var, constants.BOOTSTRAP_MODULE
                      )
                  else:
                      self.terraform.apply(self.terraform_var, bootstrap_complete=True)
                  os.chdir(self.previous_dir)
  
              OCP.set_kubeconfig(self.kubeconfig)
  
              # wait for all nodes to generate CSR
              # From OCP version 4.4 and above, we have to approve CSR manually
              # for all the nodes
              ocp_version = get_ocp_version()
              if Version.coerce(ocp_version) >= Version.coerce("4.4"):
                  wait_for_all_nodes_csr_and_approve(timeout=1500, sleep=10)
  
              # wait for image registry to show-up
              co = "image-registry"
              wait_for_co(co)
  
              # patch image registry to null
              self.configure_storage_for_image_registry(self.kubeconfig)
  
              # wait for install to complete
              logger.info("waiting for install to complete")
              run_cmd(
                  f"{self.installer} wait-for install-complete "
                  f"--dir {self.cluster_path} "
                  f"--log-level {log_cli_level}",
                  timeout=1800,
              )
  
              # Approving CSRs here in-case if any exists
              approve_pending_csr()
  
              self.test_cluster()
  
      def deploy_ocp(self, log_cli_level="DEBUG"):
          """
          Deployment specific to OCP cluster on vSphere platform
  
          Args:
              log_cli_level (str): openshift installer's log level
                  (default: "DEBUG")
  
          """
          cluster_name_parts = config.ENV_DATA.get("cluster_name").split("-")
          prefix = cluster_name_parts[0]
          if not (
              prefix.startswith(tuple(constants.PRODUCTION_JOBS_PREFIX))
              or config.DEPLOYMENT.get("force_deploy_multiple_clusters")
          ):
              if self.check_cluster_existence(prefix):
                  raise exceptions.SameNamePrefixClusterAlreadyExistsException(
                      f"Cluster with name prefix {prefix} already exists. "
                      f"Please destroy the existing cluster for a new cluster "
                      f"deployment"
                  )
          super(VSPHEREUPI, self).deploy_ocp(log_cli_level)
          if config.ENV_DATA.get("scale_up"):
              logger.info("Adding extra nodes to cluster")
              self.add_nodes()
  
          # remove RHCOS compute nodes
          if config.ENV_DATA.get("scale_up") and not config.ENV_DATA.get("mixed_cluster"):
              rhcos_nodes = get_typed_worker_nodes()
              logger.info(
                  f"RHCOS compute nodes to delete: "
                  f"{[node.name for node in rhcos_nodes]}"
              )
              logger.info("Removing RHCOS compute nodes from a cluster")
              remove_nodes(rhcos_nodes)
  
          if config.DEPLOYMENT.get("thick_sc"):
              sc_data = templating.load_yaml(constants.VSPHERE_THICK_STORAGECLASS_YAML)
              sc_data_yaml = tempfile.NamedTemporaryFile(
                  mode="w+", prefix="storageclass", delete=False
              )
              if config.DEPLOYMENT.get("eager_zeroed_thick_sc"):
                  sc_data["parameters"]["diskformat"] = "eagerzeroedthick"
              else:
                  sc_data["parameters"]["diskformat"] = "zeroedthick"
              templating.dump_data_to_temp_yaml(sc_data, sc_data_yaml.name)
              run_cmd(f"oc create -f {sc_data_yaml.name}")
              self.DEFAULT_STORAGECLASS = "thick"
  
      def destroy_cluster(self, log_level="DEBUG"):
          """
          Destroy OCP cluster specific to vSphere UPI
  
          Args:
              log_level (str): log level openshift-installer (default: DEBUG)
  
          """
          previous_dir = os.getcwd()
  
          # Download terraform binary based on terraform version
          # in terraform.log
          terraform_log_path = os.path.join(
              config.ENV_DATA.get("cluster_path"), config.ENV_DATA.get("TF_LOG_FILE")
          )
  
          # check for terraform.log, this check is for partially
          # deployed clusters
          try:
              with open(terraform_log_path, "r") as fd:
                  logger.debug(f"Reading terraform version from {terraform_log_path}")
                  version_line = fd.readline()
                  terraform_version = version_line.split()[-1]
          except FileNotFoundError:
              logger.debug(f"{terraform_log_path} file not found")
              terraform_version = config.DEPLOYMENT["terraform_version"]
  
          terraform_installer = get_terraform(version=terraform_version)
          config.ENV_DATA["terraform_installer"] = terraform_installer
  
          # getting OCP version here since we run destroy job as
          # separate job in jenkins
          ocp_version = get_ocp_version()
          self.folder_structure = False
          if Version.coerce(ocp_version) >= Version.coerce("4.5"):
              set_aws_region()
              self.folder_structure = True
              config.ENV_DATA["folder_structure"] = self.folder_structure
  
          # delete the extra disks
          self.delete_disks()
  
          # check whether cluster has scale-up nodes
          scale_up_terraform_data_dir = os.path.join(
              self.cluster_path,
              constants.TERRAFORM_DATA_DIR,
              constants.SCALEUP_TERRAFORM_DATA_DIR,
          )
          scale_up_terraform_var = os.path.join(
              scale_up_terraform_data_dir, "scale_up_terraform.tfvars"
          )
          if os.path.exists(scale_up_terraform_var):
              os.chdir(scale_up_terraform_data_dir)
              self.destroy_scaleup_nodes(
                  scale_up_terraform_data_dir, scale_up_terraform_var
              )
              os.chdir(previous_dir)
  
          terraform_data_dir = os.path.join(
              self.cluster_path, constants.TERRAFORM_DATA_DIR
          )
          upi_repo_path = os.path.join(
              constants.EXTERNAL_DIR,
              "installer",
          )
          tfvars = os.path.join(
              config.ENV_DATA.get("cluster_path"),
              constants.TERRAFORM_DATA_DIR,
              constants.TERRAFORM_VARS,
          )
  
          clone_openshift_installer()
  
          # comment sensitive variable as current terraform version doesn't support
          if version.get_semantic_ocp_version_from_config() >= version.VERSION_4_11:
              comment_sensitive_var()
  
          rename_files = [constants.VSPHERE_MAIN, constants.VM_MAIN]
          for each_file in rename_files:
              if os.path.exists(f"{each_file}.backup") and os.path.exists(
                  f"{each_file}.json"
              ):
                  os.rename(
                      f"{each_file}.json",
                      f"{each_file}.json.backup",
                  )
  
          # change the keep_on_remove state to false
          terraform_tfstate = os.path.join(terraform_data_dir, "terraform.tfstate")
          str_to_modify = '"keep_on_remove": true,'
          target_str = '"keep_on_remove": false,'
          logger.debug(f"changing state from {str_to_modify} to {target_str}")
          replace_content_in_file(terraform_tfstate, str_to_modify, target_str)
  
          # remove csi users in case of external deployment
          if config.DEPLOYMENT["external_mode"]:
              logger.debug("deleting csi users")
              # In some cases where deployment of external cluster is failed, external tool box doesn't exist
              try:
                  toolbox = pod.get_ceph_tools_pod(skip_creating_pod=True)
                  toolbox.exec_cmd_on_pod("ceph auth del client.csi-cephfs-node")
                  toolbox.exec_cmd_on_pod("ceph auth del client.csi-cephfs-provisioner")
                  toolbox.exec_cmd_on_pod("ceph auth del client.csi-rbd-node")
                  toolbox.exec_cmd_on_pod("ceph auth del client.csi-rbd-provisioner")
              except exceptions.CephToolBoxNotFoundException:
                  logger.warning(
                      "Failed to setup the Ceph toolbox pod. Probably due to installation was not successful"
                  )
  
          # terraform initialization and destroy cluster
          terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/"))
          os.chdir(terraform_data_dir)
          if Version.coerce(ocp_version) >= Version.coerce("4.6"):
              # Download terraform ignition provider. For OCP upgrade clusters,
              # ignition provider doesn't exist, so downloading in destroy job
              # as well
              terraform_plugins_path = ".terraform/plugins/linux_amd64/"
              if version.get_semantic_ocp_version_from_config() >= version.VERSION_4_11:
                  terraform_provider_ignition_file = "terraform-provider-ignition_v2.1.2"
              else:
                  terraform_provider_ignition_file = "terraform-provider-ignition"
              terraform_ignition_provider_path = os.path.join(
                  terraform_data_dir,
                  terraform_plugins_path,
                  terraform_provider_ignition_file,
              )
  
              # check the upgrade history of cluster and checkout to the
              # original installer release. This is due to the issue of not
              # supporting terraform state of OCP 4.5 in installer
              # release of 4.6 branch. More details in
              # https://github.com/red-hat-storage/ocs-ci/issues/2941
              is_cluster_upgraded = False
              try:
                  upgrade_history = get_ocp_upgrade_history()
                  if len(upgrade_history) > 1:
                      is_cluster_upgraded = True
                      original_installed_ocp_version = upgrade_history[-1]
                      installer_release_branch = (
                          f"release-{original_installed_ocp_version[0:3]}"
                      )
                      clone_repo(
                          constants.VSPHERE_INSTALLER_REPO,
                          upi_repo_path,
                          installer_release_branch,
                      )
              except Exception as ex:
                  logger.error(ex)
  
              if not (
                  os.path.exists(terraform_ignition_provider_path) or is_cluster_upgraded
              ):
                  get_terraform_ignition_provider(
                      terraform_data_dir, version=get_ignition_provider_version()
                  )
              terraform.initialize()
          else:
              terraform.initialize(upgrade=True)
          terraform.destroy(tfvars, refresh=(not self.folder_structure))
          os.chdir(previous_dir)
  
          # post destroy checks
          self.post_destroy_checks()
  
      def destroy_scaleup_nodes(
          self, scale_up_terraform_data_dir, scale_up_terraform_var
      ):
          """
          Destroy the scale-up nodes
  
          Args:
              scale_up_terraform_data_dir (str): Path to scale-up terraform
                  data directory
              scale_up_terraform_var (str): Path to scale-up
                  terraform.tfvars file
  
          """
          clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path)
          # git clone repo from v4-scaleup
          clone_repo(constants.VSPHERE_CLUSTER_LAUNCHER, self.cluster_launcer_repo_path)
  
          # modify scale-up repo
          helpers = VSPHEREHELPERS()
          helpers.modify_scaleup_repo()
  
          vsphere_dir = constants.SCALEUP_VSPHERE_DIR
          if Version.coerce(self.ocp_version) >= Version.coerce("4.5"):
              vsphere_dir = os.path.join(
                  constants.CLUSTER_LAUNCHER_VSPHERE_DIR,
                  f"aos-{get_ocp_version('_')}",
                  "vsphere",
              )
  
          terraform_scale_up = Terraform(vsphere_dir)
          os.chdir(scale_up_terraform_data_dir)
          terraform_scale_up.initialize(upgrade=True)
          terraform_scale_up.destroy(scale_up_terraform_var)
  
  
  class VSPHEREIPI(VSPHEREBASE):
      """
      A class to handle vSphere IPI specific deployment
      """
  
      def __init__(self):
          super(VSPHEREIPI, self).__init__()
  
      class OCPDeployment(BaseOCPDeployment):
          def __init__(self):
              super(VSPHEREIPI.OCPDeployment, self).__init__()
  
          def deploy_prereq(self):
              """
              Overriding deploy_prereq from parent. Perform all necessary
              prerequisites for VSPHEREIPI here.
              """
              #  Assign IPs from IPAM server
              ips = assign_ips(constants.NUM_OF_VIPS)
              config.ENV_DATA["vips"] = ips
  
              super(VSPHEREIPI.OCPDeployment, self).deploy_prereq()
  
              # create DNS records
              create_dns_records(ips)
  
          def create_config(self):
              """
              Creates the OCP deploy config for the vSphere
              """
              # Generate install-config from template
              _templating = Templating()
              ocp_install_template = (
                  f"install-config-{self.deployment_platform}-"
                  f"{self.deployment_type}.yaml.j2"
              )
              ocp_install_template_path = os.path.join(
                  "ocp-deployment", ocp_install_template
              )
              install_config_str = _templating.render_template(
                  ocp_install_template_path, config.ENV_DATA
              )
              install_config_obj = yaml.safe_load(install_config_str)
              install_config_obj["pullSecret"] = self.get_pull_secret()
              install_config_obj["sshKey"] = self.get_ssh_key()
              install_config_obj["platform"]["vsphere"]["apiVIP"] = config.ENV_DATA[
                  "vips"
              ][0]
              install_config_obj["platform"]["vsphere"]["ingressVIP"] = config.ENV_DATA[
                  "vips"
              ][1]
              install_config_obj["metadata"]["name"] = config.ENV_DATA.get("cluster_name")
              install_config_obj["baseDomain"] = config.ENV_DATA.get("base_domain")
              install_config_str = yaml.safe_dump(install_config_obj)
              install_config = os.path.join(self.cluster_path, "install-config.yaml")
  
              with open(install_config, "w") as f:
                  f.write(install_config_str)
  
          def deploy(self, log_cli_level="DEBUG"):
              """
              Deployment specific to OCP cluster on this platform
  
              Args:
                  log_cli_level (str): openshift installer's log level
                      (default: "DEBUG")
              """
  
              logger.info("Deploying OCP cluster")
              logger.info(f"Openshift-installer will be using loglevel:{log_cli_level}")
              try:
                  run_cmd(
                      f"{self.installer} create cluster "
                      f"--dir {self.cluster_path} "
                      f"--log-level {log_cli_level}",
                      timeout=3600,
                  )
              except CommandFailed as e:
                  if constants.GATHER_BOOTSTRAP_PATTERN in str(e):
                      try:
                          gather_bootstrap()
                      except Exception as ex:
                          logger.error(ex)
                  raise e
              self.test_cluster()
  
      def deploy_ocp(self, log_cli_level="DEBUG"):
          """
          Deployment specific to OCP cluster on this platform
  
          Args:
              log_cli_level (str): openshift installer's log level
                  (default: "DEBUG")
          """
          super(VSPHEREIPI, self).deploy_ocp(log_cli_level)
  
      def destroy_cluster(self, log_level="DEBUG"):
          """
          Destroy OCP cluster specific to vSphere IPI
  
          Args:
              log_level (str): log level openshift-installer (default: DEBUG)
          """
          force_download = config.DEPLOYMENT["force_download_installer"]
          installer = get_openshift_installer(
              config.DEPLOYMENT["installer_version"], force_download=force_download
          )
          try:
              run_cmd(
                  f"{installer} destroy cluster "
                  f"--dir {self.cluster_path} "
                  f"--log-level {log_level}",
                  timeout=3600,
              )
          except CommandFailed as e:
              logger.error(e)
  
          # Delete DNS records
          delete_dns_records()
  
          # release the IP's
          ipam = IPAM(appiapp="address")
          hosts = [
              f"{config.ENV_DATA.get('cluster_name')}-{i}"
              for i in range(constants.NUM_OF_VIPS)
          ]
          ipam.release_ips(hosts)
  
  
  class VSPHEREUPIFlexy(VSPHEREBASE):
      """
      A class to handle vSphere UPI Flexy deployment
      """
  
      def __init__(self):
          super().__init__()
  
      class OCPDeployment(BaseOCPDeployment):
          def __init__(self):
              self.flexy_deployment = True
              super().__init__()
              self.flexy_instance = FlexyVSPHEREUPI()
  
              # create terraform_data directory (used for compatibility with rest
              # of the ocs-ci)
              self.terraform_data_dir = os.path.join(
                  self.cluster_path, constants.TERRAFORM_DATA_DIR
              )
              create_directory_path(self.terraform_data_dir)
  
          def deploy_prereq(self):
              """
              Instantiate proper flexy class here
  
              """
              super().deploy_prereq()
              self.flexy_instance.deploy_prereq()
  
          def deploy(self, log_level=""):
              """
              Deployment specific to OCP cluster on this platform
  
              Args:
                  log_cli_level (str): openshift installer's log level
                      (default: "DEBUG")
  
              """
              self.flexy_instance.deploy(log_level)
              self.test_cluster()
  
          def destroy(self, log_level=""):
              """
              Destroy cluster using Flexy
              """
              self.flexy_instance.destroy()
  
      def destroy_cluster(self, log_level="DEBUG"):
          """
          Destroy OCP cluster specific to vSphere UPI Flexy
  
          Args:
              log_level (str): log level openshift-installer (default: DEBUG)
  
          """
          super().destroy_cluster(log_level)
  
  
  def change_vm_root_disk_size(machine_file):
      """
      Change the root disk size of VM from constants.CURRENT_VM_ROOT_DISK_SIZE
      to constants.VM_ROOT_DISK_SIZE
  
      Args:
           machine_file (str): machine file to change the disk size
      """
      disk_size_prefix = "size             = "
      current_vm_root_disk_size = (
          f"{disk_size_prefix}{constants.CURRENT_VM_ROOT_DISK_SIZE}"
      )
      vm_root_disk_size = f"{disk_size_prefix}{constants.VM_ROOT_DISK_SIZE}"
      replace_content_in_file(machine_file, current_vm_root_disk_size, vm_root_disk_size)
  
  
  def sync_time_with_host(machine_file, enable=False):
      """
      Syncs the guest time with host
  
      Args:
           machine_file (str): machine file to sync the guest time with host
           enable (bool): True to sync guest time with host
  
      """
      # terraform will support only lowercase bool
      enable = str(enable).lower()
      to_change = 'enable_disk_uuid = "true"'
      sync_time = f'{to_change}\n sync_time_with_host = "{enable}"'
  
      replace_content_in_file(machine_file, to_change, sync_time)
  
  
  def clone_openshift_installer():
      """
      Clone the openshift installer repo
      """
      # git clone repo from openshift installer
      # installer ( https://github.com/openshift/installer ) master and
      # other branches (greater than release-4.3) structure has been
      # changed. Use appropriate branch when ocs-ci is ready
      # with the changes.
      # Note: Currently use release-4.3 branch for the ocp versions
      # which is greater than 4.3
      upi_repo_path = os.path.join(constants.EXTERNAL_DIR, "installer")
      ocp_version = get_ocp_version()
      # supporting folder structure from ocp4.5
      if Version.coerce(ocp_version) >= Version.coerce("4.5"):
          clone_repo(
              constants.VSPHERE_INSTALLER_REPO, upi_repo_path, f"release-{ocp_version}"
          )
      elif Version.coerce(ocp_version) == Version.coerce("4.4"):
          clone_repo(
              constants.VSPHERE_INSTALLER_REPO,
              upi_repo_path,
              constants.VSPHERE_INSTALLER_BRANCH,
          )
      else:
          clone_repo(
              constants.VSPHERE_INSTALLER_REPO, upi_repo_path, f"release-{ocp_version}"
          )
  
  
  def change_mem_and_cpu():
      """
      Increase CPUs and memory for nodes
      """
      worker_num_cpus = config.ENV_DATA.get("worker_num_cpus")
      master_num_cpus = config.ENV_DATA.get("master_num_cpus")
      worker_memory = config.ENV_DATA.get("compute_memory")
      master_memory = config.ENV_DATA.get("master_memory")
      if worker_num_cpus or master_num_cpus or master_memory or worker_memory:
          with open(constants.VSPHERE_MAIN, "r") as fd:
              obj = hcl2.load(fd)
              if worker_num_cpus:
                  obj["module"]["compute"]["num_cpu"] = worker_num_cpus
              if master_num_cpus:
                  obj["module"]["control_plane"]["num_cpu"] = master_num_cpus
              if worker_memory:
                  obj["module"]["compute"]["memory"] = worker_memory
              if master_memory:
                  obj["module"]["control_plane"]["memory"] = master_memory
          # Dump data to json file since hcl module
          # doesn't support dumping of data in HCL format
          dump_data_to_json(obj, f"{constants.VSPHERE_MAIN}.json")
          os.rename(constants.VSPHERE_MAIN, f"{constants.VSPHERE_MAIN}.backup")
  
  
  def update_gw(str_to_replace, config_file):
      """
      Updates the gateway
  
      Args:
          str_to_replace (str): string to replace in config file
          config_file (str): file to replace the string
  
      """
      # update gateway
      if config.ENV_DATA.get("gateway"):
          replace_content_in_file(
              config_file, str_to_replace, f"{config.ENV_DATA.get('gateway')}"
          )
  
  
  def add_shutdown_wait_timeout():
      """
      Add shutdown_wait_timeout to VM's
  
      shutdown_wait_timeout is the amount of time, in minutes, to wait for a graceful guest shutdown
      when making necessary updates to the virtual machine. If force_power_off is set to true, the VM will be
      force powered-off after this timeout, otherwise an error is returned. Default: 3 minutes.
  
      """
      with open(constants.VM_MAIN, "r") as fd:
          obj = hcl2.load(fd)
          obj["resource"][0]["vsphere_virtual_machine"]["vm"][
              "shutdown_wait_timeout"
          ] = 10
      dump_data_to_json(obj, f"{constants.VM_MAIN}.json")
      os.rename(constants.VM_MAIN, f"{constants.VM_MAIN}.backup")
  
  
  def update_dns():
      """
      Updates the DNS
      """
      # update DNS
      if config.ENV_DATA.get("dns"):
          replace_content_in_file(
              constants.INSTALLER_IGNITION,
              constants.INSTALLER_DEFAULT_DNS,
              f"{config.ENV_DATA.get('dns')}",
          )
  
  
  def update_zone():
      """
      Updates the zone in constants.INSTALLER_ROUTE53
      """
      # update the zone in route
      if config.ENV_DATA.get("region"):
          def_zone = 'provider "aws" { region = "%s" } \n' % config.ENV_DATA.get("region")
          replace_content_in_file(constants.INSTALLER_ROUTE53, "xyz", def_zone)
  
  
  def update_path():
      """
      Updates Path to var.folder in resource vsphere_folder
      """
      logger.debug(f"Updating path to var.folder in {constants.VSPHERE_MAIN}")
      replace_str = "path          = var.cluster_id"
      replace_content_in_file(
          constants.VSPHERE_MAIN, replace_str, "path          = var.folder"
      )
  
  
  def add_var_folder():
      """
      Add folder variable to vsphere variables.tf
      """
      # read the variables.tf data to var_data
      with open(constants.VSPHERE_VAR, "r") as fd:
          var_data = fd.read()
  
      # backup the variables.tf
      os.rename(constants.VSPHERE_VAR, f"{constants.VSPHERE_VAR}.backup")
  
      # write var_data along with folder variable at the end of file
      with open(constants.VSPHERE_VAR, "w+") as fd:
          fd.write(var_data)
          fd.write('\nvariable "folder" {\n')
          fd.write("  type    = string\n")
          fd.write("}\n")
  
  
  def update_machine_conf(folder_structure=True):
      """
      Updates the machine configurations
  
      Args:
          folder_structure (bool): True if folder structure installations.
              Currently True for OCP release greater than 4.4 versions
  
      """
      if not folder_structure:
          gw_string = "${cidrhost(var.machine_cidr,1)}"
          gw_conf_file = constants.INSTALLER_IGNITION
          disk_size_conf_file = constants.INSTALLER_MACHINE_CONF
          # update dns
          update_dns()
  
          # update the zone in route
          update_zone()
  
          # increase CPUs and memory
          change_mem_and_cpu()
  
      else:
          if Version.coerce(get_ocp_version()) >= Version.coerce("4.6"):
              gw_string = "${cidrhost(var.machine_cidr, 1)}"
              gw_conf_file = constants.VM_MAIN
          else:
              gw_string = "${cidrhost(machine_cidr, 1)}"
              gw_conf_file = constants.VM_IFCFG
  
          disk_size_conf_file = constants.VM_MAIN
  
          # change cluster ID to folder
          update_path()
  
          # Add variable folder to variables.tf
          add_var_folder()
  
      # update gateway
      update_gw(gw_string, gw_conf_file)
  
      # change root disk size
      change_vm_root_disk_size(disk_size_conf_file)
  
  
  def generate_terraform_vars_and_update_machine_conf():
      """
      Generates the terraform.tfvars file
      """
      ocp_version = get_ocp_version()
      folder_structure = False
      if Version.coerce(ocp_version) >= Version.coerce("4.5"):
  
          folder_structure = True
          # export AWS_REGION
          set_aws_region()
  
          # generate terraform variable file
          generate_terraform_vars_with_folder()
  
          # update the machine configurations
          update_machine_conf(folder_structure)
  
          if Version.coerce(ocp_version) >= Version.coerce("4.5"):
              modify_haproxyservice()
      else:
          # generate terraform variable file
          generate_terraform_vars_with_out_folder()
  
          # update the machine configurations
          update_machine_conf(folder_structure)
  
  
  def generate_terraform_vars_with_folder():
      """
      Generates the terraform.tfvars file which includes folder structure
      """
      # generate terraform variables from template
      logger.info("Generating terraform variables with folder structure")
      cluster_domain = (
          f"{config.ENV_DATA.get('cluster_name')}."
          f"{config.ENV_DATA.get('base_domain')}"
      )
      config.ENV_DATA["cluster_domain"] = cluster_domain
  
      # Form the ignition paths
      bootstrap_ignition_path = os.path.join(
          config.ENV_DATA["cluster_path"], constants.BOOTSTRAP_IGN
      )
      control_plane_ignition_path = os.path.join(
          config.ENV_DATA["cluster_path"], constants.MASTER_IGN
      )
      compute_ignition_path = os.path.join(
          config.ENV_DATA["cluster_path"], constants.WORKER_IGN
      )
  
      # Update ignition paths to ENV_DATA
      config.ENV_DATA["bootstrap_ignition_path"] = bootstrap_ignition_path
      config.ENV_DATA["control_plane_ignition_path"] = control_plane_ignition_path
      config.ENV_DATA["compute_ignition_path"] = compute_ignition_path
  
      # Copy DNS address to vm_dns_addresses
      config.ENV_DATA["vm_dns_addresses"] = config.ENV_DATA["dns"]
  
      # Get the infra ID from metadata.json and update in ENV_DATA
      metadata_path = os.path.join(config.ENV_DATA["cluster_path"], "metadata.json")
      metadata_dct = json_to_dict(metadata_path)
      config.ENV_DATA["folder"] = metadata_dct["infraID"]
  
      # expand ssh_public_key_path and update in ENV_DATA
      ssh_public_key_path = os.path.expanduser(config.DEPLOYMENT["ssh_key"])
      config.ENV_DATA["ssh_public_key_path"] = ssh_public_key_path
  
      # overwrite RHCOS template
      # This use-case is mainly used for early RHCOS testing
      if config.ENV_DATA.get("vm_template_overwrite"):
          config.ENV_DATA["vm_template"] = config.ENV_DATA["vm_template_overwrite"]
  
      create_terraform_var_file("terraform_4_5.tfvars.j2")
  
  
  def create_terraform_var_file(terraform_var_template):
      """
      Creates the terraform variable file from jinja template
  
      Args:
          terraform_var_template (str): terraform template in jinja format
  
      """
      _templating = Templating()
      terraform_var_template_path = os.path.join("ocp-deployment", terraform_var_template)
      terraform_config_str = _templating.render_template(
          terraform_var_template_path, config.ENV_DATA
      )
  
      terraform_var_yaml = os.path.join(
          config.ENV_DATA["cluster_path"],
          constants.TERRAFORM_DATA_DIR,
          "terraform.tfvars.yaml",
      )
      with open(terraform_var_yaml, "w") as f:
          f.write(terraform_config_str)
  
      convert_yaml2tfvars(terraform_var_yaml)
  
  
  def generate_terraform_vars_with_out_folder():
      """
      Generates the normal ( old structure ) terraform.tfvars file
      """
      logger.info("Generating terraform variables without folder structure")
  
      # upload bootstrap ignition to public access server
      bootstrap_path = os.path.join(
          config.ENV_DATA.get("cluster_path"), constants.BOOTSTRAP_IGN
      )
      remote_path = os.path.join(
          config.ENV_DATA.get("path_to_upload"),
          f"{config.RUN.get('run_id')}_{constants.BOOTSTRAP_IGN}",
      )
      upload_file(
          config.ENV_DATA.get("httpd_server"),
          bootstrap_path,
          remote_path,
          config.ENV_DATA.get("httpd_server_user"),
          config.ENV_DATA.get("httpd_server_password"),
      )
  
      # generate bootstrap ignition url
      path_to_bootstrap_on_remote = remote_path.replace("/var/www/html/", "")
      bootstrap_ignition_url = (
          f"http://{config.ENV_DATA.get('httpd_server')}/"
          f"{path_to_bootstrap_on_remote}"
      )
      logger.info(f"bootstrap_ignition_url: {bootstrap_ignition_url}")
      config.ENV_DATA["bootstrap_ignition_url"] = bootstrap_ignition_url
  
      # load master and worker ignitions to variables
      master_ignition_path = os.path.join(
          config.ENV_DATA.get("cluster_path"), constants.MASTER_IGN
      )
      master_ignition = read_file_as_str(f"{master_ignition_path}")
      config.ENV_DATA["control_plane_ignition"] = master_ignition
  
      worker_ignition_path = os.path.join(
          config.ENV_DATA.get("cluster_path"), constants.WORKER_IGN
      )
      worker_ignition = read_file_as_str(f"{worker_ignition_path}")
      config.ENV_DATA["compute_ignition"] = worker_ignition
  
      cluster_domain = (
          f"{config.ENV_DATA.get('cluster_name')}."
          f"{config.ENV_DATA.get('base_domain')}"
      )
      config.ENV_DATA["cluster_domain"] = cluster_domain
  
      # generate terraform variables from template
      create_terraform_var_file("terraform.tfvars.j2")
  
  
  def comment_bootstrap_in_lb_module():
      """
      Commenting the bootstrap module in vsphere main.tf
      """
      logger.debug(f"Commenting bootstrap module in {constants.VSPHERE_MAIN}")
      replace_str = "module.ipam_bootstrap.ip_addresses[0]"
      replace_content_in_file(constants.VSPHERE_MAIN, replace_str, f"//{replace_str}")
  
  
  def modify_haproxyservice():
      """
      Add ExecStop in haproxy service
      """
      to_change = "TimeoutStartSec=0"
      execstop = f"{to_change}\nExecStop=/bin/podman rm -f haproxy"
  
      replace_content_in_file(constants.TERRAFORM_HAPROXY_SERVICE, to_change, execstop)
  
  
  def assign_ips(num_of_vips):
      """
      Assign IPs to hosts
  
      Args:
          num_of_vips (int): Number of IPs to assign
  
      """
      ipam = IPAM(appiapp="address")
      subnet = config.ENV_DATA["machine_cidr"].split("/")[0]
      hosts = [f"{config.ENV_DATA.get('cluster_name')}-{i}" for i in range(num_of_vips)]
      ips = ipam.assign_ips(hosts, subnet)
      logger.debug(f"IPs reserved for hosts {hosts} are {ips}")
      return ips
  
  
  def create_dns_records(ips):
      """
      Create DNS records
  
      Args:
          ips (list): List if IPs for creating DNS records
  
      """
      logger.info("creating DNS records")
      aws = AWS()
      dns_record_names = [
          f"api.{config.ENV_DATA.get('cluster_name')}",
          f"*.apps.{config.ENV_DATA.get('cluster_name')}",
      ]
      responses = []
      dns_record_mapping = {}
      for index, record in enumerate(dns_record_names):
          dns_record_mapping[record] = ips[index]
      logger.debug(f"dns_record_mapping: {dns_record_mapping}")
      zone_id = aws.get_hosted_zone_id_for_domain(config.ENV_DATA["base_domain"])
  
      for record in dns_record_names:
          responses.append(
              aws.update_hosted_zone_record(
                  zone_id=zone_id,
                  record_name=record,
                  data=dns_record_mapping[record],
                  type="A",
                  operation_type="Add",
              )
          )
  
      # wait for records to create
      logger.info("Waiting for record response")
      aws.wait_for_record_set(response_list=responses)
      logger.info("Records created successfully")
  
  
  def delete_dns_records():
      """
      Deletes DNS records
      """
      logger.info("Deleting DNS records")
      aws = AWS()
      cluster_domain = (
          f"{config.ENV_DATA.get('cluster_name')}."
          f"{config.ENV_DATA.get('base_domain')}"
      )
      # get the record sets
      record_sets = aws.get_record_sets()
  
      # form the record sets to delete
      records_to_delete = [
          f"api.{cluster_domain}.",
          f"\\052.apps.{cluster_domain}.",
      ]
  
      # delete the records
      hosted_zone_id = aws.get_hosted_zone_id_for_domain()
      logger.info(f"hosted zone id: {hosted_zone_id}")
      for record in record_sets:
          if record["Name"] in records_to_delete:
              aws.delete_record(record, hosted_zone_id)
  
  
  def get_ignition_provider_version():
      """
      Gets the ignition provider version based on OCP version
  
      Returns:
          str: ignition provider version
  
      """
      if version.get_semantic_ocp_version_from_config() >= version.VERSION_4_11:
          return "v2.1.2"
      else:
          return "v2.1.0"
  
  
  def comment_sensitive_var():
      """
      Comment out sensitive var in vm/variables.tf
      """
      str_to_modify = "sensitive = true"
      target_str = "//sensitive = true"
      logger.debug(
          f"commenting out {str_to_modify} in {constants.VM_VAR} as current terraform version doesn't support"
      )
      replace_content_in_file(constants.VM_VAR, str_to_modify, target_str)
  