text: #***************************************************************************************
  # Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
  # Copyright (c) 2020-2021 Peng Cheng Laboratory
  #
  # XiangShan is licensed under Mulan PSL v2.
  # You can use this software according to the terms and conditions of the Mulan PSL v2.
  # You may obtain a copy of Mulan PSL v2 at:
  #          http://license.coscl.org.cn/MulanPSL2
  #
  # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
  # EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
  # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
  #
  # See the Mulan PSL v2 for more details.
  #***************************************************************************************
  
  # Simple version of xiangshan python wrapper
  
  import argparse
  import json
  import os
  import random
  import signal
  import subprocess
  import sys
  import time
  
  import psutil
  
  
  def load_all_gcpt(gcpt_path, json_path):
      all_gcpt = []
      with open(json_path) as f:
          data = json.load(f)
      for benchspec in data:
          for point in data[benchspec]:
              weight = data[benchspec][point]
              gcpt = os.path.join(gcpt_path, "_".join([benchspec, point, weight]))
              bin_dir = os.path.join(gcpt, "0")
              bin_file = list(os.listdir(bin_dir))
              assert(len(bin_file) == 1)
              bin_path = os.path.join(bin_dir, bin_file[0])
              assert(os.path.isfile(bin_path))
              all_gcpt.append(bin_path)
      return all_gcpt
  
  class XSArgs(object):
      script_path = os.path.realpath(__file__)
      # default path to the repositories
      noop_home = os.path.join(os.path.dirname(script_path), "..")
      nemu_home = os.path.join(noop_home, "../NEMU")
      am_home = os.path.join(noop_home, "../nexus-am")
      dramsim3_home = os.path.join(noop_home, "../DRAMsim3")
      rvtest_home = os.path.join(noop_home, "../riscv-tests")
      default_wave_home = os.path.join(noop_home, "build")
      wave_home   = default_wave_home
  
      def __init__(self, args):
          # all path environment variables that should be set
          all_path = [
              # (python argument, environment variable, default, target function)
              (None, "NOOP_HOME", self.noop_home, self.set_noop_home),
              (args.nemu, "NEMU_HOME", self.nemu_home, self.set_nemu_home),
              (args.am, "AM_HOME", self.am_home, self.set_am_home),
              (args.dramsim3, "DRAMSIM3_HOME", self.dramsim3_home, self.set_dramsim3_home),
              (args.rvtest, "RVTEST_HOME", self.rvtest_home, self.set_rvtest_home),
          ]
          for (arg_in, env, default, set_func) in all_path:
              set_func(self.__extract_path(arg_in, env, default))
          # Chisel arguments
          self.enable_log = args.enable_log
          self.num_cores = args.num_cores
          # Makefile arguments
          self.threads = args.threads
          self.with_dramsim3 = 1 if args.with_dramsim3 else None
          self.is_release = 1 if args.release else None
          self.is_spike = "spike" if args.spike else None
          self.trace = 1 if args.trace or not args.disable_fork  else None
          self.config = args.config
          # emu arguments
          self.max_instr = args.max_instr
          self.ram_size = args.ram_size
          self.seed = random.randint(0, 9999)
          self.numa = args.numa
          self.diff = args.diff
          if args.spike and "nemu" in args.diff:
              self.diff = self.diff.replace("nemu-interpreter", "spike")
          self.fork = not args.disable_fork
          self.disable_diff = args.no_diff
          # wave dump path
          if args.wave_dump is not None:
              self.set_wave_home(args.wave_dump)
          else:
              self.set_wave_home(self.default_wave_home)
  
      def get_env_variables(self):
          all_env = {
              "NOOP_HOME"    : self.noop_home,
              "NEMU_HOME"    : self.nemu_home,
              "WAVE_HOME"    : self.wave_home,
              "AM_HOME"      : self.am_home,
              "DRAMSIM3_HOME": self.dramsim3_home
          }
          return all_env
  
      def get_chisel_args(self, prefix=None):
          chisel_args = [
              (self.enable_log, "enable-log")
          ]
          args = map(lambda x: x[1], filter(lambda arg: arg[0], chisel_args))
          if prefix is not None:
              args = map(lambda x: prefix + x, args)
          return args
  
      def get_makefile_args(self):
          makefile_args = [
              (self.threads,       "EMU_THREADS"),
              (self.with_dramsim3, "WITH_DRAMSIM3"),
              (self.is_release,    "RELEASE"),
              (self.is_spike,      "REF"),
              (self.trace,         "EMU_TRACE"),
              (self.config,        "CONFIG"),
              (self.num_cores,     "NUM_CORES")
          ]
          args = filter(lambda arg: arg[0] is not None, makefile_args)
          return args
  
      def get_emu_args(self):
          emu_args = [
              (self.max_instr, "max-instr"),
              (self.diff,      "diff"),
              (self.seed,      "seed"),
              (self.ram_size,  "ram-size"),
          ]
          args = filter(lambda arg: arg[0] is not None, emu_args)
          return args
  
      def show(self):
          print("Extra environment variables:")
          env = self.get_env_variables()
          for env_name in env:
              print(f"{env_name}: {env[env_name]}")
          print()
          print("Chisel arguments:")
          print(" ".join(self.get_chisel_args()))
          print()
          print("Makefile arguments:")
          for val, name in self.get_makefile_args():
              print(f"{name}={val}")
          print()
          print("emu arguments:")
          for val, name in self.get_emu_args():
              print(f"--{name} {val}")
          print()
  
      def __extract_path(self, path, env=None, default=None):
          if path is None and env is not None:
              path = os.getenv(env)
          if path is None and default is not None:
              path = default
          path = os.path.realpath(path)
          return path
  
      def set_noop_home(self, path):
          self.noop_home = path
  
      def set_nemu_home(self, path):
          self.nemu_home = path
  
      def set_am_home(self, path):
          self.am_home = path
  
      def set_dramsim3_home(self, path):
          self.dramsim3_home = path
  
      def set_rvtest_home(self, path):
          self.rvtest_home = path
  
      def set_wave_home(self, path):
          print(f"set wave home to {path}")
          self.wave_home = path
  
  # XiangShan environment
  class XiangShan(object):
      def __init__(self, args):
          self.args = XSArgs(args)
          self.timeout = args.timeout
  
      def show(self):
          self.args.show()
  
      def make_clean(self):
          print("Clean up CI workspace")
          self.show()
          return_code = self.__exec_cmd(f'make -C $NOOP_HOME clean')
          return return_code
  
      def generate_verilog(self):
          print("Generating XiangShan verilog with the following configurations:")
          self.show()
          sim_args = " ".join(self.args.get_chisel_args(prefix="--"))
          make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args()))
          return_code = self.__exec_cmd(f'make -C $NOOP_HOME verilog SIM_ARGS="{sim_args}" {make_args}')
          return return_code
  
      def build_emu(self):
          print("Building XiangShan emu with the following configurations:")
          self.show()
          sim_args = " ".join(self.args.get_chisel_args(prefix="--"))
          make_args = " ".join(map(lambda arg: f"{arg[1]}={arg[0]}", self.args.get_makefile_args()))
          return_code = self.__exec_cmd(f'make -C $NOOP_HOME emu -j200 SIM_ARGS="{sim_args}" {make_args}')
          return return_code
  
      def run_emu(self, workload):
          print("Running XiangShan emu with the following configurations:")
          self.show()
          emu_args = " ".join(map(lambda arg: f"--{arg[1]} {arg[0]}", self.args.get_emu_args()))
          print("workload:", workload)
          numa_args = ""
          if self.args.numa:
              numa_info = get_free_cores(self.args.threads)
              numa_args = f"numactl -m {numa_info[0]} -C {numa_info[1]}-{numa_info[2]}"
          fork_args = "--enable-fork" if self.args.fork else ""
          diff_args = "--no-diff" if self.args.disable_diff else ""
          return_code = self.__exec_cmd(f'{numa_args} $NOOP_HOME/build/emu -i {workload} {emu_args} {fork_args} {diff_args}')
          return return_code
  
      def run(self, args):
          if args.ci is not None:
              return self.run_ci(args.ci)
          actions = [
              (args.generate, lambda _ : self.generate_verilog()),
              (args.build, lambda _ : self.build_emu()),
              (args.workload, lambda args: self.run_emu(args.workload)),
              (args.clean, lambda _ : self.make_clean())
          ]
          valid_actions = map(lambda act: act[1], filter(lambda act: act[0], actions))
          for i, action in enumerate(valid_actions):
              print(f"Action {i}:")
              ret = action(args)
              if ret:
                  return ret
          return 0
  
      def __exec_cmd(self, cmd):
          env = dict(os.environ)
          env.update(self.args.get_env_variables())
          print("subprocess call cmd:", cmd)
          start = time.time()
          proc = subprocess.Popen(cmd, shell=True, env=env, preexec_fn=os.setsid)
          try:
              return_code = proc.wait(self.timeout)
              end = time.time()
              print(f"Elapsed time: {end - start} seconds")
              return return_code
          except (KeyboardInterrupt, subprocess.TimeoutExpired):
              os.killpg(os.getpgid(proc.pid), signal.SIGINT)
              print(f"KeyboardInterrupt or TimeoutExpired.")
              return 0
  
      def __get_ci_cputest(self, name=None):
          base_dir = os.path.join(self.args.am_home, "tests/cputest/build")
          cputest = os.listdir(base_dir)
          cputest = filter(lambda x: x.endswith(".bin"), cputest)
          cputest = map(lambda x: os.path.join(base_dir, x), cputest)
          return cputest
  
      def __get_ci_rvtest(self, name=None):
          base_dir = os.path.join(self.args.rvtest_home, "isa/build")
          riscv_tests = os.listdir(base_dir)
          riscv_tests = filter(lambda x: x.endswith(".bin"), riscv_tests)
          all_rv_tests = ["rv64ui", "rv64um", "rv64ua", "rv64uf", "rv64ud"]
          riscv_tests = filter(lambda x: x[:6] in all_rv_tests, riscv_tests)
          riscv_tests = map(lambda x: os.path.join(base_dir, x), riscv_tests)
          return riscv_tests
  
      def __get_ci_misc(self, name=None):
          base_dir = "/nfs/home/share/ci-workloads"
          workloads = [
              "bitmanip/bitMisc.bin",
              "crypto/crypto-riscv64-noop.bin",
              "coremark_rv64gc_o2/coremark-riscv64-xs.bin",
              "coremark_rv64gc_o3/coremark-riscv64-xs.bin",
              "coremark_rv64gcb_o3/coremark-riscv64-xs.bin",
              "ext_intr/amtest-riscv64-xs.bin",
              "cache-alias/aliastest-riscv64-xs.bin",
              "Svinval/rv64mi-p-svinval.bin",
              "pmp/pmp.riscv.bin",
              "asid/asid.bin",
              "isa_misc/xret_clear_mprv.bin",
              "isa_misc/satp_ppn.bin",
              "cache-management/softprefetchtest-riscv64-xs.bin"
          ]
          misc_tests = map(lambda x: os.path.join(base_dir, x), workloads)
          return misc_tests
  
      def __get_ci_mc(self, name=None):
          base_dir = "/nfs/home/share/ci-workloads"
          workloads = [
              "dualcoretest/ldvio-riscv64-xs.bin"
          ]
          mc_tests = map(lambda x: os.path.join(base_dir, x), workloads)
          return mc_tests
  
      def __get_ci_nodiff(self, name=None):
          base_dir = "/nfs/home/share/ci-workloads"
          workloads = [
              "cache-management/cacheoptest-riscv64-xs.bin"
          ]
          tests = map(lambda x: os.path.join(base_dir, x), workloads)
          return tests
  
      def __am_apps_path(self, bench):
          filename = f"{bench}-riscv64-noop.bin"
          return [os.path.join(self.args.am_home, "apps", bench, "build", filename)]
  
      def __get_ci_workloads(self, name):
          workloads = {
              "linux-hello": "bbl.bin",
              "linux-hello-smp": "bbl.bin",
              "povray": "_700480000000_.gz",
              "mcf": "_17520000000_.gz",
              "xalancbmk": "_266100000000_.gz",
              "gcc": "_39720000000_.gz",
              "namd": "_434640000000_.gz",
              "milc": "_103620000000_.gz",
              "lbm": "_140840000000_.gz",
              "gromacs": "_275480000000_.gz",
              "wrf": "_1916220000000_.gz",
              "astar": "_122060000000_.gz"
          }
          if name in workloads:
              return [os.path.join("/nfs/home/share/ci-workloads", name, workloads[name])]
          # select a random SPEC checkpoint
          assert(name == "random")
          all_cpt = [
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o2_20m/take_cpt",
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o3_20m/take_cpt",
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_20m/take_cpt",
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_50m/take_cpt",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o2_20m/take_cpt",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o3_20m/take_cpt",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gc_o2_50m/take_cpt",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_speed_rv64gcb_o3_20m/take_cpt"
          ]
          all_json = [
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o2_20m/json/simpoint_summary.json",
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gcb_o3_20m/simpoint_summary.json",
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_20m/simpoint_summary.json",
              "/nfs-nvme/home/share/checkpoints_profiles/spec06_rv64gc_o2_50m/simpoint_summary.json",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o2_20m/simpoint_summary.json",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gcb_o3_20m/simpoint_summary.json",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_rv64gc_o2_50m/simpoint_summary.json",
              "/nfs-nvme/home/share/checkpoints_profiles/spec17_speed_rv64gcb_o3_20m/simpoint_summary.json"
          ]
          assert(len(all_cpt) == len(all_json))
          cpt_path, json_path = random.choice(list(zip(all_cpt, all_json)))
          all_gcpt = load_all_gcpt(cpt_path, json_path)
          return [random.choice(all_gcpt)]
  
      def run_ci(self, test):
          all_tests = {
              "cputest": self.__get_ci_cputest,
              "riscv-tests": self.__get_ci_rvtest,
              "misc-tests": self.__get_ci_misc,
              "mc-tests": self.__get_ci_mc,
              "nodiff-tests": self.__get_ci_nodiff,
              "microbench": self.__am_apps_path,
              "coremark": self.__am_apps_path
          }
          for target in all_tests.get(test, self.__get_ci_workloads)(test):
              print(target)
              ret = self.run_emu(target)
              if ret:
                  if self.args.default_wave_home != self.args.wave_home:
                      print("copy wave file to " + self.args.wave_home)
                      self.__exec_cmd(f"cp $NOOP_HOME/build/*.vcd $WAVE_HOME")
                      self.__exec_cmd(f"cp $NOOP_HOME/build/emu $WAVE_HOME")
                      self.__exec_cmd(f"cp $NOOP_HOME/build/SimTop.v $WAVE_HOME")
                  return ret
          return 0
  
  def get_free_cores(n):
      while True:
          # To avoid potential conflicts, we allow CI to use SMT.
          num_logical_core = psutil.cpu_count(logical=False)
          core_usage = psutil.cpu_percent(interval=1, percpu=True)
          num_window = num_logical_core // n
          for i in range(num_window):
              window_usage = core_usage[i * n : i * n + n]
              if sum(window_usage) < 0.3 * n and True not in map(lambda x: x > 0.5, window_usage):
                  return (((i * n) % 128)// 64, i * n, i * n + n - 1)
          print(f"No free {n} cores found. CPU usage: {core_usage}\n")
  
  if __name__ == "__main__":
      parser = argparse.ArgumentParser(description='Python wrapper for XiangShan')
      parser.add_argument('workload', nargs='?', type=str, default="",
                          help='input workload file in binary format')
      # actions
      parser.add_argument('--build', action='store_true', help='build XS emu')
      parser.add_argument('--generate', action='store_true', help='generate XS verilog')
      parser.add_argument('--ci', nargs='?', type=str, const="", help='run CI tests')
      parser.add_argument('--clean', action='store_true', help='clean up XiangShan CI workspace')
      parser.add_argument('--timeout', nargs='?', type=int, default=None, help='timeout (in seconds)')
      # environment variables
      parser.add_argument('--nemu', nargs='?', type=str, help='path to nemu')
      parser.add_argument('--am', nargs='?', type=str, help='path to nexus-am')
      parser.add_argument('--dramsim3', nargs='?', type=str, help='path to dramsim3')
      parser.add_argument('--rvtest', nargs='?', type=str, help='path to riscv-tests')
      parser.add_argument('--wave-dump', nargs='?', type=str , help='path to dump wave')
      # chisel arguments
      parser.add_argument('--enable-log', action='store_true', help='enable log')
      parser.add_argument('--num-cores', type=int, help='number of cores')
      # makefile arguments
      parser.add_argument('--release', action='store_true', help='enable release')
      parser.add_argument('--spike', action='store_true', help='enable spike diff')
      parser.add_argument('--with-dramsim3', action='store_true', help='enable dramsim3')
      parser.add_argument('--threads', nargs='?', type=int, help='number of emu threads')
      parser.add_argument('--trace', action='store_true', help='enable waveform')
      parser.add_argument('--config', nargs='?', type=str, help='config')
      # emu arguments
      parser.add_argument('--numa', action='store_true', help='use numactl')
      parser.add_argument('--diff', nargs='?', default="./ready-to-run/riscv64-nemu-interpreter-so", type=str, help='nemu so')
      parser.add_argument('--max-instr', nargs='?', type=int, help='max instr')
      parser.add_argument('--disable-fork', action='store_true', help='disable lightSSS')
      parser.add_argument('--no-diff', action='store_true', help='disable difftest')
      parser.add_argument('--ram-size', nargs='?', type=str, help='manually set simulation memory size (8GB by default)')
  
      args = parser.parse_args()
  
      xs = XiangShan(args)
      ret = xs.run(args)
  
      sys.exit(ret)
  