text: /**
   * Copyright (C) 2019-2020 Xilinx, Inc
   *
   * Licensed under the Apache License, Version 2.0 (the "License"). You may
   * not use this file except in compliance with the License. A copy of the
   * License is located at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   * License for the specific language governing permissions and limitations
   * under the License.
   */
  
  // ------ I N C L U D E   F I L E S -------------------------------------------
  // Local - Include Files
  #include "SubCmdValidate.h"
  #include "tools/common/Report.h"
  #include "tools/common/ReportHost.h"
  #include "tools/common/XBUtilities.h"
  #include "tools/common/XBHelpMenus.h"
  #include "core/tools/common/ProgressBar.h"
  #include "core/tools/common/EscapeCodes.h"
  #include "core/tools/common/Process.h"
  #include "core/common/query_requests.h"
  #include "core/pcie/common/dmatest.h"
  namespace XBU = XBUtilities;
  
  // 3rd Party Library - Include Files
  #include <boost/program_options.hpp>
  #include <boost/property_tree/json_parser.hpp>
  #include <boost/range/iterator_range.hpp>
  #include <boost/filesystem.hpp>
  #include <boost/format.hpp>
  namespace po = boost::program_options;
  
  // System - Include Files
  #include <iostream>
  #include <sstream>
  #include <thread>
  #include <regex>
  #ifdef __GNUC__
  #include <sys/mman.h> //munmap
  #endif
  
  
  // =============================================================================
  
  // ------ L O C A L   F U N C T I O N S ---------------------------------------
  
  namespace {
  
  enum class test_status
  {
    passed,
    warning,
    failed
  };
  
  /*
   * xclbin locking
   */
  struct xclbin_lock
  {
    xclDeviceHandle m_handle;
    xuid_t m_uuid;
  
    xclbin_lock(std::shared_ptr<xrt_core::device> _dev)
      : m_handle(_dev->get_device_handle())
    {
      auto xclbinid = xrt_core::device_query<xrt_core::query::xclbin_uuid>(_dev);
  
      uuid_parse(xclbinid.c_str(), m_uuid);
  
      if (uuid_is_null(m_uuid))
        throw std::runtime_error("'uuid' invalid, please re-program xclbin.");
  
      if (xclOpenContext(m_handle, m_uuid, std::numeric_limits<unsigned int>::max(), true))
        throw std::runtime_error("'Failed to lock down xclbin");
    }
  
    ~xclbin_lock(){
      xclCloseContext(m_handle, m_uuid, std::numeric_limits<unsigned int>::max());
    }
  };
  
  /*
   * mini logger to log errors, warnings and details produced by the test cases
   */
  void logger(boost::property_tree::ptree& _ptTest, const std::string& tag, const std::string& msg)
  {
    boost::property_tree::ptree _ptLog;
    boost::property_tree::ptree _ptExistingLog;
    boost::optional<boost::property_tree::ptree&> _ptChild = _ptTest.get_child_optional("log");
    if(_ptChild)
      _ptExistingLog = _ptChild.get();
  
    _ptLog.put(tag, msg);
    _ptExistingLog.push_back(std::make_pair("", _ptLog));
    _ptTest.put_child("log", _ptExistingLog);
  }
  
  /*
   * search for xclbin for an SSV2 platform
   */
  std::string
  searchSSV2Xclbin(const std::string& logic_uuid,
                    const std::string& xclbin, boost::property_tree::ptree& _ptTest)
  {
    std::string formatted_fw_path("/opt/xilinx/firmware/");
    boost::filesystem::path fw_dir(formatted_fw_path);
    if(!boost::filesystem::is_directory(fw_dir)) {
      logger(_ptTest, "Error", boost::str(boost::format("Failed to find %s") % fw_dir));
      logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
      _ptTest.put("status", "failed");
      return "";
    }
  
    std::vector<std::string> suffix = { "dsabin", "xsabin" };
  
    for(const std::string& t : suffix) {
      std::regex e("(^" + formatted_fw_path + "[^/]+/[^/]+/[^/]+/).+\\." + t);
      for(boost::filesystem::recursive_directory_iterator iter(fw_dir,
            boost::filesystem::symlink_option::recurse), end; iter != end;) {
        std::string name = iter->path().string();
        std::smatch cm;
        if (!boost::filesystem::is_directory(boost::filesystem::path(name.c_str()))) {
          iter.no_push();
        }
        else {
          iter.no_push(false);
        }
  
        std::regex_match(name, cm, e);
        if (cm.size() > 0) {
          auto dtbbuf = XBUtilities::get_axlf_section(name, PARTITION_METADATA);
          if (dtbbuf.empty()) {
            ++iter;
            continue;
          }
          std::vector<std::string> uuids = XBUtilities::get_uuids(dtbbuf.data());
          if (!uuids.size()) {
            ++iter;
  		    }
          else if (uuids[0].compare(logic_uuid) == 0) {
            return cm.str(1) + "test/" + xclbin;
          }
        }
        else if (iter.level() > 4) {
          iter.pop();
          continue;
        }
  		  ++iter;
      }
    }
    logger(_ptTest, "Error", boost::str(boost::format("Failed to find xclbin in %s") % fw_dir));
    logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
    _ptTest.put("status", "failed");
    return "";
  }
  
  std::string
  getXsaPath(const uint16_t vendor)
  {
    if (vendor == 0 || (vendor == INVALID_ID))
      return std::string();
  
    std::string vendorName;
    switch (vendor) {
      case ARISTA_ID:
        vendorName = "arista";
        break;
      default:
      case XILINX_ID:
        vendorName = "xilinx";
        break;
    }
    return "/opt/" + vendorName + "/xsa/";
  }
  
  /*
   * search for xclbin for a legacy platform
   */
  std::string
  searchLegacyXclbin(const uint16_t vendor, const std::string& dev_name, const std::string& xclbin, boost::property_tree::ptree& _ptTest)
  {
    const std::string dsapath("/opt/xilinx/dsa/");
    const std::string xsapath(getXsaPath(vendor));
    //create possible xclbin paths
    std::string xsaXclbinPath = xsapath + dev_name + "/test/" + xclbin;
    std::string dsaXclbinPath = dsapath + dev_name + "/test/" + xclbin;
    boost::filesystem::path xsa_xclbin(xsaXclbinPath);
    boost::filesystem::path dsa_xclbin(dsaXclbinPath);
    if (boost::filesystem::exists(xsa_xclbin)) {
      return xsaXclbinPath;
    }
    else if (boost::filesystem::exists(dsa_xclbin)) {
      return dsaXclbinPath;
    }
  
    logger(_ptTest, "Error", boost::str(boost::format("Failed to find %s or %s") % xsaXclbinPath % dsaXclbinPath));
    logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
  
    _ptTest.put("status", "failed");
    return "";
  }
  
  /*
   * helper funtion for kernel and bandwidth test cases
   * Steps:
   * 1. Find xclbin after determining if the shell is 1RP or 2RP
   * 2. Find testcase
   * 3. Spawn a testcase process
   * 4. Check results
   */
  void
  runTestCase(const std::shared_ptr<xrt_core::device>& _dev, const std::string& py, const std::string& xclbin,
              boost::property_tree::ptree& _ptTest)
  {
    std::string name;
    try {
      name = xrt_core::device_query<xrt_core::query::rom_vbnv>(_dev);
    } catch(...) {
      logger(_ptTest, "Error", "Unable to find device VBNV");
  
      _ptTest.put("status", "failed");
      return;
    }
  
    //check if a 2RP platform
    std::vector<std::string> logic_uuid;
    try{
      logic_uuid = xrt_core::device_query<xrt_core::query::logic_uuids>(_dev);
    } catch(...) { }
  
    std::string xclbinPath;
    if(!logic_uuid.empty()) {
      xclbinPath = searchSSV2Xclbin(logic_uuid.front(), xclbin, _ptTest);
    } else {
      auto vendor = xrt_core::device_query<xrt_core::query::pcie_vendor>(_dev);
      xclbinPath = searchLegacyXclbin(vendor, name, xclbin, _ptTest);
    }
  
    // 0RP (nonDFX) flat shell support.  
    // Currently, there isn't a clean way to determine if a nonDFX shell's interface is truly flat.  
    // At this time, this is determined by whether or not it delivers an accelerator (e.g., verify.xclbin)
    if(!logic_uuid.empty() && !boost::filesystem::exists(xclbinPath)) {
      //if bandwidth xclbin isn't present, skip the test
      logger(_ptTest, "Details", "Verify xclbin not available or shell partition is not programmed. Skipping validation.");
      _ptTest.put("status", "skipped");
      return;
    }
  
    //check if xclbin is present
    if(xclbinPath.empty()) {
      if(xclbin.compare("bandwidth.xclbin") == 0) {
        //if bandwidth xclbin isn't present, skip the test
        logger(_ptTest, "Details", "Bandwidth xclbin not available. Skipping validation.");
        _ptTest.put("status", "skipped");
      }
      return;
    }
    // log xclbin path for debugging purposes
    logger(_ptTest, "Xclbin", xclbinPath);
  
    //check if testcase is present
    std::string xrtTestCasePath = "/opt/xilinx/xrt/test/" + py;
    boost::filesystem::path xrt_path(xrtTestCasePath);
    if (!boost::filesystem::exists(xrt_path)) {
      logger(_ptTest, "Error", boost::str(boost::format("Failed to find %s") % xrtTestCasePath));
      logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
      _ptTest.put("status", "failed");
      return;
    }
    // log testcase path for debugging purposes
    logger(_ptTest, "Testcase", xrtTestCasePath);
  
    std::vector<std::string> args = { "-k", xclbinPath, "-d", std::to_string(_dev.get()->get_device_id()) };
    std::ostringstream os_stdout;
    std::ostringstream os_stderr;
    int exit_code = XBU::runPythonScript(xrtTestCasePath, args, os_stdout, os_stderr);
    if (exit_code != 0) {
      logger(_ptTest, "Error", os_stdout.str());
      logger(_ptTest, "Error", os_stderr.str());
      _ptTest.put("status", "failed");
    }
    else {
      _ptTest.put("status", "passed");
    }
  
    // Get out max thruput for bandwidth testcase
    if(xclbin.compare("bandwidth.xclbin") == 0) {
      size_t st = os_stdout.str().find("Maximum");
      if (st != std::string::npos) {
        size_t end = os_stdout.str().find("\n", st);
        logger(_ptTest, "Details", os_stdout.str().substr(st, end - st));
      }
    }
  }
  
  /*
   * helper function for kernelVersionTest
   */
  static void
  checkOSRelease(const std::vector<std::string> kernel_versions, const std::string& release,
                  boost::property_tree::ptree& _ptTest)
  {
    for (const auto& ver : kernel_versions) {
      if (release.find(ver) != std::string::npos) {
        _ptTest.put("status", "passed");
        return;
      }
    }
    _ptTest.put("status", "passed");
    logger(_ptTest, "Warning", boost::str(boost::format("Kernel version %s is not officially supported. %s is the latest supported version")
                              % release % kernel_versions.back()));
  }
  
  /*
   * helper function for M2M and P2P test
   */
  static void
  free_unmap_bo(xclDeviceHandle handle, xclBufferHandle boh, void * boptr, size_t bo_size)
  {
  #ifdef __GNUC__
    if(boptr != nullptr)
      munmap(boptr, bo_size);
  #endif
  /* windows doesn't have munmap
   * FreeUserPhysicalPages might be the windows equivalent
   */
  #ifdef _WIN32
    boptr = boptr;
    bo_size = bo_size;
  #endif
  
    if (boh)
      xclFreeBO(handle, boh);
  }
  
  /*
   * helper function for P2P test
   */
  static bool
  p2ptest_set_or_cmp(char *boptr, size_t size, char pattern, bool set)
  {
    int stride = xrt_core::getpagesize();
  
    assert((size % stride) == 0);
    for (size_t i = 0; i < size; i += stride) {
      if (set) {
        boptr[i] = pattern;
      }
      else if (boptr[i] != pattern) {
        return false;
      }
    }
    return true;
  }
  
  /*
   * helper function for P2P test
   */
  static bool
  p2ptest_chunk(xclDeviceHandle handle, char *boptr, uint64_t dev_addr, uint64_t size)
  {
    char *buf = nullptr;
  
    if (xrt_core::posix_memalign(reinterpret_cast<void **>(&buf), xrt_core::getpagesize(), size))
      return false;
  
    p2ptest_set_or_cmp(buf, size, 'A', true);
    if (xclUnmgdPwrite(handle, 0, buf, size, dev_addr) < 0)
      return false;
    if (!p2ptest_set_or_cmp(boptr, size, 'A', false))
      return false;
  
    p2ptest_set_or_cmp(boptr, size, 'B', true);
    if (xclUnmgdPread(handle, 0, buf, size, dev_addr) < 0)
      return false;
    if (!p2ptest_set_or_cmp(buf, size, 'B', false))
      return false;
  
    free(buf);
    return true;
  }
  
  /*
   * helper function for P2P test
   */
  static bool
  p2ptest_bank(xclDeviceHandle handle, boost::property_tree::ptree& _ptTest, std::string m_tag, 
               unsigned int mem_idx, uint64_t addr, uint64_t bo_size)
  {
    const size_t chunk_size = 16 * 1024 * 1024; //16 MB
  
    xclBufferHandle boh = xclAllocBO(handle, bo_size, 0, XCL_BO_FLAGS_P2P | mem_idx);
    if (boh == NULLBO) {
      _ptTest.put("status", "failed");
      logger(_ptTest, "Error", "Couldn't allocate BO");
      return false;
    }
    char *boptr = (char *)xclMapBO(handle, boh, true);
    if (boptr == nullptr) {
      _ptTest.put("status", "failed");
      logger(_ptTest, "Error", "Couldn't map BO");
      free_unmap_bo(handle, boh, boptr, bo_size);
      return false;
    }
  
    int counter = 0;
    XBU::ProgressBar run_test("Running Test on " + m_tag, 1024, XBU::is_esc_enabled(), std::cout);
    for(uint64_t c = 0; c < bo_size; c += chunk_size) {
      if(!p2ptest_chunk(handle, boptr + c, addr + c, chunk_size)) {
        _ptTest.put("status", "failed");
        logger(_ptTest, "Error", boost::str(boost::format("P2P failed at offset 0x%x, on memory index %d") % c % mem_idx));
        free_unmap_bo(handle, boh, boptr, bo_size);
        run_test.finish(false, "");
        std::cout << EscapeCodes::cursor().prev_line() << EscapeCodes::cursor().clear_line();
        return false;
      }
      run_test.update(++counter);
    }
    free_unmap_bo(handle, boh, boptr, bo_size);
    run_test.finish(true, "");
    std::cout << EscapeCodes::cursor().prev_line() << EscapeCodes::cursor().clear_line();
    _ptTest.put("status", "passed");
    return true;
  }
  
  /*
   * helper function for M2M test
   */
  static int
  m2m_alloc_init_bo(xclDeviceHandle handle, boost::property_tree::ptree& _ptTest, xclBufferHandle &boh,
                     char * &boptr, size_t bo_size, int bank, char pattern)
  {
    boh = xclAllocBO(handle, bo_size, 0, bank);
    if (boh == NULLBO) {
      _ptTest.put("status", "failed");
      logger(_ptTest, "Error", "Couldn't allocate BO");
      return 1;
    }
    boptr = (char*) xclMapBO(handle, boh, true);
    if (boptr == nullptr) {
      _ptTest.put("status", "failed");
      logger(_ptTest, "Error", "Couldn't map BO");
      free_unmap_bo(handle, boh, boptr, bo_size);
      return 1;
    }
    memset(boptr, pattern, bo_size);
    if(xclSyncBO(handle, boh, XCL_BO_SYNC_BO_TO_DEVICE, bo_size, 0)) {
      _ptTest.put("status", "failed");
      logger(_ptTest, "Error", "Couldn't sync BO");
      free_unmap_bo(handle, boh, boptr, bo_size);
      return 1;
    }
    return 0;
  }
  
  /*
   * helper function for M2M test
   */
  static double
  m2mtest_bank(xclDeviceHandle handle, boost::property_tree::ptree& _ptTest, int bank_a, int bank_b, size_t bo_size)
  {
    xclBufferHandle bo_src = NULLBO;
    xclBufferHandle bo_tgt = NULLBO;
    char *bo_src_ptr = nullptr;
    char *bo_tgt_ptr = nullptr;
    double bandwidth = 0;
  
    //Allocate and init bo_src
    if(m2m_alloc_init_bo(handle, _ptTest, bo_src, bo_src_ptr, bo_size, bank_a, 'A'))
      return bandwidth;
  
    //Allocate and init bo_tgt
    if(m2m_alloc_init_bo(handle, _ptTest, bo_tgt, bo_tgt_ptr, bo_size, bank_b, 'B')) {
      free_unmap_bo(handle, bo_src, bo_src_ptr, bo_size);
      return bandwidth;
    }
  
    XBU::Timer timer;
    if (xclCopyBO(handle, bo_tgt, bo_src, bo_size, 0, 0))
      return bandwidth;
    double timer_duration_sec = timer.stop().count();
  
    if(xclSyncBO(handle, bo_tgt, XCL_BO_SYNC_BO_FROM_DEVICE, bo_size, 0)) {
      free_unmap_bo(handle, bo_src, bo_src_ptr, bo_size);
      free_unmap_bo(handle, bo_tgt, bo_tgt_ptr, bo_size);
      _ptTest.put("status", "failed");
      logger(_ptTest, "Error", "Unable to sync target BO");
      return bandwidth;
    }
  
    bool match = (memcmp(bo_src_ptr, bo_tgt_ptr, bo_size) == 0);
  
    // Clean up
    free_unmap_bo(handle, bo_src, bo_src_ptr, bo_size);
    free_unmap_bo(handle, bo_tgt, bo_tgt_ptr, bo_size);
  
    if (!match) {
      _ptTest.put("status", "failed");
      logger(_ptTest, "Error", "Memory comparison failed");
      return bandwidth;
    }
  
    //bandwidth
    double total_Mb = static_cast<double>(bo_size) / static_cast<double>(1024 * 1024); //convert to MB
    return static_cast<double>(total_Mb / timer_duration_sec);
  }
  
  static int 
  program_xclbin(const xclDeviceHandle hdl, const std::string& xclbin, boost::property_tree::ptree& _ptTest)
  {
    std::ifstream stream(xclbin, std::ios::binary);
    if (!stream) {
      logger(_ptTest, "Error", boost::str(boost::format("Could not open %s for reding") % xclbin));
      return 1;
    }
  
    stream.seekg(0,stream.end);
    size_t size = stream.tellg();
    stream.seekg(0,stream.beg);
    
    std::vector<char> raw(size);
    stream.read(raw.data(),size);
  
    std::string v(raw.data(),raw.data()+7);
    if (v != "xclbin2") {
      logger(_ptTest, "Error", boost::str(boost::format("Bad binary version '%s'") % v));
      return 1;
    }
  
    if (xclLoadXclBin(hdl,reinterpret_cast<const axlf*>(raw.data()))) {
      logger(_ptTest, "Error", "Could not program device");
      return 1;
    }
    return 0;
  }
  
  static bool
  search_and_program_xclbin(const std::shared_ptr<xrt_core::device>& dev, boost::property_tree::ptree& ptTest)
  {
    xuid_t uuid;
    uuid_parse(xrt_core::device_query<xrt_core::query::xclbin_uuid>(dev).c_str(), uuid);
    std::string xclbin = ptTest.get<std::string>("xclbin", "");
    
    //if no xclbin is loaded, locate the default xclbin
    if (uuid_is_null(uuid) && !xclbin.empty()) {
      //check if a 2RP platform
      std::vector<std::string> logic_uuid;
      try{
        logic_uuid = xrt_core::device_query<xrt_core::query::logic_uuids>(dev);
      } catch(...) { }
  
      std::string xclbinPath;
      if(!logic_uuid.empty()) {
        xclbinPath = searchSSV2Xclbin(logic_uuid.front(), xclbin, ptTest);
      } else {
        auto vendor = xrt_core::device_query<xrt_core::query::pcie_vendor>(dev);
        auto name = xrt_core::device_query<xrt_core::query::rom_vbnv>(dev);
        xclbinPath = searchLegacyXclbin(vendor, name, xclbin, ptTest);
      }
  
      if(!boost::filesystem::exists(xclbinPath)) {
        logger(ptTest, "Details", boost::str(boost::format("%s not available. Skipping validation.") % xclbin));
        ptTest.put("status", "skipped");
        return false;
      }
  
      if(program_xclbin(dev->get_device_handle(), xclbinPath, ptTest) != 0) {
        ptTest.put("status", "failed");
        return false;
      }
    }
    return true;
  }
  
  /*
   * TEST #1
   */
  void
  kernelVersionTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    //please append the new supported versions
    const std::vector<std::string> ubuntu_kernel_versions = { "4.4.0", "4.13.0", "4.15.0", "4.18.0", "5.0.0", "5.3.0" };
    const std::vector<std::string> centos_rh_kernel_versions = { "3.10.0-693", "3.10.0-862", "3.10.0-957", "3.10.0-1062", "3.10.0-1127", "4.18.0-147", "4.18.0-193" };
  
    boost::property_tree::ptree _pt_host;
    std::make_shared<ReportHost>()->getPropertyTreeInternal(_dev.get(), _pt_host);
    const std::string os = _pt_host.get<std::string>("host.os.distribution");
    const std::string release = _pt_host.get<std::string>("host.os.release");
  
    if(os.find("Ubuntu") != std::string::npos) {
      checkOSRelease(ubuntu_kernel_versions, release, _ptTest);
    }
    else if(os.find("Red Hat") != std::string::npos || os.find("CentOS") != std::string::npos) {
      checkOSRelease(centos_rh_kernel_versions, release, _ptTest);
    }
    else {
      _ptTest.put("status", "failed");
    }
  }
  
  /*
   * TEST #2
   */
  void
  auxConnectionTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    const std::vector<std::string> auxPwrRequiredDevice = { "VCU1525", "U200", "U250", "U280" };
  
    std::string name = xrt_core::device_query<xrt_core::query::xmc_board_name>(_dev);
    uint64_t max_power = xrt_core::device_query<xrt_core::query::xmc_max_power>(_dev);
  
    //check if device has aux power connector
    bool auxDevice = false;
    for (auto bd : auxPwrRequiredDevice) {
      if (name.find(bd) != std::string::npos) {
        auxDevice = true;
        break;
      }
    }
  
    if (!auxDevice) {
        logger(_ptTest, "Details", "Aux power connector is not available on this board");
        _ptTest.put("status", "skipped");
        return;
    }
  
    //check aux cable if board u200, u250, u280
    if(max_power == 0) {
      logger(_ptTest, "Warning", "Aux power is not connected");
      logger(_ptTest, "Warning", "Device is not stable for heavy acceleration tasks");
    }
    _ptTest.put("status", "passed");
  }
  
  /*
   * TEST #3
   */
  void
  pcieLinkTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    uint64_t speed     = xrt_core::device_query<xrt_core::query::pcie_link_speed>(_dev);
    uint64_t max_speed = xrt_core::device_query<xrt_core::query::pcie_link_speed_max>(_dev);
    uint64_t width     = xrt_core::device_query<xrt_core::query::pcie_express_lane_width>(_dev);
    uint64_t max_width = xrt_core::device_query<xrt_core::query::pcie_express_lane_width_max>(_dev);
    if (speed != max_speed || width != max_width) {
      logger(_ptTest, "Warning", "Link is active");
      logger(_ptTest, "Warning", boost::str(boost::format("Please make sure that the device is plugged into Gen %dx%d, instead of Gen %dx%d. %s.")
                                            % max_speed % max_width % speed % width % "Lower performance maybe experienced"));
    }
    _ptTest.put("status", "passed");
  }
  
  /*
   * TEST #4
   */
  void
  scVersionTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    auto sc_ver = xrt_core::device_query<xrt_core::query::xmc_bmc_version>(_dev);
    std::string exp_sc_ver = "";
    try{
      exp_sc_ver = xrt_core::device_query<xrt_core::query::expected_bmc_version>(_dev);
    } catch(...) {}
  
    if (!exp_sc_ver.empty() && sc_ver.compare(exp_sc_ver) != 0) {
      logger(_ptTest, "Warning", "SC firmware misatch");
      logger(_ptTest, "Warning", boost::str(boost::format("SC firmware version %s is running on the board, but SC firmware version %s is expected from the installed shell. %s.")
                                            % sc_ver % exp_sc_ver % "Please use xbmgmt --new status to check the installed shell"));
    }
    _ptTest.put("status", "passed");
  }
  
  /*
   * TEST #5
   */
  void
  verifyKernelTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    runTestCase(_dev, "22_verify.py", _ptTest.get<std::string>("xclbin"), _ptTest);
  }
  
  /*
   * TEST #6
   */
  void
  dmaTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    if(!search_and_program_xclbin(_dev, _ptTest)) {
      return;
    }
  
    // get DDR bank count from mem_topology if possible
    auto membuf = xrt_core::device_query<xrt_core::query::mem_topology_raw>(_dev);
    auto mem_topo = reinterpret_cast<const mem_topology*>(membuf.data());
  
    auto vendor = xrt_core::device_query<xrt_core::query::pcie_vendor>(_dev);
    size_t totalSize = 0;
    switch (vendor) {
      case ARISTA_ID:
        totalSize = 0x20000000;
        break;
      default:
      case XILINX_ID:
        break;
    }
  
    for (auto& mem : boost::make_iterator_range(mem_topo->m_mem_data, mem_topo->m_mem_data + mem_topo->m_count)) {
      auto midx = std::distance(mem_topo->m_mem_data, &mem);
      if (mem.m_type == MEM_STREAMING)
        continue;
  
      if (!mem.m_used)
        continue;
  
      std::stringstream run_details;
      size_t block_size = 16 * 1024 * 1024; // Default block size 16MB
      xcldev::DMARunner runner(_dev->get_device_handle(), block_size, static_cast<unsigned int>(midx), totalSize);
      try {
        runner.run(run_details);
        _ptTest.put("status", "passed");
        std::string line;
        while(std::getline(run_details, line))
          logger(_ptTest, "Details", line);
      }
      catch (xrt_core::error& ex) {
        _ptTest.put("status", "failed");
        logger(_ptTest, "Error", ex.what());
      }
    }
  }
  
  /*
   * TEST #7
   */
  void
  bandwidthKernelTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    std::string name;
    try {
      name = xrt_core::device_query<xrt_core::query::rom_vbnv>(_dev);
    } catch(...) {
      logger(_ptTest, "Error", "Unable to find device VBNV");
      _ptTest.put("status", "failed");
      return;
    }
    std::string testcase = (name.find("vck5000") != std::string::npos) ? "versal_23_bandwidth.py" : "23_bandwidth.py";
    runTestCase(_dev, testcase, _ptTest.get<std::string>("xclbin"), _ptTest);
  }
  
  /*
   * TEST #8
   */
  void
  p2pTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    if(!search_and_program_xclbin(_dev, _ptTest)) {
      return;
    }
  
    std::string msg;
    xclbin_lock xclbin_lock(_dev);
    XBU::check_p2p_config(_dev, msg);
  
    if(msg.find("Error") == 0) {
      logger(_ptTest, "Error", msg.substr(msg.find(':')+1));
      _ptTest.put("status", "error");
      return;
    }
    else if(msg.find("Warning") == 0) {
      logger(_ptTest, "Warning", msg.substr(msg.find(':')+1));
      _ptTest.put("status", "skipped");
      return;
    }
    else if (!msg.empty()) {
      logger(_ptTest, "Details", msg);
      _ptTest.put("status", "skipped");
      return;
    }
  
    auto membuf = xrt_core::device_query<xrt_core::query::mem_topology_raw>(_dev);
    auto mem_topo = reinterpret_cast<const mem_topology*>(membuf.data());
    std::string name = xrt_core::device_query<xrt_core::query::rom_vbnv>(_dev);
  
    for (auto& mem : boost::make_iterator_range(mem_topo->m_mem_data, mem_topo->m_mem_data + mem_topo->m_count)) {
      auto midx = std::distance(mem_topo->m_mem_data, &mem);
      std::vector<std::string> sup_list = { "HBM", "bank", "DDR" };
      //p2p is not supported for DDR on u280
      if(name.find("_u280_") != std::string::npos)
        sup_list.pop_back();
  
      const std::string mem_tag(reinterpret_cast<const char *>(mem.m_tag));
      for(const auto& x : sup_list) {
        if(mem_tag.find(x) != std::string::npos && mem.m_used) {
          if(!p2ptest_bank(_dev->get_device_handle(), _ptTest, mem_tag, static_cast<unsigned int>(midx), mem.m_base_address, mem.m_size << 10))
            break;
          logger(_ptTest, "Details", mem_tag +  " validated");
        }
      }
    }
  }
  
  /*
   * TEST #9
   */
  void
  m2mTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    if(!search_and_program_xclbin(_dev, _ptTest)) {
      return;
    }
  
    xclbin_lock xclbin_lock(_dev);
    uint32_t m2m_enabled = xrt_core::device_query<xrt_core::query::kds_numcdmas>(_dev);
    std::string name = xrt_core::device_query<xrt_core::query::rom_vbnv>(_dev);
  
    // Workaround:
    // u250_xdma_201830_1 falsely shows that m2m is available
    // which causes a hang. Skip m2mtest if this platform is installed
    if (m2m_enabled == 0 || name.find("_u250_xdma_201830_1") != std::string::npos) {
      logger(_ptTest, "Details", "M2M is not available");
      _ptTest.put("status", "skipped");
      return;
    }
  
    std::vector<mem_data> used_banks;
    const size_t bo_size = 256L * 1024 * 1024;
    auto membuf = xrt_core::device_query<xrt_core::query::mem_topology_raw>(_dev);
    auto mem_topo = reinterpret_cast<const mem_topology*>(membuf.data());
  
    for (auto& mem : boost::make_iterator_range(mem_topo->m_mem_data, mem_topo->m_mem_data + mem_topo->m_count)) {
      if (!strncmp(reinterpret_cast<const char *>(mem.m_tag), "HOST", 4))
          continue;
  
      if(mem.m_used && mem.m_size * 1024 >= bo_size)
        used_banks.push_back(mem);
    }
  
    for(unsigned int i = 0; i < used_banks.size()-1; i++) {
      for(unsigned int j = i+1; j < used_banks.size(); j++) {
        if(!used_banks[i].m_size || !used_banks[j].m_size)
          continue;
  
        double m2m_bandwidth = m2mtest_bank(_dev->get_device_handle(), _ptTest, i, j, bo_size);
        logger(_ptTest, "Details", boost::str(boost::format("%s -> %s M2M bandwidth: %.2f MB/s") % used_banks[i].m_tag
                    %used_banks[j].m_tag % m2m_bandwidth));
  
        if(m2m_bandwidth == 0) //test failed, exit
          return;
      }
    }
    _ptTest.put("status", "passed");
  }
  
  /*
   * TEST #10
   */
  void
  hostMemBandwidthKernelTest(const std::shared_ptr<xrt_core::device>& _dev, boost::property_tree::ptree& _ptTest)
  {
    uint64_t host_mem_size = 0;
    try {
      host_mem_size = xrt_core::device_query<xrt_core::query::host_mem_size>(_dev);
    } catch(...) {
      logger(_ptTest, "Details", "Address translator IP is not available");
      _ptTest.put("status", "skipped");
      return;
    }
  
    if (!host_mem_size) {
        logger(_ptTest, "Details", "Host memory is not enabled");
        _ptTest.put("status", "skipped");
        return;
    }
    runTestCase(_dev, "host_mem_23_bandwidth.py", _ptTest.get<std::string>("xclbin"), _ptTest);
  }
  
  /*
  * helper function to initialize test info
  */
  static boost::property_tree::ptree
  create_init_test(const std::string& name, const std::string& desc, const std::string& xclbin) {
    boost::property_tree::ptree _ptTest;
    _ptTest.put("name", name);
    _ptTest.put("description", desc);
    _ptTest.put("xclbin", xclbin);
    return _ptTest;
  }
  
  struct TestCollection {
    boost::property_tree::ptree ptTest;
    std::function<void(const std::shared_ptr<xrt_core::device>&, boost::property_tree::ptree&)> testHandle;
  };
  
  /*
  * create test suite
  */
  static std::vector<TestCollection> testSuite = {
    { create_init_test("Kernel version", "Check if kernel version is supported by XRT", ""), kernelVersionTest },
    { create_init_test("Aux connection", "Check if auxiliary power is connected", ""), auxConnectionTest },
    { create_init_test("PCIE link", "Check if PCIE link is active", ""), pcieLinkTest },
    { create_init_test("SC version", "Check if SC firmware is up-to-date", ""), scVersionTest },
    { create_init_test("Verify kernel", "Run 'Hello World' kernel test", "verify.xclbin"), verifyKernelTest },
    { create_init_test("DMA", "Run dma test", "verify.xclbin"), dmaTest },
    { create_init_test("Bandwidth kernel", "Run 'bandwidth kernel' and check the throughput", "bandwidth.xclbin"), bandwidthKernelTest },
    { create_init_test("Peer to peer bar", "Run P2P test", "bandwidth.xclbin"), p2pTest },
    { create_init_test("Memory to memory DMA", "Run M2M test", "bandwidth.xclbin"), m2mTest },
    { create_init_test("Host memory bandwidth test", "Run 'bandwidth kernel' when slave bridge is enabled", "bandwidth.xclbin"), hostMemBandwidthKernelTest }
  };
  
  /*
   * print basic information about a test
   */
  static void
  pretty_print_test_desc(const boost::property_tree::ptree& test, int test_idx,
                         size_t testSuiteSize, std::ostream & _ostream, const std::string& bdf)
  {
    std::string test_desc = boost::str(boost::format("%d/%d Test #%d [%s]") % test_idx % testSuiteSize % test_idx % bdf);
    _ostream << boost::format("%-28s: %s \n") % test_desc % test.get<std::string>("name");
    _ostream << boost::format("    %-24s: %s\n") % "Description" % test.get<std::string>("description");
  }
  
  /*
   * print test run
   */
  static void
  pretty_print_test_run(const boost::property_tree::ptree& test, 
                        test_status& status, std::ostream & _ostream)
  {
    std::string _status = test.get<std::string>("status");
    auto color = EscapeCodes::FGC_PASS;
    bool warn = false;
    bool error = false;
  
    try {
      for (const auto& dict : test.get_child("log")) {
        for (const auto& kv : dict.second) {
          _ostream<< boost::format("    %-24s: %s\n") % kv.first % kv.second.get_value<std::string>();
          if (boost::iequals(kv.first, "warning"))
            warn = true;
          else if (boost::iequals(kv.first, "error"))
            error = true;
        }
      }
    }
    catch(...) {}
  
    if(error) {
      color = EscapeCodes::FGC_FAIL;
      status = test_status::failed;
    }
    else if(warn) {
      _status.append(" with warnings");
      color = EscapeCodes::FGC_WARN;
      status = test_status::warning;
    }
  
    boost::to_upper(_status);
    _ostream << boost::format("    %-24s:") % "Test Status" << EscapeCodes::fgcolor(color).string() << boost::format(" [%s]\n") % _status
              << EscapeCodes::fgcolor::reset();
    _ostream << "-------------------------------------------------------------------------------" << std::endl;
  }
  
  /*
   * print final status of the card
   */
  static void
  print_status(test_status status, std::ostream & _ostream)
  {
    if (status == test_status::failed)
      _ostream<< "Validation failed";
    else
      _ostream << "Validation completed";
    if (status == test_status::warning)
      _ostream<< ", but with warnings";
    _ostream<< std::endl;
  }
  
  /*
   * Get basic information about the platform running on the device
   */
  
  static void
  get_platform_info(const std::shared_ptr<xrt_core::device>& device, boost::property_tree::ptree& _ptTree, 
                    Report::SchemaVersion schemaVersion, std::ostream & _ostream)
  {
    auto bdf = xrt_core::device_query<xrt_core::query::pcie_bdf>(device);
    _ptTree.put("device_id", xrt_core::query::pcie_bdf::to_string(bdf));
    _ptTree.put("platform", xrt_core::device_query<xrt_core::query::rom_vbnv>(device));
    _ptTree.put("sc_version", xrt_core::device_query<xrt_core::query::xmc_bmc_version>(device));
    _ptTree.put("platform_id", (boost::format("0x%x") % xrt_core::device_query<xrt_core::query::rom_time_since_epoch>(device)));
    if (schemaVersion == Report::SchemaVersion::text) {
      _ostream << boost::format("Validate device[%s]\n") % _ptTree.get<std::string>("device_id");
      _ostream << boost::format("%-20s: %s\n") % "Platform" % _ptTree.get<std::string>("platform");
      _ostream << boost::format("%-20s: %s\n") % "SC Version" % _ptTree.get<std::string>("sc_version");
      _ostream << boost::format("%-20s: %s\n\n") % "Platform ID" % _ptTree.get<std::string>("platform_id");
    }
  }
  
  void
  run_test_suite_device(const std::shared_ptr<xrt_core::device>& device, 
                        Report::SchemaVersion schemaVersion, 
                        std::vector<TestCollection *> testObjectsToRun,
                        boost::property_tree::ptree& _ptDevCollectionTestSuite,
                        std::ostream & _ostream)
  {
    boost::property_tree::ptree ptDeviceTestSuite;
    boost::property_tree::ptree ptDeviceInfo;
    test_status status = test_status::passed;
    
    if (testObjectsToRun.empty())
      throw std::runtime_error("No test given to validate against.");
  
    get_platform_info(device, ptDeviceInfo, schemaVersion, _ostream);
  
    int test_idx = 0;
    for (TestCollection * testPtr : testObjectsToRun) {
      boost::property_tree::ptree ptTest = testPtr->ptTest; // Create a copy of our entry
  
      if(schemaVersion == Report::SchemaVersion::text) {
        auto bdf = xrt_core::device_query<xrt_core::query::pcie_bdf>(device);
        pretty_print_test_desc(ptTest, ++test_idx, testObjectsToRun.size(), _ostream, xrt_core::query::pcie_bdf::to_string(bdf));
      }
  
      testPtr->testHandle(device, ptTest);
      ptDeviceTestSuite.push_back( std::make_pair("", ptTest) );
  
      if(schemaVersion == Report::SchemaVersion::text)
        pretty_print_test_run(ptTest, status, _ostream);
  
      // If a test fails, exit immediately
      if(status == test_status::failed) {
        break;
      }
    }
  
    if(schemaVersion == Report::SchemaVersion::text)
      print_status(status, _ostream);
  
    ptDeviceInfo.put_child("tests", ptDeviceTestSuite);
    _ptDevCollectionTestSuite.push_back( std::make_pair("", ptDeviceInfo) );
  }
  
  void
  run_tests_on_devices( xrt_core::device_collection &deviceCollection, 
                        Report::SchemaVersion schemaVersion, 
                        std::vector<TestCollection *> testObjectsToRun,
                        boost::property_tree::ptree& ptDevCollectionTestSuite,
                        std::ostream &_ostream)
  {
    if (schemaVersion ==  Report::SchemaVersion::text)
      _ostream << boost::format("Starting validation for %d devices\n\n") % deviceCollection.size();
  
    boost::property_tree::ptree ptDeviceTested;
    for(auto const& dev : deviceCollection) {
      run_test_suite_device(dev, schemaVersion, testObjectsToRun, ptDeviceTested, _ostream);
    }
  
    ptDevCollectionTestSuite.put_child("logical_devices", ptDeviceTested);
  
    if(schemaVersion !=  Report::SchemaVersion::text) {
      std::stringstream ss;
      boost::property_tree::json_parser::write_json(ss, ptDevCollectionTestSuite);
      _ostream << ss.str() << std::endl;
    }
  }
  
  
  }
  //end anonymous namespace
  
  // ----- C L A S S   M E T H O D S -------------------------------------------
  
  SubCmdValidate::SubCmdValidate(bool _isHidden, bool _isDepricated, bool _isPreliminary)
      : SubCmd("validate",
               "Validates the basic shell accelleration functionality")
  {
    const std::string longDescription = "Validates the given card by executing the platform's validate executable.";
    setLongDescription(longDescription);
    setExampleSyntax("");
    setIsHidden(_isHidden);
    setIsDeprecated(_isDepricated);
    setIsPreliminary(_isPreliminary);
  }
  
  XBU::VectorPairStrings
  getTestNameDescriptions(bool addAdditionOptions)
  {
    XBU::VectorPairStrings reportDescriptionCollection;
  
    // 'verbose' option
    if (addAdditionOptions) {
      reportDescriptionCollection.emplace_back("all", "All known validate tests will be executed (default)");
      reportDescriptionCollection.emplace_back("quick", "Only the first 5 tests will be executed");
    }
  
    // report names and discription
    for (const auto & test : testSuite) {
      reportDescriptionCollection.emplace_back(test.ptTest.get("name", "<unknown>"), test.ptTest.get("description", "<no description>"));
    }
  
    return reportDescriptionCollection;
  }
  
  
  void
  SubCmdValidate::execute(const SubCmdOptions& _options) const
  
  {
    XBU::verbose("SubCommand: validate");
  
    // -- Build up the format options
    const std::string formatOptionValues = XBU::create_suboption_list_string(Report::getSchemaDescriptionVector());
  
    const XBU::VectorPairStrings testNameDescription = getTestNameDescriptions(true /* Add "all" and "quick" options*/);
    const std::string formatRunValues = XBU::create_suboption_list_string(testNameDescription);
  
    // -- Retrieve and parse the subcommand options -----------------------------
    std::vector<std::string> device  = {"all"};
    std::vector<std::string> testsToRun = {"all"};
    std::string sFormat = "text";
    std::string sOutput = "";
    bool help = false;
  
    po::options_description commonOptions("Commmon Options");
    commonOptions.add_options()
      ("device,d", boost::program_options::value<decltype(device)>(&device)->multitoken(), "The device of interest. This is specified as follows:\n"
                                                                             "  <BDF> - Bus:Device.Function (e.g., 0000:d8:00.0)\n"
                                                                             "  all   - Examines all known devices (default)")
      ("format,f", boost::program_options::value<decltype(sFormat)>(&sFormat), (std::string("Report output format. Valid values are:\n") + formatOptionValues).c_str() )
      ("run,r", boost::program_options::value<decltype(testsToRun)>(&testsToRun)->multitoken(), (std::string("Run a subset of the test suite.  Valid options are:\n") + formatRunValues).c_str() )
      ("output,o", boost::program_options::value<decltype(sOutput)>(&sOutput), "Direct the output to the given file")
      ("help,h", boost::program_options::bool_switch(&help), "Help to use this sub-command")
    ;
  
    po::options_description hiddenOptions("Hidden Options");
  
    po::options_description allOptions("All Options");
    allOptions.add(commonOptions);
    allOptions.add(hiddenOptions);
  
    // Parse sub-command ...
    po::variables_map vm;
  
    try {
      po::store(po::command_line_parser(_options).options(allOptions).run(), vm);
      po::notify(vm); // Can throw
    } catch (po::error& e) {
      std::cerr << "ERROR: " << e.what() << std::endl << std::endl;
      printHelp(commonOptions, hiddenOptions);
      return;
    }
  
    // Check to see if help was requested or no command was found
    if (help == true)  {
      printHelp(commonOptions, hiddenOptions);
      return;
    }
  
    // -- Process the options --------------------------------------------
    Report::SchemaVersion schemaVersion = Report::SchemaVersion::unknown;    // Output schema version
    try {
      // Output Format
      schemaVersion = Report::getSchemaDescription(sFormat).schemaVersion;
      if (schemaVersion == Report::SchemaVersion::unknown) 
        throw xrt_core::error((boost::format("Unknown output format: '%s'") % sFormat).str());
  
      // Output file
      if (!sOutput.empty() && boost::filesystem::exists(sOutput)) 
          throw xrt_core::error((boost::format("Output file already exists: '%s'") % sOutput).str());
  
      if (testsToRun.empty()) 
        throw std::runtime_error("No test given to validate against.");
  
      // Examine test entries
      for (const auto &userTestName : testsToRun) {
        const std::string userTestNameLC = boost::algorithm::to_lower_copy(userTestName);   // Lower case the string entry
  
        if ((userTestNameLC == "all") && (testsToRun.size() > 1)) 
          throw xrt_core::error("The 'all' value for the tests to run cannot be used with any other named tests.");
  
        if ((userTestNameLC == "quick") && (testsToRun.size() > 1)) 
          throw xrt_core::error("The 'quick' value for the tests to run cannot be used with any other name tests.");
  
        // Validate all of the test names
        bool nameFound = false;
        for (auto &test : testNameDescription) {
          const std::string testNameLC = boost::algorithm::to_lower_copy(test.first);
          if (userTestNameLC.compare(testNameLC) == 0) {
            nameFound = true;
            break;
          }
        }
  
        // Did we have a hit?  If not then let the user know of a typo
        if (nameFound == false) {
          throw xrt_core::error((boost::format("Invalided test name: '%s'") % userTestName).str());
        }
      }
  
      // Now lower case all of the entries
      for (auto &userTestName : testsToRun) 
        boost::algorithm::to_lower(userTestName);   // Lower case the string entry
  
    } catch (const xrt_core::error& e) {
      // Catch only the exceptions that we have generated earlier
      std::cerr << boost::format("ERROR: %s\n") % e.what();
      printHelp(commonOptions, hiddenOptions);
      return;
    }
  
  
    // Collect all of the devices of interest
    std::set<std::string> deviceNames;
    xrt_core::device_collection deviceCollection;
    for (const auto & deviceName : device)
      deviceNames.insert(boost::algorithm::to_lower_copy(deviceName));
  
    try {
      XBU::collect_devices(deviceNames, true /*inUserDomain*/, deviceCollection);
    } catch (const std::runtime_error& e) {
      std::cerr << boost::format("ERROR: %s\n") % e.what();
      return;
    }
  
    // Collect all of the tests of interests
    std::vector<TestCollection *> testObjectsToRun;
  
    for (unsigned index = 0; index < testSuite.size(); ++index) {
      if (testsToRun[0] == "all") {
        testObjectsToRun.push_back(&testSuite[index]);
        continue;
      }
  
      if (testsToRun[0] == "quick") {
        testObjectsToRun.push_back(&testSuite[index]);
        // Only the first 5 should be processed
        if (index == 4)
          break;
      }
  
      // Must be a test name, look to see if should be added
      const std::string testSuiteName = boost::algorithm::to_lower_copy(testSuite[index].ptTest.get("name",""));
      for (const auto & testName : testsToRun) {
        if (testName.compare(testSuiteName) == 0) {
          testObjectsToRun.push_back(&testSuite[index]);
          break;
        }
      }
    }
  
    boost::property_tree::ptree ptDevCollectionTestSuite;
    // -- Run the tests --------------------------------------------------
    if (sOutput.empty()) {
      run_tests_on_devices(deviceCollection, schemaVersion, testObjectsToRun, ptDevCollectionTestSuite, std::cout);
    }
    else {
      std::ofstream fOutput;
      fOutput.open(sOutput, std::ios::out | std::ios::binary);
      if (!fOutput.is_open()) 
        throw xrt_core::error((boost::format("Unable to open the file '%s' for writing.") % sOutput).str());
  
      run_tests_on_devices(deviceCollection, schemaVersion, testObjectsToRun, ptDevCollectionTestSuite, fOutput);
  
      fOutput.close();
    }
  }
  
  