text: //===----------------------------------------------------------------------===//
  //
  //                     The LLVM Compiler Infrastructure
  //
  // This file is dual licensed under the MIT and the University of Illinois Open
  // Source Licenses. See LICENSE.txt for details.
  //
  //===----------------------------------------------------------------------===//
  
  
  #ifndef OFFLOAD_ENGINE_H_INCLUDED
  #define OFFLOAD_ENGINE_H_INCLUDED
  
  #include <limits.h>
  
  #include <list>
  #include <set>
  #include <map>
  #include "offload_common.h"
  #include "coi/coi_client.h"
  
  // Address range
  class MemRange {
  public:
      MemRange() : m_start(0), m_length(0) {}
      MemRange(const void *addr, uint64_t len) : m_start(addr), m_length(len) {}
  
      const void* start() const {
          return m_start;
      }
  
      const void* end() const {
          return static_cast<const char*>(m_start) + m_length;
      }
  
      uint64_t length() const {
          return m_length;
      }
  
      // returns true if given range overlaps with another one
      bool overlaps(const MemRange &o) const {
          // Two address ranges A[start, end) and B[start,end) overlap
          // if A.start < B.end and A.end > B.start.
          return start() < o.end() && end() > o.start();
      }
  
      // returns true if given range contains the other range
      bool contains(const MemRange &o) const {
          return start() <= o.start() && o.end() <= end();
      }
  
  private:
      const void* m_start;
      uint64_t    m_length;
  };
  
  // Data associated with a pointer variable
  class PtrData {
  public:
      PtrData(const void *addr, uint64_t len) :
          cpu_addr(addr, len), cpu_buf(0),
          mic_addr(0), alloc_disp(0), mic_buf(0), mic_offset(0),
          ref_count(0), is_static(false)
      {}
  
      //
      // Copy constructor
      //
      PtrData(const PtrData& ptr):
          cpu_addr(ptr.cpu_addr), cpu_buf(ptr.cpu_buf),
          mic_addr(ptr.mic_addr), alloc_disp(ptr.alloc_disp),
          mic_buf(ptr.mic_buf), mic_offset(ptr.mic_offset),
          ref_count(ptr.ref_count), is_static(ptr.is_static)
      {}
  
      bool operator<(const PtrData &o) const {
          // Variables are sorted by the CPU start address.
          // Overlapping memory ranges are considered equal.
          return (cpu_addr.start() < o.cpu_addr.start()) &&
                 !cpu_addr.overlaps(o.cpu_addr);
      }
  
      long add_reference() {
          if (is_static) {
              return LONG_MAX;
          }
  #ifndef TARGET_WINNT
          return __sync_fetch_and_add(&ref_count, 1);
  #else // TARGET_WINNT
          return _InterlockedIncrement(&ref_count) - 1;
  #endif // TARGET_WINNT
      }
  
      long remove_reference() {
          if (is_static) {
              return LONG_MAX;
          }
  #ifndef TARGET_WINNT
          return __sync_sub_and_fetch(&ref_count, 1);
  #else // TARGET_WINNT
          return _InterlockedDecrement(&ref_count);
  #endif // TARGET_WINNT
      }
  
      long get_reference() const {
          if (is_static) {
              return LONG_MAX;
          }
          return ref_count;
      }
  
  public:
      // CPU address range
      const MemRange  cpu_addr;
  
      // CPU and MIC buffers
      COIBUFFER       cpu_buf;
      COIBUFFER       mic_buf;
  
      // placeholder for buffer address on mic
      uint64_t        mic_addr;
  
      uint64_t        alloc_disp;
  
      // additional offset to pointer data on MIC for improving bandwidth for
      // data which is not 4K aligned
      uint32_t        mic_offset;
  
      // if true buffers are created from static memory
      bool            is_static;
      mutex_t         alloc_ptr_data_lock;
  
  private:
      // reference count for the entry
      long            ref_count;
  };
  
  typedef std::list<PtrData*> PtrDataList;
  
  // Data associated with automatic variable
  class AutoData {
  public:
      AutoData(const void *addr, uint64_t len) :
          cpu_addr(addr, len), ref_count(0)
      {}
  
      bool operator<(const AutoData &o) const {
          // Variables are sorted by the CPU start address.
          // Overlapping memory ranges are considered equal.
          return (cpu_addr.start() < o.cpu_addr.start()) &&
                 !cpu_addr.overlaps(o.cpu_addr);
      }
  
      long add_reference() {
  #ifndef TARGET_WINNT
          return __sync_fetch_and_add(&ref_count, 1);
  #else // TARGET_WINNT
          return _InterlockedIncrement(&ref_count) - 1;
  #endif // TARGET_WINNT
      }
  
      long remove_reference() {
  #ifndef TARGET_WINNT
          return __sync_sub_and_fetch(&ref_count, 1);
  #else // TARGET_WINNT
          return _InterlockedDecrement(&ref_count);
  #endif // TARGET_WINNT
      }
  
      long get_reference() const {
          return ref_count;
      }
  
  public:
      // CPU address range
      const MemRange cpu_addr;
  
  private:
      // reference count for the entry
      long ref_count;
  };
  
  // Set of autimatic variables
  typedef std::set<AutoData> AutoSet;
  
  // Target image data
  struct TargetImage
  {
      TargetImage(const char *_name, const void *_data, uint64_t _size,
                  const char *_origin, uint64_t _offset) :
          name(_name), data(_data), size(_size),
          origin(_origin), offset(_offset)
      {}
  
      // library name
      const char* name;
  
      // contents and size
      const void* data;
      uint64_t    size;
  
      // file of origin and offset within that file
      const char* origin;
      uint64_t    offset;
  };
  
  typedef std::list<TargetImage> TargetImageList;
  
  // Data associated with persistent auto objects
  struct PersistData
  {
      PersistData(const void *addr, uint64_t routine_num, uint64_t size) :
          stack_cpu_addr(addr), routine_id(routine_num)
      {
          stack_ptr_data = new PtrData(0, size);
      }
      // 1-st key value - beginning of the stack at CPU
      const void *   stack_cpu_addr;
      // 2-nd key value - identifier of routine invocation at CPU
      uint64_t   routine_id;
      // corresponded PtrData; only stack_ptr_data->mic_buf is used
      PtrData * stack_ptr_data;
      // used to get offset of the variable in stack buffer
      char * cpu_stack_addr;
  };
  
  typedef std::list<PersistData> PersistDataList;
  
  // class representing a single engine
  struct Engine {
      friend void __offload_init_library_once(void);
      friend void __offload_fini_library(void);
  
  #define check_result(res, tag, ...) \
      { \
          if (res == COI_PROCESS_DIED) { \
              fini_process(true); \
              exit(1); \
          } \
          if (res != COI_SUCCESS) { \
              __liboffload_error_support(tag, __VA_ARGS__); \
              exit(1); \
          } \
      }
  
      int get_logical_index() const {
          return m_index;
      }
  
      int get_physical_index() const {
          return m_physical_index;
      }
  
      const COIPROCESS& get_process() const {
          return m_process;
      }
  
      // initialize device
      void init(void);
  
      // add new library
      void add_lib(const TargetImage &lib)
      {
          m_lock.lock();
          m_ready = false;
          m_images.push_back(lib);
          m_lock.unlock();
      }
  
      COIRESULT compute(
          const std::list<COIBUFFER> &buffers,
          const void*         data,
          uint16_t            data_size,
          void*               ret,
          uint16_t            ret_size,
          uint32_t            num_deps,
          const COIEVENT*     deps,
          COIEVENT*           event
      );
  
  #ifdef MYO_SUPPORT
      // temporary workaround for blocking behavior for myoiLibInit/Fini calls
      void init_myo(COIEVENT *event) {
          COIRESULT res;
          res = COI::PipelineRunFunction(get_pipeline(),
                                         m_funcs[c_func_myo_init],
                                         0, 0, 0, 0, 0, 0, 0, 0, 0,
                                         event);
          check_result(res, c_pipeline_run_func, m_index, res);
      }
  
      void fini_myo(COIEVENT *event) {
          COIRESULT res;
          res = COI::PipelineRunFunction(get_pipeline(),
                                         m_funcs[c_func_myo_fini],
                                         0, 0, 0, 0, 0, 0, 0, 0, 0,
                                         event);
          check_result(res, c_pipeline_run_func, m_index, res);
      }
  #endif // MYO_SUPPORT
  
      //
      // Memory association table
      //
      PtrData* find_ptr_data(const void *ptr) {
          m_ptr_lock.lock();
          PtrSet::iterator res = m_ptr_set.find(PtrData(ptr, 0));
          m_ptr_lock.unlock();
          if (res == m_ptr_set.end()) {
              return 0;
          }
          return const_cast<PtrData*>(res.operator->());
      }
  
      PtrData* insert_ptr_data(const void *ptr, uint64_t len, bool &is_new) {
          m_ptr_lock.lock();
          std::pair<PtrSet::iterator, bool> res =
              m_ptr_set.insert(PtrData(ptr, len));
          PtrData* ptr_data = const_cast<PtrData*>(res.first.operator->());
          m_ptr_lock.unlock();
  
          is_new = res.second;
          if (is_new) {
              // It's necessary to lock as soon as possible.
              // unlock must be done at call site of insert_ptr_data at
              // branch for is_new
              ptr_data->alloc_ptr_data_lock.lock();
          }
          return ptr_data;
      }
  
      void remove_ptr_data(const void *ptr) {
          m_ptr_lock.lock();
          m_ptr_set.erase(PtrData(ptr, 0));
          m_ptr_lock.unlock();
      }
  
      //
      // Automatic variables
      //
      AutoData* find_auto_data(const void *ptr) {
          AutoSet &auto_vars = get_auto_vars();
          AutoSet::iterator res = auto_vars.find(AutoData(ptr, 0));
          if (res == auto_vars.end()) {
              return 0;
          }
          return const_cast<AutoData*>(res.operator->());
      }
  
      AutoData* insert_auto_data(const void *ptr, uint64_t len) {
          AutoSet &auto_vars = get_auto_vars();
          std::pair<AutoSet::iterator, bool> res =
              auto_vars.insert(AutoData(ptr, len));
          return const_cast<AutoData*>(res.first.operator->());
      }
  
      void remove_auto_data(const void *ptr) {
          get_auto_vars().erase(AutoData(ptr, 0));
      }
  
      //
      // Signals
      //
      void add_signal(const void *signal, OffloadDescriptor *desc) {
          m_signal_lock.lock();
          m_signal_map[signal] = desc;
          m_signal_lock.unlock();
      }
  
      OffloadDescriptor* find_signal(const void *signal, bool remove) {
          OffloadDescriptor *desc = 0;
  
          m_signal_lock.lock();
          {
              SignalMap::iterator it = m_signal_map.find(signal);
              if (it != m_signal_map.end()) {
                  desc = it->second;
                  if (remove) {
                      m_signal_map.erase(it);
                  }
              }
          }
          m_signal_lock.unlock();
  
          return desc;
      }
  
      // stop device process
      void fini_process(bool verbose);
  
      // list of stacks active at the engine
      PersistDataList m_persist_list;
  
  private:
      Engine() : m_index(-1), m_physical_index(-1), m_process(0), m_ready(false),
                 m_proc_number(0)
      {}
  
      ~Engine() {
          if (m_process != 0) {
              fini_process(false);
          }
      }
  
      // set indexes
      void set_indexes(int logical_index, int physical_index) {
          m_index = logical_index;
          m_physical_index = physical_index;
      }
  
      // start process on device
      void init_process();
  
      void load_libraries(void);
      void init_ptr_data(void);
  
      // performs library intialization on the device side
      pid_t init_device(void);
  
  private:
      // get pipeline associated with a calling thread
      COIPIPELINE get_pipeline(void);
  
      // get automatic vars set associated with the calling thread
      AutoSet& get_auto_vars(void);
  
      // destructor for thread data
      static void destroy_thread_data(void *data);
  
  private:
      typedef std::set<PtrData> PtrSet;
      typedef std::map<const void*, OffloadDescriptor*> SignalMap;
  
      // device indexes
      int         m_index;
      int         m_physical_index;
  
      // number of COI pipes created for the engine
      long        m_proc_number;
  
      // process handle
      COIPROCESS  m_process;
  
      // If false, device either has not been initialized or new libraries
      // have been added.
      bool        m_ready;
      mutex_t     m_lock;
  
      // List of libraries to be loaded
      TargetImageList m_images;
  
      // var table
      PtrSet      m_ptr_set;
      mutex_t     m_ptr_lock;
  
      // signals
      SignalMap m_signal_map;
      mutex_t   m_signal_lock;
  
      // constants for accessing device function handles
      enum {
          c_func_compute = 0,
  #ifdef MYO_SUPPORT
          c_func_myo_init,
          c_func_myo_fini,
  #endif // MYO_SUPPORT
          c_func_init,
          c_func_var_table_size,
          c_func_var_table_copy,
          c_funcs_total
      };
      static const char* m_func_names[c_funcs_total];
  
      // device function handles
      COIFUNCTION m_funcs[c_funcs_total];
  
      // int -> name mapping for device signals
      static const int   c_signal_max = 32;
      static const char* c_signal_names[c_signal_max];
  };
  
  #endif // OFFLOAD_ENGINE_H_INCLUDED
  