text: // @COPYRIGHT@
  // Licensed under MIT license.
  // See LICENSE.TXT file in the project root for more information.
  // ==============================================================
  #ifndef __SHADOW_MEMORY_ML__
  #define __SHADOW_MEMORY_ML__
  #include<stdint.h>
  #include<atomic>
  #include<stdlib.h>
  #include<sys/mman.h>
  #include<tuple>
  // 64KB shadow pages
  #define PAGE_OFFSET_BITS_ML (16LL)
  #define PAGE_OFFSET_ML(addr) ( addr & 0xFFFF)
  #define PAGE_OFFSET_MASK_ML ( 0xFFFF)
  #define SHADOW_PAGE_SIZE_ML (1 << PAGE_OFFSET_BITS_ML)
  // 2 level page table
  #define PTR_SIZE_ML (sizeof(struct Status *))
  #define LEVEL_1_PAGE_TABLE_BITS_ML  (20)
  #define LEVEL_1_PAGE_TABLE_ENTRIES_ML  (1 << LEVEL_1_PAGE_TABLE_BITS_ML )
  #define LEVEL_1_PAGE_TABLE_SIZE_ML  (LEVEL_1_PAGE_TABLE_ENTRIES_ML * PTR_SIZE_ML )
  #define LEVEL_2_PAGE_TABLE_BITS_ML  (12)
  #define LEVEL_2_PAGE_TABLE_ENTRIES_ML  (1 << LEVEL_2_PAGE_TABLE_BITS_ML )
  #define LEVEL_2_PAGE_TABLE_SIZE_ML  (LEVEL_2_PAGE_TABLE_ENTRIES_ML * PTR_SIZE_ML )
  #define LEVEL_1_PAGE_TABLE_SLOT_ML(addr) ((((uint64_t)addr) >> (LEVEL_2_PAGE_TABLE_BITS_ML + PAGE_OFFSET_BITS_ML)) & 0xfffff)
  #define LEVEL_2_PAGE_TABLE_SLOT_ML(addr) ((((uint64_t)addr) >> (PAGE_OFFSET_BITS_ML)) & 0xFFF)
  #define SHADOW_STRUCT_SIZE_ML (sizeof (T))
  using namespace std;
  #define PIN_ExitProcess(a) exit(a)
  /*
  template <typename... Ts>
  struct ShadowType {
      tuple<Ts[SHADOW_PAGE_SIZE_ML]...> f;
      void * operator new(size_t sz) {
          void * p = mmap(0, sz, PROT_WRITE | PROT_READ, MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
          if(p == MAP_FAILED) {
              perror("mmap l2pg");
              PIN_ExitProcess(-1);
          }
          return p;
      }
      void operator delete(void *p) {
          munmap(p, sizeof(ShadowType<Ts...>));
      }
  };
  */
  template<class ...Args>
  using ShadowTuple = std::tuple<Args[SHADOW_PAGE_SIZE_ML]...>;
  #if 1
  template <typename... Ts>
  class ConcurrentShadowMemoryMl {
      // All fwd declarations
      atomic< atomic< ShadowTuple<Ts...>  *> *> * pageDirectory;
      // Given a address generated by the program, returns the corresponding shadow address FLOORED to  SHADOW_PAGE_SIZE_ML
      // If the shadow page does not exist a new one is MMAPed
  public:
      inline ConcurrentShadowMemoryMl() {
          pageDirectory = (atomic< atomic< ShadowTuple<Ts...> *> *> *) mmap(0, LEVEL_1_PAGE_TABLE_SIZE_ML, PROT_WRITE | PROT_READ, MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
          if(pageDirectory == MAP_FAILED) {
              perror("mmap pageDirectory");
              PIN_ExitProcess(-1);
          }
      }
      inline ~ConcurrentShadowMemoryMl(){
          for(uint64_t i = 0; i < LEVEL_1_PAGE_TABLE_ENTRIES_ML; i++) {
              atomic< ShadowTuple<Ts...> *> * l1Page;
              if( (l1Page=pageDirectory[i].load(memory_order_relaxed)) != 0) {
                  for(uint64_t j = 0; j < LEVEL_2_PAGE_TABLE_ENTRIES_ML; j++) {
                      ShadowTuple<Ts...> * l2Page;
                      if( (l2Page=l1Page[j].load(memory_order_relaxed)) != 0){
                          delete l2Page;
                      }
                  }
                  if(0 != munmap(l1Page, LEVEL_2_PAGE_TABLE_SIZE_ML)) {
                      perror("munmap pageDirectory");
                      PIN_ExitProcess(-1);
                  }
              }
          }
          if(0 != munmap(pageDirectory, LEVEL_1_PAGE_TABLE_SIZE_ML)){
              perror("munmap pageDirectory");
              PIN_ExitProcess(-1);
          }
      }
      inline ShadowTuple<Ts...> & GetOrCreateShadowBaseAddress(const size_t address) {
          atomic< atomic< ShadowTuple<Ts...> *> *>  * l1Ptr = & pageDirectory[LEVEL_1_PAGE_TABLE_SLOT_ML(address)];
          atomic< ShadowTuple<Ts...> *> * v1;
          if ( (v1=l1Ptr->load(memory_order_consume)) == 0) {
              atomic< ShadowTuple<Ts...>    *> * l1pg = (atomic<  ShadowTuple<Ts...> *> *) mmap(0, LEVEL_2_PAGE_TABLE_SIZE_ML, PROT_WRITE | PROT_READ, MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
              if(l1pg == MAP_FAILED) {
                  perror("mmap l1pg");
                  PIN_ExitProcess(-1);
              }
              atomic<ShadowTuple<Ts...> *> * nullVal = 0;
              if(!l1Ptr->compare_exchange_strong(nullVal, l1pg, memory_order_acq_rel, memory_order_relaxed)) {
                  free(l1pg);
                  v1 = l1Ptr->load(memory_order_consume);
              } else {
                  v1 = l1pg;
              }
          }
          atomic<ShadowTuple<Ts...> *>  * l2Ptr = & v1[LEVEL_2_PAGE_TABLE_SLOT_ML(address)];
          ShadowTuple<Ts...> * v2;
          if( (v2=l2Ptr->load(memory_order_consume)) == 0 ){
              ShadowTuple<Ts...> * l2pg = new ShadowTuple<Ts...>;
              if(l2pg == MAP_FAILED) {
                  perror("mmap l2pg");
                  PIN_ExitProcess(-1);
              }
              ShadowTuple<Ts...> * nullVal = 0;
              if( !l2Ptr->compare_exchange_strong(nullVal, l2pg, memory_order_acq_rel, memory_order_relaxed)){
                  delete l2pg;
                  v2 = l2Ptr->load(memory_order_consume);
              } else {
                  v2 = l2pg;
              }
          }
          return (*v2);
      }
  };
  template<int I, typename... Ts>
      inline __attribute__((always_inline)) typename std::tuple_element<I, tuple<Ts...> >::type * GetOrCreateShadowAddressMl(ConcurrentShadowMemoryMl<Ts...> & sm, const size_t address) {
   auto  shadowPage = sm.GetOrCreateShadowBaseAddress(address);
   return &(get<I>(shadowPage)[PAGE_OFFSET_ML((uint64_t)address)]);
  }
  #endif
  template <typename... Ts>
  class ShadowMemoryMl {
      // All fwd declarations
      ShadowTuple<Ts...> *** pageDirectory;
      // Given a address generated by the program, returns the corresponding shadow address FLOORED to  SHADOW_PAGE_SIZE_ML
      // If the shadow page does not exist a new one is MMAPed
  public:
      inline ShadowMemoryMl() {
          pageDirectory = (ShadowTuple<Ts...> ***) mmap(0, LEVEL_1_PAGE_TABLE_SIZE_ML, PROT_WRITE | PROT_READ, MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
          if(pageDirectory == MAP_FAILED) {
              perror("mmap pageDirectory");
              PIN_ExitProcess(-1);
          }
      }
      inline ~ShadowMemoryMl(){
          for(uint64_t i = 0; i < LEVEL_1_PAGE_TABLE_ENTRIES_ML; i++) {
              ShadowTuple<Ts...> ** l1Page;
              if( (l1Page=pageDirectory[i]) != 0) {
                  for(uint64_t j = 0; j < LEVEL_2_PAGE_TABLE_ENTRIES_ML; j++) {
                      ShadowTuple<Ts...> * l2Page;
                      if( (l2Page=l1Page[j]) != 0){
                          delete l2Page;
                      }
                  }
                  if(0 != munmap(l1Page, LEVEL_2_PAGE_TABLE_SIZE_ML)) {
                      perror("munmap pageDirectory");
                      PIN_ExitProcess(-1);
                  }
              }
          }
          if(0 != munmap(pageDirectory, LEVEL_1_PAGE_TABLE_SIZE_ML)){
              perror("munmap pageDirectory");
              PIN_ExitProcess(-1);
          }
      }
      inline ShadowTuple<Ts...> & GetOrCreateShadowBaseAddress(const size_t address) {
          ShadowTuple<Ts...> *** l1Ptr = & pageDirectory[LEVEL_1_PAGE_TABLE_SLOT_ML(address)];
          ShadowTuple<Ts...> ** v1;
          if ( (v1=*l1Ptr) == 0) {
              ShadowTuple<Ts...> ** l1pg = (ShadowTuple<Ts...> **) mmap(0, LEVEL_2_PAGE_TABLE_SIZE_ML, PROT_WRITE | PROT_READ, MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
              if(l1pg == MAP_FAILED) {
                  perror("mmap l1pg");
                  PIN_ExitProcess(-1);
              }
              *l1Ptr = l1pg;
              v1 = l1pg;
          }
          ShadowTuple<Ts...> ** l2Ptr = & v1[LEVEL_2_PAGE_TABLE_SLOT_ML(address)];
          ShadowTuple<Ts...> * v2;
          if( (v2=*l2Ptr) == 0 ){
              ShadowTuple<Ts...> * l2pg = new ShadowTuple<Ts...>;
              if(l2pg == MAP_FAILED) {
                  perror("mmap l2pg");
                  PIN_ExitProcess(-1);
              }
              *l2Ptr = l2pg;
              v2 = l2pg;
          }
          return (*v2);
      }
  };
  template<int I, typename... Ts>
  inline __attribute__((always_inline)) typename std::tuple_element<I, tuple<Ts...> >::type * GetOrCreateShadowAddressMl(ShadowMemoryMl<Ts...> & sm, const size_t address) {
      auto  shadowPage = sm.GetOrCreateShadowBaseAddress(address);
      return &(get<I>(shadowPage)[PAGE_OFFSET_ML((uint64_t)address)]);
  }
  //#define ConcurrentShadowMemoryMl ShadowMemoryMl
  #if 0
  ShadowMemoryMl<int> s;
  ConcurrentShadowMemoryMl <int> cs;
  int main(){
  for(int i = 0; i < 0xffff; i++){
      GetOrCreateShadowAddressMl<0>(s, 0x12345678)[0] = 1234;
      int j = GetOrCreateShadowAddressMl<0>(s, 0x12345678)[0];
      GetOrCreateShadowAddressMl<0>(cs, 0x12345678)[0] = 1234;
      j = GetOrCreateShadowAddressMl<0>(cs, 0x12345678)[0];
  }
      return 0;
  }
  #endif
  #endif // __SHADOW_MEMORY_ML__
  