text: /* *******************************************************************************
   * Copyright (c) 2019-2020 Google, Inc.  All rights reserved.
   * *******************************************************************************/
  
  /*
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions are met:
   *
   * * Redistributions of source code must retain the above copyright notice,
   *   this list of conditions and the following disclaimer.
   *
   * * Redistributions in binary form must reproduce the above copyright notice,
   *   this list of conditions and the following disclaimer in the documentation
   *   and/or other materials provided with the distribution.
   *
   * * Neither the name of Google, Inc. nor the names of its contributors may be
   *   used to endorse or promote products derived from this software without
   *   specific prior written permission.
   *
   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
   * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   * DAMAGE.
   */
  
  /**************************************************************************************
   * Restartable sequence ("rseq") support (i#2350).
   * This is a kernel feature which provides cpu-atomic regions: if a thread
   * is pre-empted within an rseq region, an abort handler is invoked.
   * The feature is difficult to handle under binary instrumentation.
   * We rely on the app following certain conventions, including containing a
   * section holding a table of all rseq sequences.
   */
  
  #include "../globals.h"
  #include "../module_shared.h"
  #include "module_private.h"
  #include "os_private.h"
  #include "rseq_linux.h"
  #include "../fragment.h"
  #include "decode.h"
  #ifdef CLIENT_INTERFACE
  #    include "instrument.h"
  #endif
  #include <stddef.h>
  #ifdef HAVE_RSEQ
  #    include <linux/rseq.h>
  #else
  struct rseq_cs {
      uint version;
      uint flags;
      uint64 start_ip;
      uint64 post_commit_offset;
      uint64 abort_ip;
  } __attribute__((aligned(4 * sizeof(uint64))));
  struct rseq {
      uint cpu_id_start;
      uint cpu_id;
      union {
          uint64 ptr64;
      } rseq_cs;
      uint flags;
  } __attribute__((aligned(4 * sizeof(uint64))));
  #    define RSEQ_FLAG_UNREGISTER 1
  #endif
  #include "include/syscall.h"
  #include <errno.h>
  
  vm_area_vector_t *d_r_rseq_areas;
  DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock,
                        INIT_LOCK_FREE(rseq_trigger_lock));
  static volatile bool rseq_enabled;
  
  /* We require all threads to use the same TLS offset to point at struct rseq. */
  static int rseq_tls_offset;
  
  /* The signature is registered per thread, but we require all registrations
   * to be the same.
   */
  static int rseq_signature;
  
  typedef struct _rseq_region_t {
      app_pc start;
      app_pc end;
      app_pc handler;
      /* We need to preserve input registers for targeting "start" instead of "handler"
       * for our 2nd invocation, if they're written in the rseq region.  We only support
       * GPR inputs.  We document that we do not support any other inputs (no flags, no
       * SIMD registers).
       */
      bool reg_written[DR_NUM_GPR_REGS];
  } rseq_region_t;
  
  /* We need to store a struct rseq_cs per fragment_t.  To avoid the cost of adding a
   * pointer field to every fragment_t, and the complexity of another subclass like
   * trace_t, we store them externally in a hashtable.  The FRAG_HAS_RSEQ_ENDPOINT flag
   * avoids the hashtable lookup on every fragment.
   */
  static generic_table_t *rseq_cs_table;
  #define INIT_RSEQ_CS_TABLE_SIZE 5
  
  /* vmvector callbacks */
  static void
  rseq_area_free(void *data)
  {
      HEAP_TYPE_FREE(GLOBAL_DCONTEXT, data, rseq_region_t, ACCT_VMAREAS, PROTECTED);
  }
  
  static void *
  rseq_area_dup(void *data)
  {
      rseq_region_t *src = (rseq_region_t *)data;
      rseq_region_t *dst =
          HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, rseq_region_t, ACCT_VMAREAS, PROTECTED);
      ASSERT(src != NULL);
      *dst = *src;
      return dst;
  }
  
  static inline size_t
  rseq_cs_alloc_size(void)
  {
      return sizeof(struct rseq) + __alignof(struct rseq_cs);
  }
  
  static void
  rseq_cs_free(dcontext_t *dcontext, void *data)
  {
      global_heap_free(data, rseq_cs_alloc_size() HEAPACCT(ACCT_OTHER));
  }
  
  void
  d_r_rseq_init(void)
  {
      VMVECTOR_ALLOC_VECTOR(d_r_rseq_areas, GLOBAL_DCONTEXT,
                            VECTOR_SHARED | VECTOR_NEVER_MERGE, rseq_areas);
      vmvector_set_callbacks(d_r_rseq_areas, rseq_area_free, rseq_area_dup, NULL, NULL);
  
      rseq_cs_table = generic_hash_create(GLOBAL_DCONTEXT, INIT_RSEQ_CS_TABLE_SIZE, 80,
                                          HASHTABLE_SHARED | HASHTABLE_PERSISTENT,
                                          rseq_cs_free _IF_DEBUG("rseq_cs table"));
      /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */
      if (rseq_is_registered_for_current_thread())
          rseq_locate_rseq_regions();
  }
  
  void
  d_r_rseq_exit(void)
  {
      generic_hash_destroy(GLOBAL_DCONTEXT, rseq_cs_table);
      vmvector_delete_vector(GLOBAL_DCONTEXT, d_r_rseq_areas);
      DELETE_LOCK(rseq_trigger_lock);
  }
  
  void
  rseq_thread_attach(dcontext_t *dcontext)
  {
      rseq_region_t *info;
      if (!vmvector_lookup_data(d_r_rseq_areas, dcontext->next_tag, NULL, NULL,
                                (void **)&info))
          return;
      /* The thread missed the save of its state on rseq entry.  We could try to save here
       * so the restore on rseq exit won't read incorrect values, but it's simpler and
       * less error-prone to send it to the abort handler, like we do on detach or other
       * translation points.
       */
      dcontext->next_tag = info->handler;
  }
  
  bool
  rseq_get_region_info(app_pc pc, app_pc *start OUT, app_pc *end OUT, app_pc *handler OUT,
                       bool **reg_written OUT, int *reg_written_size OUT)
  {
      rseq_region_t *info;
      if (!vmvector_lookup_data(d_r_rseq_areas, pc, start, end, (void **)&info))
          return false;
      if (handler != NULL)
          *handler = info->handler;
      if (reg_written != NULL)
          *reg_written = info->reg_written;
      if (reg_written_size != NULL)
          *reg_written_size = sizeof(info->reg_written) / sizeof(info->reg_written[0]);
      return true;
  }
  
  int
  rseq_get_tls_ptr_offset(void)
  {
      /* This read is assumed to be atomic. */
      ASSERT(rseq_tls_offset != 0);
      return rseq_tls_offset + offsetof(struct rseq, rseq_cs);
  }
  
  static void
  rseq_clear_tls_ptr(dcontext_t *dcontext)
  {
      ASSERT(rseq_tls_offset != 0);
      byte *base = get_segment_base(LIB_SEG_TLS);
      struct rseq *app_rseq = (struct rseq *)(base + rseq_tls_offset);
      /* We're directly writing this in the cache, so we do not bother with safe_read
       * or safe_write here either.  We already cannot handle rseq adversarial cases.
       */
      if (is_dynamo_address((byte *)(ptr_uint_t)app_rseq->rseq_cs.ptr64))
          app_rseq->rseq_cs.ptr64 = 0;
  }
  
  int
  rseq_get_signature(void)
  {
      /* This is only called after rseq is initialized and the signature determined. */
      ASSERT(rseq_enabled);
      return rseq_signature;
  }
  
  byte *
  rseq_get_rseq_cs_alloc(byte **rseq_cs_aligned OUT)
  {
      byte *rseq_cs_alloc = global_heap_alloc(rseq_cs_alloc_size() HEAPACCT(ACCT_OTHER));
      *rseq_cs_aligned = (byte *)ALIGN_FORWARD(rseq_cs_alloc, __alignof(struct rseq_cs));
      return rseq_cs_alloc;
  }
  
  void
  rseq_record_rseq_cs(byte *rseq_cs_alloc, fragment_t *f, cache_pc start, cache_pc end,
                      cache_pc abort)
  {
      struct rseq_cs *target =
          (struct rseq_cs *)ALIGN_FORWARD(rseq_cs_alloc, __alignof(struct rseq_cs));
      target->version = 0;
      target->flags = 0;
      target->start_ip = (ptr_uint_t)start;
      target->post_commit_offset = (ptr_uint_t)(end - start);
      target->abort_ip = (ptr_uint_t)abort;
      TABLE_RWLOCK(rseq_cs_table, write, lock);
      generic_hash_add(GLOBAL_DCONTEXT, rseq_cs_table, (ptr_uint_t)f, rseq_cs_alloc);
      TABLE_RWLOCK(rseq_cs_table, write, unlock);
  }
  
  void
  rseq_remove_fragment(dcontext_t *dcontext, fragment_t *f)
  {
      if (!rseq_enabled)
          return;
      /* Avoid freeing a live rseq_cs for a thread-private fragment deletion. */
      rseq_clear_tls_ptr(dcontext);
      TABLE_RWLOCK(rseq_cs_table, write, lock);
      generic_hash_remove(GLOBAL_DCONTEXT, rseq_cs_table, (ptr_uint_t)f);
      TABLE_RWLOCK(rseq_cs_table, write, unlock);
  }
  
  void
  rseq_shared_fragment_flushtime_update(dcontext_t *dcontext)
  {
      if (!rseq_enabled)
          return;
      /* Avoid freeing a live rseq_cs for thread-shared fragment deletion.
       * We clear the pointer on completion of the native rseq execution, but it's
       * not easy to clear it on midpoint exits.  We instead clear prior to
       * rseq_cs being freed: for thread-private in rseq_remove_fragment() and for
       * thread-shared each thread should come here prior to deletion.
       */
      rseq_clear_tls_ptr(dcontext);
  }
  
  bool
  rseq_is_registered_for_current_thread(void)
  {
      /* Unfortunately there's no way to query the current rseq struct.
       * For 64-bit we can pass a kernel address and look for EFAULT
       * vs EINVAL, but there is no kernel address for 32-bit.
       * So we try to perform a legitimate registration.
       */
      struct rseq test_rseq = {};
      int res = dynamorio_syscall(SYS_rseq, 4, &test_rseq, sizeof(test_rseq), 0, 0);
      if (res == -EINVAL) /* Our struct != registered struct. */
          return true;
      if (res == -ENOSYS)
          return false;
      /* If seccomp blocks SYS_rseq we'll get -EPERM.  SYS_rseq also returns -EPERM
       * if &test_rseq == the app's struct but the signature is different, but that
       * seems so unlikely that we just assume -EPERM implies seccomp.
       */
      if (res == -EPERM)
          return false;
      ASSERT(res == 0); /* If not, the struct size or sthg changed! */
      if (dynamorio_syscall(SYS_rseq, 4, &test_rseq, sizeof(test_rseq),
                            RSEQ_FLAG_UNREGISTER, 0) != 0) {
          ASSERT_NOT_REACHED();
      }
      return false;
  }
  
  static void
  rseq_analyze_instructions(rseq_region_t *info)
  {
      /* We analyze the instructions inside [start,end) looking for register state that we
       * need to preserve for our restart.  We do not want to blindly spill and restore
       * 16+ registers for every sequence (too much overhead).
       */
      instr_t instr;
      instr_init(GLOBAL_DCONTEXT, &instr);
      app_pc pc = info->start;
      int i;
      bool reached_cti = false;
      memset(info->reg_written, 0, sizeof(info->reg_written));
      while (pc < info->end) {
          instr_reset(GLOBAL_DCONTEXT, &instr);
          app_pc next_pc = decode(GLOBAL_DCONTEXT, pc, &instr);
          if (next_pc == NULL) {
              REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3,
                                          get_application_name(), get_application_pid(),
                                          "Rseq sequence contains invalid instructions");
              ASSERT_NOT_REACHED();
          }
          if (instr_is_syscall(&instr)
              /* Allow a syscall for our test in debug build. */
              IF_DEBUG(
                  &&!check_filter("api.rseq;linux.rseq;linux.rseq_table;linux.rseq_noarray",
                                  get_short_name(get_application_name())))) {
              REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3,
                                          get_application_name(), get_application_pid(),
                                          "Rseq sequence contains a system call");
              ASSERT_NOT_REACHED();
          }
          if (instr_is_call(&instr)) {
              REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3,
                                          get_application_name(), get_application_pid(),
                                          "Rseq sequence contains a call");
              ASSERT_NOT_REACHED();
          }
          if (instr_is_cti(&instr))
              reached_cti = true;
          /* We potentially need to preserve any register written anywhere inside
           * the sequence.  We can't limit ourselves to registers clearly live on
           * input, since code *after* the sequence could read them.  We do disallow
           * callouts to helper functions to simplify our lives.
           *
           * We only preserve GPR's, for simplicity, and because they are far more likely
           * as inputs than flags or SIMD registers.  We'd like to verify that only GPR's
           * are used, but A) we can't easily check values read *after* the sequence (the
           * handler could set up state read afterward and sometimes clobbered inside), B)
           * we do want to support SIMD and flags writes in the sequence, and C) even
           * checking for values read in the sequence would want new interfaces like
           * DR_REG_START_SIMD or register iterators for reasonable code.
           */
          for (i = 0; i < DR_NUM_GPR_REGS; i++) {
              if (info->reg_written[i])
                  continue;
              reg_id_t reg = DR_REG_START_GPR + (reg_id_t)i;
              if (instr_writes_to_reg(&instr, reg, DR_QUERY_DEFAULT)) {
                  LOG(GLOBAL, LOG_LOADER, 3,
                      "Rseq region @" PFX " writes register %s at " PFX "\n", info->start,
                      reg_names[reg], pc);
                  info->reg_written[i] = true;
              }
          }
          pc = next_pc;
      }
      instr_free(GLOBAL_DCONTEXT, &instr);
  }
  
  static void
  rseq_process_entry(struct rseq_cs *entry, ssize_t load_offs)
  {
      LOG(GLOBAL, LOG_LOADER, 2,
          "Found rseq region: ver=%u; flags=%u; start=" PFX "; end=" PFX "; abort=" PFX
          "\n",
          entry->version, entry->flags, entry->start_ip + load_offs,
          entry->start_ip + entry->post_commit_offset + load_offs,
          entry->abort_ip + load_offs);
      rseq_region_t *info =
          HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, rseq_region_t, ACCT_VMAREAS, PROTECTED);
      info->start = (app_pc)(ptr_uint_t)entry->start_ip + load_offs;
      info->end = info->start + entry->post_commit_offset;
      info->handler = (app_pc)(ptr_uint_t)entry->abort_ip + load_offs;
      int signature;
      if (!d_r_safe_read(info->handler - sizeof(signature), sizeof(signature),
                         &signature)) {
          REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(),
                                      get_application_pid(),
                                      "Rseq signature is unreadable");
          ASSERT_NOT_REACHED();
      }
      if (signature != rseq_signature) {
          if (rseq_signature == 0) {
              SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
              ATOMIC_4BYTE_WRITE(&rseq_signature, signature, false);
              SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
              LOG(GLOBAL, LOG_LOADER, 2, "Rseq signature is 0x%08x\n", rseq_signature);
          } else {
              REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3,
                                          get_application_name(), get_application_pid(),
                                          "Rseq signatures are not all identical");
              ASSERT_NOT_REACHED();
          }
      }
      rseq_analyze_instructions(info);
      vmvector_add(d_r_rseq_areas, info->start, info->end, (void *)info);
      RSTATS_INC(num_rseq_regions);
      /* Check the start pc.  We don't take the effort to check for non-tags or
       * interior pc's.
       */
      if (fragment_lookup(GLOBAL_DCONTEXT, info->start) != NULL) {
          /* We rely on the app not running rseq code for non-rseq purposes (since we
           * can't easily tell the difference; plus we avoid a flush for lazy rseq
           * activation).
           */
          REPORT_FATAL_ERROR_AND_EXIT(
              RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), get_application_pid(),
              "Rseq sequences must not be used for non-rseq purposes");
          ASSERT_NOT_REACHED();
      }
  }
  
  static void
  rseq_process_elf_sections(module_area_t *ma, bool at_map,
                            ELF_SECTION_HEADER_TYPE *sec_hdr_start, const char *strtab,
                            ssize_t load_offs)
  {
      bool found_array = false;
      uint i;
      ELF_HEADER_TYPE *elf_hdr = (ELF_HEADER_TYPE *)ma->start;
      ELF_SECTION_HEADER_TYPE *sec_hdr = sec_hdr_start;
      /* The section entries on disk need load_offs.  The rseq entries in memory are
       * relocated and only need the offset if relocations have not yet been applied.
       */
      ssize_t entry_offs = 0;
      if (at_map || (DYNAMO_OPTION(early_inject) && !dr_api_entry && !dynamo_started))
          entry_offs = load_offs;
      for (i = 0; i < elf_hdr->e_shnum; i++) {
  #define RSEQ_PTR_ARRAY_SEC_NAME "__rseq_cs_ptr_array"
          if (strcmp(strtab + sec_hdr->sh_name, RSEQ_PTR_ARRAY_SEC_NAME) == 0) {
              found_array = true;
              byte **ptrs = (byte **)(sec_hdr->sh_addr + load_offs);
              int j;
              for (j = 0; j < sec_hdr->sh_size / sizeof(ptrs); ++j) {
                  /* We require that the table is loaded.  If not, bail, but unlike
                   * failing to find section headers, make this a fatal error: better
                   * to notify the user than try to run the rseq w/o proper handling.
                   */
                  if (ptrs < (byte **)ma->start || ptrs > (byte **)ma->end) {
                      REPORT_FATAL_ERROR_AND_EXIT(
                          RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(),
                          get_application_pid(),
                          RSEQ_PTR_ARRAY_SEC_NAME " is not in a loaded segment");
                      ASSERT_NOT_REACHED();
                  }
                  /* We assume this is a full mapping and it's safe to read the data
                   * (a partial map shouldn't make it to module list processing).
                   * We do perform a sanity check to handle unusual non-relocated
                   * cases (it's possible this array is not in a loaded segment?).
                   */
                  byte *entry = *ptrs + entry_offs;
                  if (entry < ma->start || entry > ma->end) {
                      REPORT_FATAL_ERROR_AND_EXIT(
                          RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(),
                          get_application_pid(),
                          RSEQ_PTR_ARRAY_SEC_NAME "'s entries are not in a loaded segment");
                      ASSERT_NOT_REACHED();
                  }
                  rseq_process_entry((struct rseq_cs *)entry, entry_offs);
                  ++ptrs;
              }
              break;
          }
          ++sec_hdr;
      }
      if (!found_array) {
          sec_hdr = sec_hdr_start;
          for (i = 0; i < elf_hdr->e_shnum; i++) {
  #define RSEQ_SEC_NAME "__rseq_cs"
  #define RSEQ_OLD_SEC_NAME "__rseq_table"
              if (strcmp(strtab + sec_hdr->sh_name, RSEQ_SEC_NAME) == 0 ||
                  strcmp(strtab + sec_hdr->sh_name, RSEQ_OLD_SEC_NAME) == 0) {
                  /* There may be padding at the start of the section, so ensure we skip
                   * over it.  We're reading the loaded data, not the file, so it will
                   * always be aligned.
                   */
  #define RSEQ_CS_ALIGNMENT (4 * sizeof(__u64))
                  struct rseq_cs *array = (struct rseq_cs *)ALIGN_FORWARD(
                      sec_hdr->sh_addr + load_offs, RSEQ_CS_ALIGNMENT);
                  int j;
                  for (j = 0; j < sec_hdr->sh_size / sizeof(*array); ++j) {
                      /* We require that the table is loaded.  If not, bail. */
                      if (array < (struct rseq_cs *)ma->start ||
                          array > (struct rseq_cs *)ma->end) {
                          REPORT_FATAL_ERROR_AND_EXIT(
                              RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(),
                              get_application_pid(),
                              RSEQ_SEC_NAME " is not in a loaded segment");
                          ASSERT_NOT_REACHED();
                      }
                      rseq_process_entry(array, entry_offs);
                      ++array;
                  }
                  break;
              }
              ++sec_hdr;
          }
      }
  }
  
  /* Returns whether successfully searched for rseq data (not whether found rseq data). */
  static bool
  rseq_process_module(module_area_t *ma, bool at_map)
  {
      bool res = false;
      ASSERT(is_elf_so_header(ma->start, ma->end - ma->start));
      ELF_HEADER_TYPE *elf_hdr = (ELF_HEADER_TYPE *)ma->start;
      ASSERT(elf_hdr->e_shentsize == sizeof(ELF_SECTION_HEADER_TYPE));
      int fd = INVALID_FILE;
      byte *sec_map = NULL, *str_map = NULL;
      size_t sec_size = 0, str_size = 0;
      ELF_SECTION_HEADER_TYPE *sec_hdr = NULL;
      char *strtab;
      ssize_t load_offs = ma->start - ma->os_data.base_address;
      if (at_map && elf_hdr->e_shoff + ma->start < ma->end) {
          sec_map = elf_hdr->e_shoff + ma->start;
          sec_hdr = (ELF_SECTION_HEADER_TYPE *)sec_map;
          /* We assume strtab is there too. */
          strtab = (char *)(ma->start + sec_hdr[elf_hdr->e_shstrndx].sh_offset);
          if (strtab > (char *)ma->end)
              goto rseq_process_module_cleanup;
      } else {
          /* The section headers are not mapped in.  Unfortunately this is the common
           * case: they are typically at the end of the file.  For this reason, we delay
           * calling this function until we see the app use rseq.
           */
          if (ma->full_path == NULL)
              goto rseq_process_module_cleanup;
          fd = os_open(ma->full_path, OS_OPEN_READ);
          if (fd == INVALID_FILE)
              goto rseq_process_module_cleanup;
          off_t offs = ALIGN_BACKWARD(elf_hdr->e_shoff, PAGE_SIZE);
          sec_size =
              ALIGN_FORWARD(elf_hdr->e_shoff + elf_hdr->e_shnum * elf_hdr->e_shentsize,
                            PAGE_SIZE) -
              offs;
          sec_map =
              os_map_file(fd, &sec_size, offs, NULL, MEMPROT_READ, MAP_FILE_COPY_ON_WRITE);
          if (sec_map == NULL)
              goto rseq_process_module_cleanup;
          sec_hdr = (ELF_SECTION_HEADER_TYPE *)(sec_map + elf_hdr->e_shoff - offs);
          /* We also need the section header string table. */
          offs = ALIGN_BACKWARD(sec_hdr[elf_hdr->e_shstrndx].sh_offset, PAGE_SIZE);
          str_size = ALIGN_FORWARD(sec_hdr[elf_hdr->e_shstrndx].sh_offset +
                                       sec_hdr[elf_hdr->e_shstrndx].sh_size,
                                   PAGE_SIZE) -
              offs;
          str_map =
              os_map_file(fd, &str_size, offs, NULL, MEMPROT_READ, MAP_FILE_COPY_ON_WRITE);
          if (str_map == NULL)
              goto rseq_process_module_cleanup;
          strtab = (char *)(str_map + sec_hdr[elf_hdr->e_shstrndx].sh_offset - offs);
      }
      rseq_process_elf_sections(ma, at_map, sec_hdr, strtab, load_offs);
      res = true;
  rseq_process_module_cleanup:
      if (str_size != 0)
          os_unmap_file(str_map, str_size);
      if (sec_size != 0)
          os_unmap_file(sec_map, sec_size);
      if (fd != INVALID_FILE)
          os_close(fd);
      DODEBUG({
          if (!res) {
              const char *name = GET_MODULE_NAME(&ma->names);
              if (name == NULL)
                  name = "(null)";
              LOG(GLOBAL, LOG_INTERP | LOG_VMAREAS, 2,
                  "%s: error looking for rseq table in %s\n", __FUNCTION__, name);
              if (strstr(name, "linux-vdso.so") == NULL) {
                  SYSLOG_INTERNAL_WARNING_ONCE(
                      "Failed to identify whether a module has an rseq table");
              }
          }
      });
      return res;
  }
  
  static int
  rseq_locate_tls_offset(void)
  {
      /* We assume (and document) that the loader's static TLS is used, so every thread
       * has a consistent %fs:-offs address.  Unfortunately, using a local copy of the
       * rseq code for our non-instrumented execution requires us to locate the app's
       * struct using heuristics, because the system call was poorly designed and will not
       * let us replace the app's. Alternatives of no local copy have worse problems.
       */
      /* Static TLS is at a negative offset from the app library segment base.  We simply
       * search all possible aligned slots.  Typically there are <64 possible slots.
       */
      int offset = 0;
      byte *addr = get_app_segment_base(LIB_SEG_TLS);
      byte *seg_bottom;
      if (addr > 0 && get_memory_info(addr, &seg_bottom, NULL, NULL)) {
          LOG(GLOBAL, LOG_LOADER, 3, "rseq within static TLS " PFX " - " PFX "\n",
              seg_bottom, addr);
          /* struct rseq_cs is aligned to 32. */
          int alignment = __alignof(struct rseq_cs);
          int i;
          for (i = 0; addr - i * alignment >= seg_bottom; i++) {
              byte *try_addr = addr - i * alignment;
              ASSERT(try_addr >= seg_bottom); /* For loop guarantees this. */
              /* Our strategy is to check all of the aligned static TLS addresses to
               * find the registered one.  Our caller is not supposed to call here
               * until the app has registered the current thread.
               */
              static const int RSEQ_RARE_SIGNATURE = 42;
              int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq),
                                          RSEQ_FLAG_UNREGISTER, RSEQ_RARE_SIGNATURE);
              LOG(GLOBAL, LOG_LOADER, 3, "Tried rseq @ " PFX " => %d\n", try_addr, res);
              if (res == -EINVAL) /* Our struct != registered struct. */
                  continue;
              /* We expect -EPERM on a signature mismatch.  On the small chance the app
               * actually used 42 for its signature we'll have to re-register it.
               */
              if (res == 0) {
                  int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), 0,
                                              RSEQ_RARE_SIGNATURE);
                  ASSERT(res == 0);
                  res = -EPERM;
              }
              if (res == -EPERM) {
                  /* Found it! */
                  LOG(GLOBAL, LOG_LOADER, 2,
                      "Found struct rseq @ " PFX " for thread => %s:-0x%x\n", try_addr,
                      get_register_name(LIB_SEG_TLS), i * alignment);
                  offset = -i * alignment;
              }
              break;
          }
      }
      return offset;
  }
  
  void
  rseq_process_syscall(dcontext_t *dcontext)
  {
      byte *seg_base = get_app_segment_base(LIB_SEG_TLS);
      byte *app_addr = (byte *)dcontext->sys_param0;
      bool constant_offset = false;
      if (rseq_tls_offset == 0) {
          SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
          int offset = app_addr - seg_base;
          /* To handle races here, we use an atomic_exchange. */
          int prior = atomic_exchange_int(&rseq_tls_offset, offset);
          SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
          constant_offset = (prior == 0 || prior == offset);
          LOG(GLOBAL, LOG_LOADER, 2,
              "Observed struct rseq @ " PFX " for thread => %s:-0x%x\n", app_addr,
              get_register_name(LIB_SEG_TLS), -rseq_tls_offset);
      } else
          constant_offset = (seg_base + rseq_tls_offset == app_addr);
      if (!constant_offset) {
          REPORT_FATAL_ERROR_AND_EXIT(
              RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), get_application_pid(),
              "struct rseq is not always in static thread-local storage");
          ASSERT_NOT_REACHED();
      }
  }
  
  /* Restartable sequence region identification.
   *
   * To avoid extra overhead going to disk to read section headers, we delay looking
   * for rseq data until the app invokes an rseq syscall (or on attach we see a thread
   * that has rseq set up).  We document that we do not handle the app using rseq
   * regions for non-rseq purposes, so we do not need to flush the cache here.
   * Since we also identify the rseq_cs address here, this should be called *after*
   * the app has registered the current thread for rseq.
   */
  void
  rseq_locate_rseq_regions(void)
  {
      if (rseq_enabled)
          return;
      /* This is a global operation, but the trigger could be hit by two threads at once,
       * thus requiring synchronization.
       */
      d_r_mutex_lock(&rseq_trigger_lock);
      if (rseq_enabled) {
          d_r_mutex_unlock(&rseq_trigger_lock);
          return;
      }
  
      int offset = 0;
      if (rseq_tls_offset == 0) {
          /* Identify the TLS offset of this thread's struct rseq. */
          offset = rseq_locate_tls_offset();
          if (offset == 0) {
              REPORT_FATAL_ERROR_AND_EXIT(
                  RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(),
                  get_application_pid(),
                  "struct rseq is not in static thread-local storage");
              ASSERT_NOT_REACHED();
          }
      }
  
      SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
      bool new_value = true;
      ATOMIC_1BYTE_WRITE(&rseq_enabled, new_value, false);
      if (rseq_tls_offset == 0)
          ATOMIC_4BYTE_WRITE(&rseq_tls_offset, offset, false);
      SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
  
      module_iterator_t *iter = module_iterator_start();
      while (module_iterator_hasnext(iter)) {
          module_area_t *ma = module_iterator_next(iter);
          rseq_process_module(ma, false /*!at_map*/);
      }
      module_iterator_stop(iter);
      d_r_mutex_unlock(&rseq_trigger_lock);
  }
  
  void
  rseq_module_init(module_area_t *ma, bool at_map)
  {
      if (rseq_enabled) {
          rseq_process_module(ma, at_map);
      }
  }
  
  void
  rseq_process_native_abort(dcontext_t *dcontext)
  {
  #ifdef CLIENT_INTERFACE
      /* Raise a transfer event. */
      LOG(THREAD, LOG_INTERP | LOG_VMAREAS, 2, "Abort triggered in rseq native code\n");
      get_mcontext(dcontext)->pc = dcontext->next_tag;
      if (instrument_kernel_xfer(dcontext, DR_XFER_RSEQ_ABORT, osc_empty,
                                 /* We do not know the source PC so we do not
                                  * supply a source state.
                                  */
                                 NULL, NULL, dcontext->next_tag,
                                 get_mcontext(dcontext)->xsp, osc_empty,
                                 get_mcontext(dcontext), 0)) {
          dcontext->next_tag = canonicalize_pc_target(dcontext, get_mcontext(dcontext)->pc);
      }
  #endif
  }
  