text: // Copyright 2016 The Fuchsia Authors
  // Copyright (c) 2014 Travis Geiselbrecht
  //
  // Use of this source code is governed by a MIT-style
  // license that can be found in the LICENSE file or at
  // https://opensource.org/licenses/MIT
  
  #include "vm/pmm.h"
  
  #include <assert.h>
  #include <inttypes.h>
  #include <lib/cmdline.h>
  #include <lib/console.h>
  #include <lib/counters.h>
  #include <lib/ktrace.h>
  #include <platform.h>
  #include <pow2.h>
  #include <stdlib.h>
  #include <string.h>
  #include <trace.h>
  #include <zircon/errors.h>
  #include <zircon/time.h>
  #include <zircon/types.h>
  
  #include <new>
  
  #include <fbl/auto_lock.h>
  #include <fbl/intrusive_double_list.h>
  #include <kernel/mp.h>
  #include <kernel/mutex.h>
  #include <kernel/timer.h>
  #include <ktl/algorithm.h>
  #include <lk/init.h>
  #include <vm/bootalloc.h>
  #include <vm/physmap.h>
  #include <vm/pmm_checker.h>
  #include <vm/scanner.h>
  #include <vm/vm.h>
  
  #include "pmm_arena.h"
  #include "pmm_node.h"
  #include "vm_priv.h"
  
  #define LOCAL_TRACE VM_GLOBAL_TRACE(0)
  
  // Number of bytes available in the PMM after kernel init, but before userspace init.
  KCOUNTER(boot_memory_bytes, "boot.memory.post_init_free_bytes")
  
  using LocalTraceDuration =
      TraceDuration<TraceEnabled<false>, KTRACE_GRP_SCHEDULER, TraceContext::Thread>;
  
  // The (currently) one and only pmm node
  static PmmNode pmm_node;
  
  static void pmm_fill_free_pages(uint level) { pmm_node.FillFreePagesAndArm(); }
  LK_INIT_HOOK(pmm_fill, &pmm_fill_free_pages, LK_INIT_LEVEL_VM)
  
  vm_page_t* paddr_to_vm_page(paddr_t addr) { return pmm_node.PaddrToPage(addr); }
  
  zx_status_t pmm_add_arena(const pmm_arena_info_t* info) { return pmm_node.AddArena(info); }
  
  size_t pmm_num_arenas() { return pmm_node.NumArenas(); }
  
  zx_status_t pmm_get_arena_info(size_t count, uint64_t i, pmm_arena_info_t* buffer,
                                 size_t buffer_size) {
    return pmm_node.GetArenaInfo(count, i, buffer, buffer_size);
  }
  
  zx_status_t pmm_alloc_page(uint alloc_flags, paddr_t* pa) {
    LocalTraceDuration trace{"pmm_alloc_page"_stringref};
    return pmm_node.AllocPage(alloc_flags, nullptr, pa);
  }
  
  zx_status_t pmm_alloc_page(uint alloc_flags, vm_page_t** page) {
    LocalTraceDuration trace{"pmm_alloc_page"_stringref};
    return pmm_node.AllocPage(alloc_flags, page, nullptr);
  }
  
  zx_status_t pmm_alloc_page(uint alloc_flags, vm_page_t** page, paddr_t* pa) {
    LocalTraceDuration trace{"pmm_alloc_page"_stringref};
    return pmm_node.AllocPage(alloc_flags, page, pa);
  }
  
  zx_status_t pmm_alloc_pages(size_t count, uint alloc_flags, list_node* list) {
    LocalTraceDuration trace{"pmm_alloc_pages"_stringref};
    return pmm_node.AllocPages(count, alloc_flags, list);
  }
  
  zx_status_t pmm_alloc_range(paddr_t address, size_t count, list_node* list) {
    LocalTraceDuration trace{"pmm_alloc_range"_stringref};
    return pmm_node.AllocRange(address, count, list);
  }
  
  zx_status_t pmm_alloc_contiguous(size_t count, uint alloc_flags, uint8_t alignment_log2,
                                   paddr_t* pa, list_node* list) {
    LocalTraceDuration trace{"pmm_alloc_contiguous"_stringref};
    // if we're called with a single page, just fall through to the regular allocation routine
    if (unlikely(count == 1 && alignment_log2 <= PAGE_SIZE_SHIFT)) {
      vm_page_t* page;
      zx_status_t status = pmm_node.AllocPage(alloc_flags, &page, pa);
      if (status != ZX_OK) {
        return status;
      }
      list_add_tail(list, &page->queue_node);
      return ZX_OK;
    }
  
    return pmm_node.AllocContiguous(count, alloc_flags, alignment_log2, pa, list);
  }
  
  void pmm_alloc_pages(uint alloc_flags, page_request_t* req) {
    LocalTraceDuration trace{"pmm_alloc_pages"_stringref};
    pmm_node.AllocPages(alloc_flags, req);
  }
  
  bool pmm_clear_request(page_request_t* req) { return pmm_node.ClearRequest(req); }
  
  void pmm_swap_request(page_request_t* old, page_request_t* new_req) {
    pmm_node.SwapRequest(old, new_req);
  }
  
  void pmm_free(list_node* list) {
    LocalTraceDuration trace{"pmm_free"_stringref};
    pmm_node.FreeList(list);
  }
  
  void pmm_free_page(vm_page* page) {
    LocalTraceDuration trace{"pmm_free_page"_stringref};
    pmm_node.FreePage(page);
  }
  
  uint64_t pmm_count_free_pages() { return pmm_node.CountFreePages(); }
  
  uint64_t pmm_count_total_bytes() { return pmm_node.CountTotalBytes(); }
  
  PageQueues* pmm_page_queues() { return pmm_node.GetPageQueues(); }
  
  zx_status_t pmm_init_reclamation(const uint64_t* watermarks, uint8_t watermark_count,
                                   uint64_t debounce, void* context,
                                   mem_avail_state_updated_callback_t callback) {
    return pmm_node.InitReclamation(watermarks, watermark_count, debounce, context, callback);
  }
  
  void pmm_checker_check_all_free_pages() { pmm_node.CheckAllFreePages(); }
  
  #if __has_feature(address_sanitizer)
  void pmm_asan_poison_all_free_pages() { pmm_node.PoisonAllFreePages(); }
  #endif
  
  int64_t pmm_get_alloc_failed_count() { return PmmNode::get_alloc_failed_count(); }
  
  static void pmm_checker_enable(size_t fill_size, PmmChecker::Action action) {
    // We might be changing the fill size.  If we increase the fill size while the checker is active,
    // we might spuriously assert so disable the checker.
    pmm_node.DisableChecker();
  
    // Enable filling of pages going forward.
    pmm_node.EnableFreePageFilling(fill_size, action);
  
    // From this point on, pages will be filled when they are freed.  However, the free list may still
    // have a lot of unfilled pages so make a pass over them and fill them all.
    pmm_node.FillFreePagesAndArm();
  
    // All free pages have now been filled with |fill_size| and the checker is armed.
  }
  
  static void pmm_checker_disable() { pmm_node.DisableChecker(); }
  
  static bool pmm_checker_is_enabled() { return pmm_node.Checker()->IsArmed(); }
  
  static void pmm_checker_print_status() { pmm_node.Checker()->PrintStatus(stdout); }
  
  void pmm_checker_init_from_cmdline() {
    bool enabled = gCmdline.GetBool(kernel_option::kPmmCheckerEnable, false);
    if (enabled) {
      size_t fill_size = gCmdline.GetUInt64(kernel_option::kPmmCheckerFillSize, PAGE_SIZE);
      if (!PmmChecker::IsValidFillSize(fill_size)) {
        printf("PMM: value from %s is invalid (%lu), using PAGE_SIZE instead\n",
               kernel_option::kPmmCheckerFillSize, fill_size);
        fill_size = PAGE_SIZE;
      }
  
      PmmChecker::Action action = PmmChecker::DefaultAction;
      const char* const action_string = gCmdline.GetString(kernel_option::kPmmCheckerAction);
      if (action_string != nullptr) {
        if (auto opt_action = PmmChecker::ActionFromString(action_string)) {
          action = opt_action.value();
        } else {
          printf("PMM: value from %s is invalid (\"%s\"), using \"%s\" instead\n",
                 kernel_option::kPmmCheckerAction, action_string, PmmChecker::ActionToString(action));
        }
      }
  
      pmm_node.EnableFreePageFilling(fill_size, action);
    }
  }
  
  static void pmm_dump_timer(Timer* t, zx_time_t now, void*) {
    zx_time_t deadline = zx_time_add_duration(now, ZX_SEC(1));
    t->SetOneshot(deadline, &pmm_dump_timer, nullptr);
    pmm_node.DumpFree();
  }
  
  static void init_request_thread(unsigned int level) { pmm_node.InitRequestThread(); }
  
  LK_INIT_HOOK(pmm, init_request_thread, LK_INIT_LEVEL_THREADING)
  
  LK_INIT_HOOK(
      pmm_boot_memory,
      [](unsigned int /*level*/) {
        // Track the amount of free memory available in the PMM after kernel init, but before
        // userspace starts.
        //
        // We record this in a kcounter to be tracked by build infrastructure over time.
        dprintf(INFO, "Free memory after kernel init: %" PRIu64 " bytes.\n",
                pmm_node.CountFreePages() * PAGE_SIZE);
        boot_memory_bytes.Set(pmm_node.CountFreePages() * PAGE_SIZE);
      },
      LK_INIT_LEVEL_USER - 1)
  
  static int cmd_pmm(int argc, const cmd_args* argv, uint32_t flags) {
    const bool is_panic = flags & CMD_FLAG_PANIC;
    auto usage = [cmd_name = argv[0].str, is_panic]() -> int {
      printf("usage:\n");
      printf("%s dump                                     : dump pmm info \n", cmd_name);
      if (!is_panic) {
        printf("%s free                                     : periodically dump free mem count\n",
               cmd_name);
        printf(
            "%s oom [<rate>]                             : leak memory until oom is triggered, "
            "optionally specify the rate at which to leak (in MB per second)\n",
            cmd_name);
        printf(
            "%s oom hard                                 : leak memory aggressively and keep on "
            "leaking\n",
            cmd_name);
        printf(
            "%s oom signal                               : trigger oom signal without leaking "
            "memory\n",
            cmd_name);
        printf("%s mem_avail_state info                     : dump memory availability state info\n",
               cmd_name);
        printf(
            "%s mem_avail_state [step] <state> [<nsecs>] : allocate memory to go to memstate "
            "<state>, hold the state for <nsecs> (10s by default). Only works if going to <state> "
            "from current state requires allocating memory, can't free up pre-allocated memory. In "
            "optional [step] mode, allocation pauses for 1 second at each intermediate memory "
            "availability state until <state> is reached.\n",
            cmd_name);
        printf("%s drop_user_pt                             : drop all user hardware page tables\n",
               cmd_name);
        printf("%s checker status                           : prints the status of the pmm checker\n",
               cmd_name);
        printf(
            "%s checker enable [<size>] [oops|panic]     : enables the pmm checker with optional "
            "fill size and optional action\n",
            cmd_name);
        printf("%s checker disable                          : disables the pmm checker\n", cmd_name);
        printf(
            "%s checker check                            : forces a check of all free pages in the "
            "pmm\n",
            cmd_name);
      }
      return ZX_ERR_INTERNAL;
    };
  
    if (argc < 2) {
      printf("not enough arguments\n");
      return usage();
    }
  
    if (!strcmp(argv[1].str, "dump")) {
      pmm_node.Dump(is_panic);
    } else if (is_panic) {
      // No other operations will work during a panic.
      printf("Only the \"arenas\" command is available during a panic.\n");
      return usage();
    } else if (!strcmp(argv[1].str, "free")) {
      static bool show_mem = false;
      static Timer timer;
  
      if (!show_mem) {
        printf("pmm free: issue the same command to stop.\n");
        zx_time_t deadline = zx_time_add_duration(current_time(), ZX_SEC(1));
        const TimerSlack slack{ZX_MSEC(20), TIMER_SLACK_CENTER};
        const Deadline slackDeadline(deadline, slack);
        timer.Set(slackDeadline, &pmm_dump_timer, nullptr);
        show_mem = true;
      } else {
        timer.Cancel();
        show_mem = false;
      }
    } else if (!strcmp(argv[1].str, "oom")) {
      if (argc > 3) {
        return usage();
      }
  
      uint64_t rate = 0;
      bool hard = false;
      if (argc > 2) {
        if (!strcmp(argv[2].str, "signal")) {
          pmm_node.DebugMemAvailStateCallback(0);
          return ZX_OK;
        }
        if (!strcmp(argv[2].str, "hard")) {
          hard = true;
        } else {
          rate = strtoul(argv[2].str, nullptr, 0) * 1024 * 1024 / PAGE_SIZE;
        }
      }
  
      // When we reach the oom state the kernel may 'try harder' to reclaim memory and prevent us from
      // hitting oom. To avoid this we disable the scanner to prevent additional memory from becoming
      // classified as evictable, and then evict anything that is already considered.
      printf("Disabling VM scanner\n");
      scanner_push_disable_count();
      uint64_t pages_evicted = scanner_synchronous_evict(
          UINT64_MAX, scanner::EvictionLevel::IncludeNewest, scanner::Output::NoPrint);
      if (pages_evicted > 0) {
        printf("Leaked %" PRIu64 " pages from eviction\n", pages_evicted);
      }
  
      uint64_t pages_till_oom;
      // In case we are racing with someone freeing pages we will leak in a loop until we are sure
      // we have hit the oom state.
      while ((pages_till_oom = pmm_node.DebugNumPagesTillMemState(0)) > 0) {
        list_node list = LIST_INITIAL_VALUE(list);
        if (rate > 0) {
          uint64_t pages_leaked = 0;
          while (pages_leaked < pages_till_oom) {
            uint64_t alloc_pages = ktl::min(rate, pages_till_oom - pages_leaked);
            if (pmm_node.AllocPages(alloc_pages, 0, &list) == ZX_OK) {
              pages_leaked += alloc_pages;
              printf("Leaked %lu pages\n", pages_leaked);
            }
            Thread::Current::SleepRelative(ZX_SEC(1));
          }
        } else {
          if (pmm_node.AllocPages(pages_till_oom, 0, &list) == ZX_OK) {
            printf("Leaked %lu pages\n", pages_till_oom);
          }
        }
        // Ignore any errors under the assumption we had a racy allocation and try again next time
        // around the loop.
      }
  
      if (hard) {
        printf("Continuing to leak pages forever\n");
        // Keep leaking as fast possible.
        while (true) {
          vm_page_t* p;
          pmm_alloc_page(0, &p);
        }
      }
    } else if (!strcmp(argv[1].str, "mem_avail_state")) {
      if (argc < 3) {
        return usage();
      }
      if (!strcmp(argv[2].str, "info")) {
        pmm_node.DumpMemAvailState();
      } else {
        bool step = false;
        int index = 2;
        if (!strcmp(argv[2].str, "step")) {
          step = true;
          index++;
        }
  
        uint8_t state = static_cast<uint8_t>(argv[index++].u);
        if (state > pmm_node.DebugMaxMemAvailState()) {
          printf("Invalid memstate %u. Specify a value between 0 and %u.\n", state,
                 pmm_node.DebugMaxMemAvailState());
          return usage();
        }
  
        uint64_t pages_to_alloc, pages_to_free = 0;
        list_node list = LIST_INITIAL_VALUE(list);
  
        if (step) {
          uint8_t s = pmm_node.DebugMaxMemAvailState();
          while (true) {
            // In case we are racing with someone freeing pages we will leak in a loop until we are
            // sure we have hit the required memory availability state.
            uint64_t pages_allocated = 0;
            while ((pages_to_alloc = pmm_node.DebugNumPagesTillMemState(s)) > 0) {
              if (pmm_node.AllocPages(pages_to_alloc, 0, &list) == ZX_OK) {
                printf("Leaked %lu pages\n", pages_to_alloc);
                pages_allocated += pages_to_alloc;
              }
            }
            pages_to_free += pages_allocated;
            if (s == state) {
              break;
            }
            s--;
            if (pages_allocated) {
              printf("Sleeping for 1 second...\n");
              Thread::Current::SleepRelative(ZX_SEC(1));
            }
          }
        } else {
          // In case we are racing with someone freeing pages we will leak in a loop until we are
          // sure we have hit the required memory availability state.
          while ((pages_to_alloc = pmm_node.DebugNumPagesTillMemState(state)) > 0) {
            if (pmm_node.AllocPages(pages_to_alloc, 0, &list) == ZX_OK) {
              printf("Leaked %lu pages\n", pages_to_alloc);
              pages_to_free += pages_to_alloc;
            }
          }
        }
  
        if (pages_to_free > 0) {
          uint64_t nsecs = 10;
          if (argc > index) {
            nsecs = argv[index].u;
          }
          printf("Sleeping for %lu seconds...\n", nsecs);
          Thread::Current::SleepRelative(ZX_SEC(nsecs));
          pmm_node.FreeList(&list);
          printf("Freed %lu pages\n", pages_to_free);
        }
      }
    } else if (!strcmp(argv[1].str, "drop_user_pt")) {
      VmAspace::DropAllUserPageTables();
    } else if (!strcmp(argv[1].str, "checker")) {
      if (argc < 3 || argc > 5) {
        return usage();
      }
      if (!strcmp(argv[2].str, "status")) {
        pmm_checker_print_status();
      } else if (!strcmp(argv[2].str, "enable")) {
        size_t fill_size = PAGE_SIZE;
        PmmChecker::Action action = PmmChecker::DefaultAction;
        if (argc >= 4) {
          fill_size = argv[3].u;
          if (!PmmChecker::IsValidFillSize(fill_size)) {
            printf(
                "error: fill size must be a multiple of 8 and be between 8 and PAGE_SIZE, "
                "inclusive\n");
            return ZX_ERR_INTERNAL;
          }
        }
        if (argc == 5) {
          if (auto opt_action = PmmChecker::ActionFromString(argv[4].str)) {
            action = opt_action.value();
          } else {
            printf("error: invalid action\n");
            return ZX_ERR_INTERNAL;
          }
        }
        pmm_checker_enable(fill_size, action);
        // No need to print status as enabling automatically prints status.
      } else if (!strcmp(argv[2].str, "disable")) {
        pmm_checker_disable();
        pmm_checker_print_status();
      } else if (!strcmp(argv[2].str, "check")) {
        if (!pmm_checker_is_enabled()) {
          printf("error: pmm checker is not enabled\n");
          return ZX_ERR_INTERNAL;
        }
        printf("checking all free pages...\n");
        pmm_checker_check_all_free_pages();
        printf("done\n");
      } else {
        return usage();
      }
    } else {
      printf("unknown command\n");
      return usage();
    }
  
    return ZX_OK;
  }
  
  STATIC_COMMAND_START
  STATIC_COMMAND_MASKED("pmm", "physical memory manager", &cmd_pmm, CMD_AVAIL_ALWAYS)
  STATIC_COMMAND_END(pmm)
  