text: /*
   * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
   *
   * SPDX-License-Identifier: BSD-2-Clause
   */
  
  #pragma once
  
  #include <AK/Atomic.h>
  #include <AK/Badge.h>
  #include <AK/Concepts.h>
  #include <AK/Function.h>
  #include <AK/Noncopyable.h>
  #include <AK/Vector.h>
  
  #include <Kernel/Arch/x86/DescriptorTable.h>
  #include <Kernel/Arch/x86/TSS.h>
  
  #include <Kernel/PhysicalAddress.h>
  #include <Kernel/VirtualAddress.h>
  #include <LibC/sys/arch/i386/regs.h>
  
  #define READONLY_AFTER_INIT __attribute__((section(".ro_after_init")))
  #define UNMAP_AFTER_INIT NEVER_INLINE __attribute__((section(".unmap_after_init")))
  
  #define PAGE_SIZE 4096
  #define GENERIC_INTERRUPT_HANDLERS_COUNT (256 - IRQ_VECTOR_BASE)
  #define PAGE_MASK ((FlatPtr)0xfffff000u)
  
  namespace Kernel {
  
  class MemoryManager;
  class PageDirectory;
  class PageTableEntry;
  
  static constexpr u32 safe_eflags_mask = 0xdff;
  static constexpr u32 iopl_mask = 3u << 12;
  
  inline u32 get_iopl_from_eflags(u32 eflags)
  {
      return (eflags & iopl_mask) >> 12;
  }
  
  class PageDirectoryEntry {
  public:
      const PageTableEntry* page_table_base() const { return reinterpret_cast<PageTableEntry*>(m_raw & 0xfffff000u); }
      PageTableEntry* page_table_base() { return reinterpret_cast<PageTableEntry*>(m_raw & 0xfffff000u); }
      void set_page_table_base(u32 value)
      {
          m_raw &= 0x8000000000000fffULL;
          m_raw |= value & 0xfffff000;
      }
  
      bool is_null() const { return m_raw == 0; }
      void clear() { m_raw = 0; }
  
      u64 raw() const { return m_raw; }
      void copy_from(Badge<PageDirectory>, const PageDirectoryEntry& other) { m_raw = other.m_raw; }
  
      enum Flags {
          Present = 1 << 0,
          ReadWrite = 1 << 1,
          UserSupervisor = 1 << 2,
          WriteThrough = 1 << 3,
          CacheDisabled = 1 << 4,
          Huge = 1 << 7,
          Global = 1 << 8,
          NoExecute = 0x8000000000000000ULL,
      };
  
      bool is_present() const { return raw() & Present; }
      void set_present(bool b) { set_bit(Present, b); }
  
      bool is_user_allowed() const { return raw() & UserSupervisor; }
      void set_user_allowed(bool b) { set_bit(UserSupervisor, b); }
  
      bool is_huge() const { return raw() & Huge; }
      void set_huge(bool b) { set_bit(Huge, b); }
  
      bool is_writable() const { return raw() & ReadWrite; }
      void set_writable(bool b) { set_bit(ReadWrite, b); }
  
      bool is_write_through() const { return raw() & WriteThrough; }
      void set_write_through(bool b) { set_bit(WriteThrough, b); }
  
      bool is_cache_disabled() const { return raw() & CacheDisabled; }
      void set_cache_disabled(bool b) { set_bit(CacheDisabled, b); }
  
      bool is_global() const { return raw() & Global; }
      void set_global(bool b) { set_bit(Global, b); }
  
      bool is_execute_disabled() const { return raw() & NoExecute; }
      void set_execute_disabled(bool b) { set_bit(NoExecute, b); }
  
      void set_bit(u64 bit, bool value)
      {
          if (value)
              m_raw |= bit;
          else
              m_raw &= ~bit;
      }
  
  private:
      u64 m_raw;
  };
  
  class PageTableEntry {
  public:
      void* physical_page_base() { return reinterpret_cast<void*>(m_raw & 0xfffff000u); }
      void set_physical_page_base(u32 value)
      {
          m_raw &= 0x8000000000000fffULL;
          m_raw |= value & 0xfffff000;
      }
  
      u64 raw() const { return (u32)m_raw; }
  
      enum Flags {
          Present = 1 << 0,
          ReadWrite = 1 << 1,
          UserSupervisor = 1 << 2,
          WriteThrough = 1 << 3,
          CacheDisabled = 1 << 4,
          Global = 1 << 8,
          NoExecute = 0x8000000000000000ULL,
      };
  
      bool is_present() const { return raw() & Present; }
      void set_present(bool b) { set_bit(Present, b); }
  
      bool is_user_allowed() const { return raw() & UserSupervisor; }
      void set_user_allowed(bool b) { set_bit(UserSupervisor, b); }
  
      bool is_writable() const { return raw() & ReadWrite; }
      void set_writable(bool b) { set_bit(ReadWrite, b); }
  
      bool is_write_through() const { return raw() & WriteThrough; }
      void set_write_through(bool b) { set_bit(WriteThrough, b); }
  
      bool is_cache_disabled() const { return raw() & CacheDisabled; }
      void set_cache_disabled(bool b) { set_bit(CacheDisabled, b); }
  
      bool is_global() const { return raw() & Global; }
      void set_global(bool b) { set_bit(Global, b); }
  
      bool is_execute_disabled() const { return raw() & NoExecute; }
      void set_execute_disabled(bool b) { set_bit(NoExecute, b); }
  
      bool is_null() const { return m_raw == 0; }
      void clear() { m_raw = 0; }
  
      void set_bit(u64 bit, bool value)
      {
          if (value)
              m_raw |= bit;
          else
              m_raw &= ~bit;
      }
  
  private:
      u64 m_raw;
  };
  
  static_assert(sizeof(PageDirectoryEntry) == 8);
  static_assert(sizeof(PageTableEntry) == 8);
  
  class PageDirectoryPointerTable {
  public:
      PageDirectoryEntry* directory(size_t index)
      {
          return (PageDirectoryEntry*)(raw[index] & ~0xfffu);
      }
  
      u64 raw[4];
  };
  
  class GenericInterruptHandler;
  struct RegisterState;
  
  template<typename T>
  void read_possibly_unaligned_data(u8* where, T& data)
  {
      // FIXME: Implement 64 bit unaligned data access
      VERIFY(sizeof(T) == 8 || sizeof(T) == 4 || sizeof(T) == 2);
      if (((FlatPtr)where % sizeof(T)) == 0) {
          data = *(T*)where;
          return;
      }
      if (sizeof(T) == 2) {
          data = *(u8*)where | ((u16)(*(where + 1)) << 8);
          return;
      }
      if (sizeof(T) == 4) {
          data = *(u8*)where | (((u32) * (where + 1)) << 8) | (((u32) * (where + 2)) << 16) | (((u32) * (u8*)(where + 3)) << 24);
          return;
      }
      VERIFY_NOT_REACHED();
  }
  
  template<typename T>
  void write_possibly_unaligned_data(u8* where, T data)
  {
      // FIXME: Implement 64 bit unaligned data access
      VERIFY(sizeof(T) == 8 || sizeof(T) == 4 || sizeof(T) == 2);
      if (((FlatPtr)where % sizeof(T)) == 0) {
          *(T*)where = data;
          return;
      }
      if (sizeof(T) == 2) {
          where[0] = (u8)(data & 0xFF);
          where[1] = (u8)((data >> 8) & 0xFF);
          return;
      }
      if (sizeof(T) == 4) {
          where[0] = (u8)(data & 0xFF);
          where[1] = (u8)((data >> 8) & 0xFF);
          where[2] = (u8)((data >> 16) & 0xFF);
          where[3] = (u8)((data >> 24) & 0xFF);
          return;
      }
      VERIFY_NOT_REACHED();
  }
  
  const DescriptorTablePointer& get_gdtr();
  const DescriptorTablePointer& get_idtr();
  void register_interrupt_handler(u8 number, void (*handler)());
  void register_user_callable_interrupt_handler(u8 number, void (*handler)());
  GenericInterruptHandler& get_interrupt_handler(u8 interrupt_number);
  void register_generic_interrupt_handler(u8 number, GenericInterruptHandler&);
  void unregister_generic_interrupt_handler(u8 number, GenericInterruptHandler&);
  void flush_idt();
  void load_task_register(u16 selector);
  [[noreturn]] void handle_crash(RegisterState&, const char* description, int signal, bool out_of_memory = false);
  
  #define LSW(x) ((u32)(x)&0xFFFF)
  #define MSW(x) (((u32)(x) >> 16) & 0xFFFF)
  #define LSB(x) ((x)&0xFF)
  #define MSB(x) (((x) >> 8) & 0xFF)
  
  #define cli() asm volatile("cli" :: \
                                 : "memory")
  #define sti() asm volatile("sti" :: \
                                 : "memory")
  
  inline FlatPtr cpu_flags()
  {
      FlatPtr flags;
      asm volatile(
          "pushf\n"
          "pop %0\n"
          : "=rm"(flags)::"memory");
      return flags;
  }
  
  inline void set_fs(u16 segment)
  {
      asm volatile(
          "mov %%ax, %%fs" ::"a"(segment)
          : "memory");
  }
  
  inline void set_gs(u16 segment)
  {
      asm volatile(
          "mov %%ax, %%gs" ::"a"(segment)
          : "memory");
  }
  
  inline u16 get_fs()
  {
      u16 fs;
      asm("mov %%fs, %%eax"
          : "=a"(fs));
      return fs;
  }
  
  inline u16 get_gs()
  {
      u16 gs;
      asm("mov %%gs, %%eax"
          : "=a"(gs));
      return gs;
  }
  
  inline u32 read_fs_u32(u32 offset)
  {
      u32 val;
      asm volatile(
          "movl %%fs:%a[off], %k[val]"
          : [val] "=r"(val)
          : [off] "ir"(offset));
      return val;
  }
  
  inline FlatPtr read_fs_ptr(u32 offset)
  {
      return read_fs_u32(offset);
  }
  
  inline void write_fs_u32(u32 offset, u32 val)
  {
      asm volatile(
          "movl %k[val], %%fs:%a[off]" ::[off] "ir"(offset), [val] "ir"(val)
          : "memory");
  }
  
  inline bool are_interrupts_enabled()
  {
      return cpu_flags() & 0x200;
  }
  
  class InterruptDisabler {
  public:
      InterruptDisabler()
      {
          m_flags = cpu_flags();
          cli();
      }
  
      ~InterruptDisabler()
      {
          if (m_flags & 0x200)
              sti();
      }
  
  private:
      u32 m_flags;
  };
  
  class NonMaskableInterruptDisabler {
  public:
      NonMaskableInterruptDisabler();
      ~NonMaskableInterruptDisabler();
  };
  
  /* Map IRQ0-15 @ ISR 0x50-0x5F */
  #define IRQ_VECTOR_BASE 0x50
  
  struct PageFaultFlags {
      enum Flags {
          NotPresent = 0x00,
          ProtectionViolation = 0x01,
          Read = 0x00,
          Write = 0x02,
          UserMode = 0x04,
          SupervisorMode = 0x00,
          ReservedBitViolation = 0x08,
          InstructionFetch = 0x10,
      };
  };
  
  class PageFault {
  public:
      PageFault(u16 code, VirtualAddress vaddr)
          : m_code(code)
          , m_vaddr(vaddr)
      {
      }
  
      enum class Type {
          PageNotPresent = PageFaultFlags::NotPresent,
          ProtectionViolation = PageFaultFlags::ProtectionViolation,
      };
  
      enum class Access {
          Read = PageFaultFlags::Read,
          Write = PageFaultFlags::Write,
      };
  
      VirtualAddress vaddr() const { return m_vaddr; }
      u16 code() const { return m_code; }
  
      Type type() const { return (Type)(m_code & 1); }
      Access access() const { return (Access)(m_code & 2); }
  
      bool is_not_present() const { return (m_code & 1) == PageFaultFlags::NotPresent; }
      bool is_protection_violation() const { return (m_code & 1) == PageFaultFlags::ProtectionViolation; }
      bool is_read() const { return (m_code & 2) == PageFaultFlags::Read; }
      bool is_write() const { return (m_code & 2) == PageFaultFlags::Write; }
      bool is_user() const { return (m_code & 4) == PageFaultFlags::UserMode; }
      bool is_supervisor() const { return (m_code & 4) == PageFaultFlags::SupervisorMode; }
      bool is_instruction_fetch() const { return (m_code & 16) == PageFaultFlags::InstructionFetch; }
  
  private:
      u16 m_code;
      VirtualAddress m_vaddr;
  };
  
  struct [[gnu::packed]] RegisterState {
      FlatPtr ss;
      FlatPtr gs;
      FlatPtr fs;
      FlatPtr es;
      FlatPtr ds;
      FlatPtr edi;
      FlatPtr esi;
      FlatPtr ebp;
      FlatPtr esp;
      FlatPtr ebx;
      FlatPtr edx;
      FlatPtr ecx;
      FlatPtr eax;
      u16 exception_code;
      u16 isr_number;
  #if ARCH(X86_64)
      u32 padding;
  #endif
      FlatPtr eip;
      FlatPtr cs;
      FlatPtr eflags;
      FlatPtr userspace_esp;
      FlatPtr userspace_ss;
  };
  
  struct [[gnu::packed]] DebugRegisterState {
      FlatPtr dr0;
      FlatPtr dr1;
      FlatPtr dr2;
      FlatPtr dr3;
      FlatPtr dr6;
      FlatPtr dr7;
  };
  
  #if ARCH(I386)
  #    define REGISTER_STATE_SIZE (19 * 4)
  #else
  #    define REGISTER_STATE_SIZE (19 * 8)
  #endif
  static_assert(REGISTER_STATE_SIZE == sizeof(RegisterState));
  
  void copy_kernel_registers_into_ptrace_registers(PtraceRegisters&, const RegisterState&);
  void copy_ptrace_registers_into_kernel_registers(RegisterState&, const PtraceRegisters&);
  
  struct [[gnu::aligned(16)]] FPUState
  {
      u8 buffer[512];
  };
  
  constexpr FlatPtr page_base_of(FlatPtr address)
  {
      return address & PAGE_MASK;
  }
  
  inline FlatPtr page_base_of(const void* address)
  {
      return page_base_of((FlatPtr)address);
  }
  
  constexpr FlatPtr offset_in_page(FlatPtr address)
  {
      return address & (~PAGE_MASK);
  }
  
  inline FlatPtr offset_in_page(const void* address)
  {
      return offset_in_page((FlatPtr)address);
  }
  
  FlatPtr read_cr0();
  FlatPtr read_cr2();
  FlatPtr read_cr3();
  FlatPtr read_cr4();
  u64 read_xcr0();
  
  void write_cr0(FlatPtr);
  void write_cr3(FlatPtr);
  void write_cr4(FlatPtr);
  void write_xcr0(u64);
  
  void read_debug_registers_into(DebugRegisterState&);
  void write_debug_registers_from(const DebugRegisterState&);
  void clear_debug_registers();
  FlatPtr read_dr0();
  void write_dr0(FlatPtr);
  FlatPtr read_dr1();
  void write_dr1(FlatPtr);
  FlatPtr read_dr2();
  void write_dr2(FlatPtr);
  FlatPtr read_dr3();
  void write_dr3(FlatPtr);
  FlatPtr read_dr6();
  void write_dr6(FlatPtr);
  FlatPtr read_dr7();
  void write_dr7(FlatPtr);
  
  static inline bool is_kernel_mode()
  {
      u16 cs;
      asm volatile(
          "mov %%cs, %[cs] \n"
          : [cs] "=g"(cs));
      return (cs & 3) == 0;
  }
  
  class CPUID {
  public:
      explicit CPUID(u32 function) { asm volatile("cpuid"
                                                  : "=a"(m_eax), "=b"(m_ebx), "=c"(m_ecx), "=d"(m_edx)
                                                  : "a"(function), "c"(0)); }
      u32 eax() const { return m_eax; }
      u32 ebx() const { return m_ebx; }
      u32 ecx() const { return m_ecx; }
      u32 edx() const { return m_edx; }
  
  private:
      u32 m_eax { 0xffffffff };
      u32 m_ebx { 0xffffffff };
      u32 m_ecx { 0xffffffff };
      u32 m_edx { 0xffffffff };
  };
  
  inline void read_tsc(u32& lsw, u32& msw)
  {
      asm volatile("rdtsc"
                   : "=d"(msw), "=a"(lsw));
  }
  
  inline u64 read_tsc()
  {
      u32 lsw;
      u32 msw;
      read_tsc(lsw, msw);
      return ((u64)msw << 32) | lsw;
  }
  
  // FIXME: This can't hold every CPU feature as-is.
  enum class CPUFeature : u32 {
      NX = (1 << 0),
      PAE = (1 << 1),
      PGE = (1 << 2),
      RDRAND = (1 << 3),
      RDSEED = (1 << 4),
      SMAP = (1 << 5),
      SMEP = (1 << 6),
      SSE = (1 << 7),
      TSC = (1 << 8),
      RDTSCP = (1 << 9),
      CONSTANT_TSC = (1 << 10),
      NONSTOP_TSC = (1 << 11),
      UMIP = (1 << 12),
      SEP = (1 << 13),
      SYSCALL = (1 << 14),
      MMX = (1 << 15),
      SSE2 = (1 << 16),
      SSE3 = (1 << 17),
      SSSE3 = (1 << 18),
      SSE4_1 = (1 << 19),
      SSE4_2 = (1 << 20),
      XSAVE = (1 << 21),
      AVX = (1 << 22),
  };
  
  class Thread;
  struct TrapFrame;
  
  #define GDT_SELECTOR_CODE0 0x08
  #define GDT_SELECTOR_DATA0 0x10
  #define GDT_SELECTOR_CODE3 0x18
  #define GDT_SELECTOR_DATA3 0x20
  #define GDT_SELECTOR_TLS 0x28
  #define GDT_SELECTOR_PROC 0x30
  #define GDT_SELECTOR_TSS 0x38
  
  // SYSENTER makes certain assumptions on how the GDT is structured:
  static_assert(GDT_SELECTOR_CODE0 + 8 == GDT_SELECTOR_DATA0); // SS0 = CS0 + 8
  
  // SYSEXIT makes certain assumptions on how the GDT is structured:
  static_assert(GDT_SELECTOR_CODE0 + 16 == GDT_SELECTOR_CODE3); // CS3 = CS0 + 16
  static_assert(GDT_SELECTOR_CODE0 + 24 == GDT_SELECTOR_DATA3); // SS3 = CS0 + 32
  
  class ProcessorInfo;
  class SchedulerPerProcessorData;
  struct MemoryManagerData;
  struct ProcessorMessageEntry;
  
  struct ProcessorMessage {
      using CallbackFunction = Function<void()>;
  
      enum Type {
          FlushTlb,
          Callback,
      };
      Type type;
      volatile u32 refs; // atomic
      union {
          ProcessorMessage* next; // only valid while in the pool
          alignas(CallbackFunction) u8 callback_storage[sizeof(CallbackFunction)];
          struct {
              const PageDirectory* page_directory;
              u8* ptr;
              size_t page_count;
          } flush_tlb;
      };
  
      volatile bool async;
  
      ProcessorMessageEntry* per_proc_entries;
  
      CallbackFunction& callback_value()
      {
          return *bit_cast<CallbackFunction*>(&callback_storage);
      }
  
      void invoke_callback()
      {
          VERIFY(type == Type::Callback);
          callback_value()();
      }
  };
  
  struct ProcessorMessageEntry {
      ProcessorMessageEntry* next;
      ProcessorMessage* msg;
  };
  
  struct DeferredCallEntry {
      using HandlerFunction = Function<void()>;
  
      DeferredCallEntry* next;
      alignas(HandlerFunction) u8 handler_storage[sizeof(HandlerFunction)];
      bool was_allocated;
  
      HandlerFunction& handler_value()
      {
          return *bit_cast<HandlerFunction*>(&handler_storage);
      }
  
      void invoke_handler()
      {
          handler_value()();
      }
  };
  
  class Processor;
  // Note: We only support processors at most at the moment,
  // so allocate 8 slots of inline capacity in the container.
  using ProcessorContainer = Array<Processor*, 8>;
  
  class Processor {
      friend class ProcessorInfo;
  
      AK_MAKE_NONCOPYABLE(Processor);
      AK_MAKE_NONMOVABLE(Processor);
  
      Processor* m_self;
  
      DescriptorTablePointer m_gdtr;
      Descriptor m_gdt[256];
      u32 m_gdt_length;
  
      u32 m_cpu;
      u32 m_in_irq;
      Atomic<u32, AK::MemoryOrder::memory_order_relaxed> m_in_critical;
      static Atomic<u32> s_idle_cpu_mask;
  
      TSS m_tss;
      static FPUState s_clean_fpu_state;
      CPUFeature m_features;
      static volatile u32 g_total_processors; // atomic
      u8 m_physical_address_bit_width;
  
      ProcessorInfo* m_info;
      MemoryManagerData* m_mm_data;
      SchedulerPerProcessorData* m_scheduler_data;
      Thread* m_current_thread;
      Thread* m_idle_thread;
  
      volatile ProcessorMessageEntry* m_message_queue; // atomic, LIFO
  
      bool m_invoke_scheduler_async;
      bool m_scheduler_initialized;
      Atomic<bool> m_halt_requested;
  
      DeferredCallEntry* m_pending_deferred_calls; // in reverse order
      DeferredCallEntry* m_free_deferred_call_pool_entry;
      DeferredCallEntry m_deferred_call_pool[5];
  
      void gdt_init();
      void write_raw_gdt_entry(u16 selector, u32 low, u32 high);
      void write_gdt_entry(u16 selector, Descriptor& descriptor);
      static ProcessorContainer& processors();
  
      static void smp_return_to_pool(ProcessorMessage& msg);
      static ProcessorMessage& smp_get_from_pool();
      static void smp_cleanup_message(ProcessorMessage& msg);
      bool smp_queue_message(ProcessorMessage& msg);
      static void smp_unicast_message(u32 cpu, ProcessorMessage& msg, bool async);
      static void smp_broadcast_message(ProcessorMessage& msg);
      static void smp_broadcast_wait_sync(ProcessorMessage& msg);
      static void smp_broadcast_halt();
  
      void deferred_call_pool_init();
      void deferred_call_execute_pending();
      DeferredCallEntry* deferred_call_get_free();
      void deferred_call_return_to_pool(DeferredCallEntry*);
      void deferred_call_queue_entry(DeferredCallEntry*);
  
      void cpu_detect();
      void cpu_setup();
  
      String features_string() const;
  
  public:
      Processor() = default;
  
      void early_initialize(u32 cpu);
      void initialize(u32 cpu);
  
      void idle_begin()
      {
          s_idle_cpu_mask.fetch_or(1u << m_cpu, AK::MemoryOrder::memory_order_relaxed);
      }
  
      void idle_end()
      {
          s_idle_cpu_mask.fetch_and(~(1u << m_cpu), AK::MemoryOrder::memory_order_relaxed);
      }
  
      static u32 count()
      {
          // NOTE: because this value never changes once all APs are booted,
          // we don't really need to do an atomic_load() on this variable
          return g_total_processors;
      }
  
      ALWAYS_INLINE static void wait_check()
      {
          Processor::current().smp_process_pending_messages();
          // TODO: pause
      }
  
      [[noreturn]] static void halt();
  
      static void flush_entire_tlb_local()
      {
          write_cr3(read_cr3());
      }
  
      static void flush_tlb_local(VirtualAddress vaddr, size_t page_count);
      static void flush_tlb(const PageDirectory*, VirtualAddress, size_t);
  
      Descriptor& get_gdt_entry(u16 selector);
      void flush_gdt();
      const DescriptorTablePointer& get_gdtr();
  
      static Processor& by_id(u32 cpu);
  
      static size_t processor_count() { return processors().size(); }
  
      template<IteratorFunction<Processor&> Callback>
      static inline IterationDecision for_each(Callback callback)
      {
          auto& procs = processors();
          size_t count = procs.size();
          for (size_t i = 0; i < count; i++) {
              if (callback(*procs[i]) == IterationDecision::Break)
                  return IterationDecision::Break;
          }
          return IterationDecision::Continue;
      }
  
      template<VoidFunction<Processor&> Callback>
      static inline IterationDecision for_each(Callback callback)
      {
          auto& procs = processors();
          size_t count = procs.size();
          for (size_t i = 0; i < count; i++) {
              if (procs[i] != nullptr)
                  callback(*procs[i]);
          }
          return IterationDecision::Continue;
      }
  
      ALWAYS_INLINE u8 physical_address_bit_width() const { return m_physical_address_bit_width; }
  
      ALWAYS_INLINE ProcessorInfo& info() { return *m_info; }
  
      ALWAYS_INLINE static Processor& current()
      {
          return *(Processor*)read_fs_ptr(__builtin_offsetof(Processor, m_self));
      }
  
      ALWAYS_INLINE static bool is_initialized()
      {
          return get_fs() == GDT_SELECTOR_PROC && read_fs_u32(__builtin_offsetof(Processor, m_self)) != 0;
      }
  
      ALWAYS_INLINE void set_scheduler_data(SchedulerPerProcessorData& scheduler_data)
      {
          m_scheduler_data = &scheduler_data;
      }
  
      ALWAYS_INLINE SchedulerPerProcessorData& get_scheduler_data() const
      {
          return *m_scheduler_data;
      }
  
      ALWAYS_INLINE void set_mm_data(MemoryManagerData& mm_data)
      {
          m_mm_data = &mm_data;
      }
  
      ALWAYS_INLINE MemoryManagerData& get_mm_data() const
      {
          return *m_mm_data;
      }
  
      ALWAYS_INLINE void set_idle_thread(Thread& idle_thread)
      {
          m_idle_thread = &idle_thread;
      }
  
      ALWAYS_INLINE static Thread* current_thread()
      {
          // If we were to use Processor::current here, we'd have to
          // disable interrupts to prevent a race where we may get pre-empted
          // right after getting the Processor structure and then get moved
          // to another processor, which would lead us to get the wrong thread.
          // To avoid having to disable interrupts, we can just read the field
          // directly in an atomic fashion, similar to Processor::current.
          return (Thread*)read_fs_ptr(__builtin_offsetof(Processor, m_current_thread));
      }
  
      ALWAYS_INLINE static void set_current_thread(Thread& current_thread)
      {
          // See comment in Processor::current_thread
          write_fs_u32(__builtin_offsetof(Processor, m_current_thread), FlatPtr(&current_thread));
      }
  
      ALWAYS_INLINE static Thread* idle_thread()
      {
          // See comment in Processor::current_thread
          return (Thread*)read_fs_u32(__builtin_offsetof(Processor, m_idle_thread));
      }
  
      ALWAYS_INLINE u32 get_id() const
      {
          // NOTE: This variant should only be used when iterating over all
          // Processor instances, or when it's guaranteed that the thread
          // cannot move to another processor in between calling Processor::current
          // and Processor::get_id, or if this fact is not important.
          // All other cases should use Processor::id instead!
          return m_cpu;
      }
  
      ALWAYS_INLINE static u32 id()
      {
          // See comment in Processor::current_thread
          return read_fs_ptr(__builtin_offsetof(Processor, m_cpu));
      }
  
      ALWAYS_INLINE static bool is_bootstrap_processor()
      {
          return Processor::id() == 0;
      }
  
      ALWAYS_INLINE u32 raise_irq()
      {
          return m_in_irq++;
      }
  
      ALWAYS_INLINE void restore_irq(u32 prev_irq)
      {
          VERIFY(prev_irq <= m_in_irq);
          if (!prev_irq) {
              u32 prev_critical = 0;
              if (m_in_critical.compare_exchange_strong(prev_critical, 1)) {
                  m_in_irq = prev_irq;
                  deferred_call_execute_pending();
                  auto prev_raised = m_in_critical.exchange(prev_critical);
                  VERIFY(prev_raised == prev_critical + 1);
                  check_invoke_scheduler();
              } else if (prev_critical == 0) {
                  check_invoke_scheduler();
              }
          } else {
              m_in_irq = prev_irq;
          }
      }
  
      ALWAYS_INLINE u32& in_irq()
      {
          return m_in_irq;
      }
  
      ALWAYS_INLINE void restore_in_critical(u32 critical)
      {
          m_in_critical = critical;
      }
  
      ALWAYS_INLINE void enter_critical(u32& prev_flags)
      {
          prev_flags = cpu_flags();
          cli();
          m_in_critical++;
      }
  
      ALWAYS_INLINE void leave_critical(u32 prev_flags)
      {
          cli(); // Need to prevent IRQs from interrupting us here!
          VERIFY(m_in_critical > 0);
          if (m_in_critical == 1) {
              if (!m_in_irq) {
                  deferred_call_execute_pending();
                  VERIFY(m_in_critical == 1);
              }
              m_in_critical--;
              if (!m_in_irq)
                  check_invoke_scheduler();
          } else {
              m_in_critical--;
          }
          if (prev_flags & 0x200)
              sti();
          else
              cli();
      }
  
      ALWAYS_INLINE u32 clear_critical(u32& prev_flags, bool enable_interrupts)
      {
          prev_flags = cpu_flags();
          u32 prev_crit = m_in_critical.exchange(0, AK::MemoryOrder::memory_order_acquire);
          if (!m_in_irq)
              check_invoke_scheduler();
          if (enable_interrupts)
              sti();
          return prev_crit;
      }
  
      ALWAYS_INLINE void restore_critical(u32 prev_crit, u32 prev_flags)
      {
          m_in_critical.store(prev_crit, AK::MemoryOrder::memory_order_release);
          VERIFY(!prev_crit || !(prev_flags & 0x200));
          if (prev_flags & 0x200)
              sti();
          else
              cli();
      }
  
      ALWAYS_INLINE u32 in_critical() { return m_in_critical.load(); }
  
      ALWAYS_INLINE const FPUState& clean_fpu_state() const
      {
          return s_clean_fpu_state;
      }
  
      static void smp_enable();
      bool smp_process_pending_messages();
  
      static void smp_broadcast(Function<void()>, bool async);
      static void smp_unicast(u32 cpu, Function<void()>, bool async);
      static void smp_broadcast_flush_tlb(const PageDirectory*, VirtualAddress, size_t);
      static u32 smp_wake_n_idle_processors(u32 wake_count);
  
      static void deferred_call_queue(Function<void()> callback);
  
      ALWAYS_INLINE bool has_feature(CPUFeature f) const
      {
          return (static_cast<u32>(m_features) & static_cast<u32>(f)) != 0;
      }
  
      void check_invoke_scheduler();
      void invoke_scheduler_async() { m_invoke_scheduler_async = true; }
  
      void enter_trap(TrapFrame& trap, bool raise_irq);
  
      void exit_trap(TrapFrame& trap);
  
      [[noreturn]] void initialize_context_switching(Thread& initial_thread);
      NEVER_INLINE void switch_context(Thread*& from_thread, Thread*& to_thread);
      [[noreturn]] static void assume_context(Thread& thread, FlatPtr flags);
      u32 init_context(Thread& thread, bool leave_crit);
      static Vector<FlatPtr> capture_stack_trace(Thread& thread, size_t max_frames = 0);
  
      String platform_string() const;
  };
  
  class ScopedCritical {
      AK_MAKE_NONCOPYABLE(ScopedCritical);
  
  public:
      ScopedCritical()
      {
          enter();
      }
  
      ~ScopedCritical()
      {
          if (m_valid)
              leave();
      }
  
      ScopedCritical(ScopedCritical&& from)
          : m_prev_flags(exchange(from.m_prev_flags, 0))
          , m_valid(exchange(from.m_valid, false))
      {
      }
  
      ScopedCritical& operator=(ScopedCritical&& from)
      {
          if (&from != this) {
              m_prev_flags = exchange(from.m_prev_flags, 0);
              m_valid = exchange(from.m_valid, false);
          }
          return *this;
      }
  
      void leave()
      {
          VERIFY(m_valid);
          m_valid = false;
          Processor::current().leave_critical(m_prev_flags);
      }
  
      void enter()
      {
          VERIFY(!m_valid);
          m_valid = true;
          Processor::current().enter_critical(m_prev_flags);
      }
  
  private:
      u32 m_prev_flags { 0 };
      bool m_valid { false };
  };
  
  struct TrapFrame {
      u32 prev_irq_level;
      TrapFrame* next_trap;
      RegisterState* regs; // must be last
  
      TrapFrame() = delete;
      TrapFrame(const TrapFrame&) = delete;
      TrapFrame(TrapFrame&&) = delete;
      TrapFrame& operator=(const TrapFrame&) = delete;
      TrapFrame& operator=(TrapFrame&&) = delete;
  };
  
  #if ARCH(I386)
  #    define TRAP_FRAME_SIZE (3 * 4)
  #else
  #    define TRAP_FRAME_SIZE (3 * 8)
  #endif
  
  static_assert(TRAP_FRAME_SIZE == sizeof(TrapFrame));
  
  extern "C" void enter_trap_no_irq(TrapFrame*) __attribute__((used));
  extern "C" void enter_trap(TrapFrame*) __attribute__((used));
  extern "C" void exit_trap(TrapFrame*) __attribute__((used));
  
  class MSR {
      uint32_t m_msr;
  
  public:
      static bool have()
      {
          CPUID id(1);
          return (id.edx() & (1 << 5)) != 0;
      }
  
      MSR(const MSR&) = delete;
      MSR& operator=(const MSR&) = delete;
  
      MSR(uint32_t msr)
          : m_msr(msr)
      {
      }
  
      void get(u32& low, u32& high)
      {
          asm volatile("rdmsr"
                       : "=a"(low), "=d"(high)
                       : "c"(m_msr));
      }
  
      void set(u32 low, u32 high)
      {
          asm volatile("wrmsr" ::"a"(low), "d"(high), "c"(m_msr));
      }
  };
  
  ALWAYS_INLINE void stac()
  {
      if (!Processor::current().has_feature(CPUFeature::SMAP))
          return;
      asm volatile("stac" ::
                       : "cc");
  }
  
  ALWAYS_INLINE void clac()
  {
      if (!Processor::current().has_feature(CPUFeature::SMAP))
          return;
      asm volatile("clac" ::
                       : "cc");
  }
  }
  