text: /*++
  
  Copyright (c) 1989  Microsoft Corporation
  
  Module Name:
  
      mminit.c
  
  Abstract:
  
      This module contains the initialization for the memory management
      system.
  
  Author:
  
      Lou Perazzoli (loup) 20-Mar-1989
      Landy Wang (landyw) 02-Jun-1997
  
  Revision History:
  
  --*/
  
  #include "mi.h"
  
  MMPTE MmSharedUserDataPte;
  
  extern MMINPAGE_SUPPORT_LIST MmInPageSupportList;
  extern MMEVENT_COUNT_LIST MmEventCountList;
  extern KMUTANT MmSystemLoadLock;
  extern ULONG_PTR MmSystemPtesStart[MaximumPtePoolTypes];
  extern ULONG MiSpecialPoolPtes;
  extern ULONG MmPagedPoolCommit;
  
  extern PVOID BBTBuffer;
  extern ULONG BBTPagesToReserve;
  
  ULONG_PTR MmSubsectionBase;
  ULONG_PTR MmSubsectionTopPage;
  ULONG MmDataClusterSize;
  ULONG MmCodeClusterSize;
  PFN_NUMBER MmResidentAvailableAtInit;
  KEVENT MmImageMappingPteEvent;
  PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
  LIST_ENTRY MmLockConflictList;
  LOGICAL MmPagedPoolMaximumDesired = FALSE;
  
  PERFINFO_MMINIT_DECL;
  
  VOID
  MiMapBBTMemory (
      IN PLOADER_PARAMETER_BLOCK LoaderBlock
      );
  
  VOID
  MiInitializeSpecialPool(
      VOID
      );
  
  VOID
  MiEnablePagingTheExecutive(
      VOID
      );
  
  VOID
  MiEnablePagingOfDriverAtInit (
      IN PMMPTE PointerPte,
      IN PMMPTE LastPte
      );
  
  VOID
  MiBuildPagedPool (
      );
  
  VOID
  MiMergeMemoryLimit (
      IN OUT PPHYSICAL_MEMORY_DESCRIPTOR Memory,
      IN PFN_NUMBER StartPage,
      IN PFN_NUMBER NumberOfPages
      );
  
  VOID
  MiWriteProtectSystemImage (
      IN PVOID DllBase
      );
  
  #ifndef NO_POOL_CHECKS
  VOID
  MiInitializeSpecialPoolCriteria (
      IN VOID
      );
  #endif
  
  #if defined (_X86PAE_)
  VOID
  MiCheckPaeLicense (
      IN PLOADER_PARAMETER_BLOCK LoaderBlock,
      IN PBOOLEAN IncludeType,
      OUT PPHYSICAL_MEMORY_DESCRIPTOR Memory
      );
  #endif
  
  #ifdef ALLOC_PRAGMA
  #pragma alloc_text(INIT,MmInitSystem)
  #pragma alloc_text(INIT,MiMapBBTMemory)
  #pragma alloc_text(INIT,MmInitializeMemoryLimits)
  #pragma alloc_text(INIT,MiMergeMemoryLimit)
  #pragma alloc_text(INIT,MmFreeLoaderBlock)
  #pragma alloc_text(INIT,MiBuildPagedPool)
  #pragma alloc_text(INIT,MiFindInitializationCode)
  #pragma alloc_text(INIT,MiEnablePagingTheExecutive)
  #pragma alloc_text(INIT,MiEnablePagingOfDriverAtInit)
  #if defined (_X86PAE_)
  #pragma alloc_text(INIT,MiCheckPaeLicense)
  #endif
  #pragma alloc_text(PAGELK,MiFreeInitializationCode)
  #endif
  
  #define MM_MAX_LOADER_BLOCKS 30
  
  //
  // Default is a 300 second life span for modified mapped pages -
  // This can be overridden in the registry.
  //
  
  ULONG MmModifiedPageLifeInSeconds = 300;
  
  LARGE_INTEGER MiModifiedPageLife;
  
  BOOLEAN MiTimerPending = FALSE;
  
  KEVENT MiMappedPagesTooOldEvent;
  
  KDPC MiModifiedPageWriterTimerDpc;
  
  KTIMER MiModifiedPageWriterTimer;
  
  //
  // The following constants are based on the number PAGES not the
  // memory size.  For convenience the number of pages is calculated
  // based on a 4k page size.  Hence 12mb with 4k page is 3072.
  //
  
  #define MM_SMALL_SYSTEM ((13*1024*1024) / 4096)
  
  #define MM_MEDIUM_SYSTEM ((19*1024*1024) / 4096)
  
  #define MM_MIN_INITIAL_PAGED_POOL ((32*1024*1024) >> PAGE_SHIFT)
  
  #define MM_DEFAULT_IO_LOCK_LIMIT (2 * 1024 * 1024)
  
  extern ULONG MmMaximumWorkingSetSize;
  
  extern ULONG MmEnforceWriteProtection;
  
  extern CHAR MiPteStr[];
  
  extern LONG MiTrimInProgressCount;
  
  #if !defined (_WIN64)
  
  #if !defined (_X86PAE_)
  PFN_NUMBER MmSystemPageDirectory;
  #else
  PFN_NUMBER MmSystemPageDirectory[PD_PER_SYSTEM];
  #endif
  
  PMMPTE MmSystemPagePtes;
  #endif
  
  ULONG MmTotalSystemCodePages;
  
  MM_SYSTEMSIZE MmSystemSize;
  
  ULONG MmLargeSystemCache;
  
  ULONG MmProductType;
  
  LIST_ENTRY MmLoadedUserImageList;
  PPAGE_FAULT_NOTIFY_ROUTINE MmPageFaultNotifyRoutine;
  PHARD_FAULT_NOTIFY_ROUTINE MmHardFaultNotifyRoutine;
  
  
  BOOLEAN
  MmInitSystem (
      IN ULONG Phase,
      IN PLOADER_PARAMETER_BLOCK LoaderBlock,
      IN PPHYSICAL_MEMORY_DESCRIPTOR PhysicalMemoryBlock
      )
  
  /*++
  
  Routine Description:
  
      This function is called during Phase 0, phase 1 and at the end
      of phase 1 ("phase 2") initialization.
  
      Phase 0 initializes the memory management paging functions,
      nonpaged and paged pool, the PFN database, etc.
  
      Phase 1 initializes the section objects, the physical memory
      object, and starts the memory management system threads.
  
      Phase 2 frees memory used by the OsLoader.
  
  Arguments:
  
      Phase - System initialization phase.
  
      LoaderBlock - Supplies a pointer to the system loader block.
  
  Return Value:
  
      Returns TRUE if the initialization was successful.
  
  Environment:
  
      Kernel Mode Only.  System initialization.
  
  --*/
  
  {
      HANDLE ThreadHandle;
      OBJECT_ATTRIBUTES ObjectAttributes;
      PMMPTE PointerPte;
      PMMPTE PointerPde;
      PMMPTE StartPde;
      PMMPTE StartPpe;
      PMMPTE StartingPte;
      PMMPTE EndPde;
      PMMPFN Pfn1;
      MMPTE Pointer;
      PFN_NUMBER i, j;
      PFN_NUMBER PageFrameIndex;
      PFN_NUMBER DirectoryFrameIndex;
      MMPTE TempPte;
      KIRQL OldIrql;
      LOGICAL First;
      PLIST_ENTRY NextEntry;
      PLDR_DATA_TABLE_ENTRY DataTableEntry;
      ULONG MaximumSystemCacheSize;
      ULONG MaximumSystemCacheSizeTotal;
      PEPROCESS Process;
      PIMAGE_NT_HEADERS NtHeaders;
      ULONG_PTR SystemPteMultiplier;
  
      BOOLEAN IncludeType[LoaderMaximum];
      ULONG MemoryAlloc[(sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
              sizeof(PHYSICAL_MEMORY_RUN)*MAX_PHYSICAL_MEMORY_FRAGMENTS) /
                sizeof(ULONG)];
  
      PPHYSICAL_MEMORY_DESCRIPTOR Memory;
  
      //
      // Make sure structure alignment is okay.
      //
  
      if (Phase == 0) {
          MmThrottleTop = 450;
          MmThrottleBottom = 127;
  
          //
          // Set the highest user address, the system range start address, the
          // user probe address, and the virtual bias.
          //
  
  #if defined(_AXP64_) || defined(_IA64_)
  
          MmHighestUserAddress = MM_HIGHEST_USER_ADDRESS;
          MmUserProbeAddress = MM_USER_PROBE_ADDRESS;
          MmSystemRangeStart = MM_SYSTEM_RANGE_START;
  
  #else
  
          MmHighestUserAddress = (PVOID)(KSEG0_BASE - 0x10000 - 1);
          MmUserProbeAddress = KSEG0_BASE - 0x10000;
          MmSystemRangeStart = (PVOID)KSEG0_BASE;
  
  #endif
  
          MiHighestUserPte = MiGetPteAddress (MmHighestUserAddress);
          MiHighestUserPde = MiGetPdeAddress (MmHighestUserAddress);
  
          MmVirtualBias = 0;
  
          //
          // Set the highest section base address.
          //
          // N.B. In 32-bit systems this address must be 2gb or less even for
          //      systems that run with 3gb enabled. Otherwise, it would not
          //      be possible to map based sections identically in all processes.
          //
  
          MmHighSectionBase = ((PCHAR)MmHighestUserAddress - 0x800000);
  
          if (ExVerifySuite(TerminalServer) == TRUE) {
              MiHydra = TRUE;
              MiSystemViewStart = MM_SYSTEM_VIEW_START_IF_HYDRA;
              MmSessionBase = (ULONG_PTR)MM_SESSION_SPACE_DEFAULT;
  
          } else {
              MiSystemViewStart = MM_SYSTEM_VIEW_START;
              MiHydra = FALSE;
          }
  
          MaximumSystemCacheSize = (MM_SYSTEM_CACHE_END - MM_SYSTEM_CACHE_START) >> PAGE_SHIFT;
  
          //
          // If the system has been biased to an alternate base address to
          // allow 3gb of user address space, then set the user probe address
          // and the maximum system cache size.
          //
  
  #if defined(_X86_)
  
          MmVirtualBias = LoaderBlock->u.I386.VirtualBias;
  
          if (MmVirtualBias != 0) {
              MmHighestUserAddress = ((PCHAR)MmHighestUserAddress + 0x40000000);
              MmSystemRangeStart = ((PCHAR)MmSystemRangeStart + 0x40000000);
              MmUserProbeAddress += 0x40000000;
              MiMaximumWorkingSet += 0x40000000 >> PAGE_SHIFT;
  
              MiHighestUserPte = MiGetPteAddress (MmHighestUserAddress);
              MiHighestUserPde = MiGetPdeAddress (MmHighestUserAddress);
  
              MaximumSystemCacheSize -= MM_BOOT_IMAGE_SIZE >> PAGE_SHIFT;
  
              if (MiHydra == TRUE) {
  
                  //
                  // Moving to 3GB means moving session space to just above
                  // the system cache (and lowering the system cache max size
                  // accordingly).
                  //
  
                  MaximumSystemCacheSize -= (MI_SESSION_SPACE_TOTAL_SIZE + MM_SYSTEM_VIEW_SIZE_IF_HYDRA) >> PAGE_SHIFT;
  
                  MiSystemViewStart = (ULONG_PTR)(MM_SYSTEM_CACHE_START +
                                        (MaximumSystemCacheSize << PAGE_SHIFT));
  
                  MmSessionBase = MiSystemViewStart + MM_SYSTEM_VIEW_SIZE_IF_HYDRA + MM_BOOT_IMAGE_SIZE;
  
              } else {
                  MaximumSystemCacheSize -= MM_SYSTEM_VIEW_SIZE >> PAGE_SHIFT;
                  MiSystemViewStart = (ULONG_PTR)(MM_SYSTEM_CACHE_START +
                                        (MaximumSystemCacheSize << PAGE_SHIFT) +
                                        MM_BOOT_IMAGE_SIZE);
              }
          }
  
  #else
  
          if (MiHydra == TRUE) {
              MaximumSystemCacheSize -= MM_SYSTEM_VIEW_SIZE_IF_HYDRA >> PAGE_SHIFT;
              MiSystemViewStart = MM_SYSTEM_VIEW_START_IF_HYDRA;
          }
  
  #endif
  
          if (MiHydra == TRUE) {
              MmSessionSpace = (PMM_SESSION_SPACE)((ULONG_PTR)MmSessionBase + MI_SESSION_IMAGE_SIZE);
  
              MiSessionBasePte = MiGetPteAddress (MmSessionBase);
              MiSessionLastPte = MiGetPteAddress (MI_SESSION_SPACE_END);
          }
  
          //
          // A few sanity checks to ensure things are as they should be.
          //
  
  #if DBG
          if ((sizeof(MMWSL) % 8) != 0) {
              DbgPrint("working set list is not a quadword sized structure\n");
          }
  
          if ((sizeof(CONTROL_AREA) % 8) != 0) {
              DbgPrint("control area list is not a quadword sized structure\n");
          }
  
          if ((sizeof(SUBSECTION) % 8) != 0) {
              DbgPrint("subsection list is not a quadword sized structure\n");
          }
  
          //
          // Some checks to make sure prototype PTEs can be placed in
          // either paged or nonpaged (prototype PTEs for paged pool are here)
          // can be put into pte format.
          //
  
          PointerPte = (PMMPTE)MmPagedPoolStart;
          Pointer.u.Long = MiProtoAddressForPte (PointerPte);
          TempPte = Pointer;
          PointerPde = MiPteToProto(&TempPte);
          if (PointerPte != PointerPde) {
              DbgPrint("unable to map start of paged pool as prototype pte %p %p\n",
                       PointerPde,
                       PointerPte);
          }
  
          PointerPte =
                  (PMMPTE)((ULONG_PTR)MM_NONPAGED_POOL_END & ~((1 << PTE_SHIFT) - 1));
  
          Pointer.u.Long = MiProtoAddressForPte (PointerPte);
          TempPte = Pointer;
          PointerPde = MiPteToProto(&TempPte);
          if (PointerPte != PointerPde) {
              DbgPrint("unable to map end of nonpaged pool as prototype pte %p %p\n",
                       PointerPde,
                       PointerPte);
          }
  
          PointerPte = (PMMPTE)(((ULONG_PTR)NON_PAGED_SYSTEM_END -
                          0x37000 + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1));
  
          for (j = 0; j < 20; j++) {
              Pointer.u.Long = MiProtoAddressForPte (PointerPte);
              TempPte = Pointer;
              PointerPde = MiPteToProto(&TempPte);
              if (PointerPte != PointerPde) {
                  DbgPrint("unable to map end of nonpaged pool as prototype pte %p %p\n",
                           PointerPde,
                           PointerPte);
              }
  
              PointerPte++;
          }
  
          PointerPte = (PMMPTE)(((ULONG_PTR)MM_NONPAGED_POOL_END - 0x133448) & ~(ULONG_PTR)7);
          Pointer.u.Long = MiGetSubsectionAddressForPte (PointerPte);
          TempPte = Pointer;
          PointerPde = (PMMPTE)MiGetSubsectionAddress(&TempPte);
          if (PointerPte != PointerPde) {
              DbgPrint("unable to map end of nonpaged pool as section pte %p %p\n",
                       PointerPde,
                       PointerPte);
  
              MiFormatPte(&TempPte);
          }
  
          //
          // End of sanity checks.
          //
  
  #endif //dbg
  
          if (MmEnforceWriteProtection) {
              MiPteStr[0] = (CHAR)1;
          }
      
          InitializeListHead( &MmLoadedUserImageList );
          InitializeListHead( &MmLockConflictList );
  
          MmCriticalSectionTimeout.QuadPart = Int32x32To64(
                                                   MmCritsectTimeoutSeconds,
                                                  -10000000);
  
  
          //
          // Initialize PFN database mutex and System Address Space creation
          // mutex.
          //
  
          ExInitializeFastMutex (&MmSectionCommitMutex);
          ExInitializeFastMutex (&MmSectionBasedMutex);
          ExInitializeFastMutex (&MmDynamicMemoryMutex);
  
          KeInitializeMutant (&MmSystemLoadLock, FALSE);
  
          KeInitializeEvent (&MmAvailablePagesEvent, NotificationEvent, TRUE);
          KeInitializeEvent (&MmAvailablePagesEventHigh, NotificationEvent, TRUE);
          KeInitializeEvent (&MmMappedFileIoComplete, NotificationEvent, FALSE);
          KeInitializeEvent (&MmImageMappingPteEvent, NotificationEvent, FALSE);
          KeInitializeEvent (&MmZeroingPageEvent, SynchronizationEvent, FALSE);
          KeInitializeEvent (&MmCollidedFlushEvent, NotificationEvent, FALSE);
          KeInitializeEvent (&MmCollidedLockEvent, NotificationEvent, FALSE);
          KeInitializeEvent (&MiMappedPagesTooOldEvent, NotificationEvent, FALSE);
  
          KeInitializeDpc( &MiModifiedPageWriterTimerDpc, MiModifiedPageWriterTimerDispatch, NULL );
          KeInitializeTimerEx( &MiModifiedPageWriterTimer, SynchronizationTimer );
  
          MiModifiedPageLife.QuadPart = Int32x32To64(
                                                   MmModifiedPageLifeInSeconds,
                                                  -10000000);
  
          InitializeListHead (&MmWorkingSetExpansionHead.ListHead);
          InitializeListHead (&MmInPageSupportList.ListHead);
          InitializeListHead (&MmEventCountList.ListHead);
  
          MmZeroingPageThreadActive = FALSE;
  
          //
          // Compute physical memory blocks yet again
          //
  
          Memory = (PPHYSICAL_MEMORY_DESCRIPTOR)&MemoryAlloc;
          Memory->NumberOfRuns = MAX_PHYSICAL_MEMORY_FRAGMENTS;
  
          // include all memory types ...
          for (i=0; i < LoaderMaximum; i++) {
              IncludeType[i] = TRUE;
          }
  
          // ... expect these..
          IncludeType[LoaderBad] = FALSE;
          IncludeType[LoaderFirmwarePermanent] = FALSE;
          IncludeType[LoaderSpecialMemory] = FALSE;
          IncludeType[LoaderBBTMemory] = FALSE;
  
          MmInitializeMemoryLimits(LoaderBlock, IncludeType, Memory);
  
  #if defined (_X86PAE_)
          MiCheckPaeLicense (LoaderBlock, IncludeType, Memory);
  #endif
  
  #if defined (_X86PAE_) || defined (_WIN64)
          Mm64BitPhysicalAddress = TRUE;
  #endif
  
          //
          // Add all memory runs in PhysicalMemoryBlock to Memory
          //
  
          for (i = 0; i < PhysicalMemoryBlock->NumberOfRuns; i += 1) {
              MiMergeMemoryLimit (Memory,
                                  PhysicalMemoryBlock->Run[i].BasePage,
                                  PhysicalMemoryBlock->Run[i].PageCount
                                  );
          }
  
          //
          // Sort and merge adjacent runs.
          //
  
          for (i=0; i < Memory->NumberOfRuns; i++) {
              for (j=i+1; j < Memory->NumberOfRuns; j++) {
                  if (Memory->Run[j].BasePage < Memory->Run[i].BasePage) {
                      // swap runs
                      PhysicalMemoryBlock->Run[0] = Memory->Run[j];
                      Memory->Run[j] = Memory->Run[i];
                      Memory->Run[i] = PhysicalMemoryBlock->Run[0];
                  }
  
                  if (Memory->Run[i].BasePage + Memory->Run[i].PageCount ==
                      Memory->Run[j].BasePage) {
                      // merge runs
                      Memory->NumberOfRuns -= 1;
                      Memory->Run[i].PageCount += Memory->Run[j].PageCount;
                      Memory->Run[j] = Memory->Run[Memory->NumberOfRuns];
                      i -= 1;
                      break;
                  }
              }
          }
  
          //
          // When safebooting, don't enable special pool, the verifier or any
          // other options that track corruption regardless of registry settings.
          //
      
          if (strstr(LoaderBlock->LoadOptions, SAFEBOOT_LOAD_OPTION_A)) {
              MmVerifyDriverBufferLength = (ULONG)-1;
              MmDontVerifyRandomDrivers = TRUE;
              MmSpecialPoolTag = (ULONG)-1;
              MmSnapUnloads = FALSE;
              MmProtectFreedNonPagedPool = FALSE;
              MmEnforceWriteProtection = 0;
              MmTrackLockedPages = FALSE;
              MmTrackPtes = FALSE;
          }
          else {
              MiTriageSystem (LoaderBlock);
          }
  
          SystemPteMultiplier = 0;
  
          if (MmNumberOfSystemPtes == 0) {
  #if defined (_WIN64)
  
              //
              // 64-bit NT is not contrained by virtual address space.  No
              // tradeoffs between nonpaged pool, paged pool and system PTEs
              // need to be made.  So just allocate PTEs on a linear scale as
              // a function of the amount of RAM.
              //
              // For example on Alpha64, 4gb of RAM gets 128gb of PTEs by default.
              // The page table cost is the inversion of the multiplier based
              // on the PTE_PER_PAGE.
              //
  
              if ((MiHydra == TRUE) && (ExpMultiUserTS == TRUE)) {
                  SystemPteMultiplier = 128;
              }
              else {
                  SystemPteMultiplier = 64;
              }
              if (Memory->NumberOfPages < 0x8000) {
                  SystemPteMultiplier >>= 1;
              }
  #else
              if (Memory->NumberOfPages < MM_MEDIUM_SYSTEM) {
                  MmNumberOfSystemPtes = MM_MINIMUM_SYSTEM_PTES;
              } else {
                  MmNumberOfSystemPtes = MM_DEFAULT_SYSTEM_PTES;
                  if (Memory->NumberOfPages > 8192) {
                      MmNumberOfSystemPtes += MmNumberOfSystemPtes;
  
                      //
                      // Any reasonable Hydra machine gets the maximum.
                      //
  
                      if ((MiHydra == TRUE) && (ExpMultiUserTS == TRUE)) {
                          MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES;
                      }
                  }
              }
  #endif
          }
          else if (MmNumberOfSystemPtes == (ULONG)-1) {
  
              //
              // This registry setting indicates the maximum number of
              // system PTEs possible for this machine must be allocated.
              // Snap this for later reference.
              //
          
              MiRequestedSystemPtes = MmNumberOfSystemPtes;
  
  #if defined (_WIN64)
              SystemPteMultiplier = 256;
  #else
              MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES;
  #endif
          }
  
          if (SystemPteMultiplier != 0) {
              if (Memory->NumberOfPages * SystemPteMultiplier > MM_MAXIMUM_SYSTEM_PTES) {
                  MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES;
              }
              else {
                  MmNumberOfSystemPtes = (ULONG)(Memory->NumberOfPages * SystemPteMultiplier);
              }
          }
  
          if (MmNumberOfSystemPtes > MM_MAXIMUM_SYSTEM_PTES)  {
              MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES;
          }
  
          if (MmNumberOfSystemPtes < MM_MINIMUM_SYSTEM_PTES) {
              MmNumberOfSystemPtes = MM_MINIMUM_SYSTEM_PTES;
          }
  
          if (MmHeapSegmentReserve == 0) {
              MmHeapSegmentReserve = 1024 * 1024;
          }
  
          if (MmHeapSegmentCommit == 0) {
              MmHeapSegmentCommit = PAGE_SIZE * 2;
          }
  
          if (MmHeapDeCommitTotalFreeThreshold == 0) {
              MmHeapDeCommitTotalFreeThreshold = 64 * 1024;
          }
  
          if (MmHeapDeCommitFreeBlockThreshold == 0) {
              MmHeapDeCommitFreeBlockThreshold = PAGE_SIZE;
          }
  
  #ifndef NO_POOL_CHECKS
          MiInitializeSpecialPoolCriteria ();
  #endif
  
          //
          // If the registry indicates drivers are in the suspect list,
          // extra system PTEs need to be allocated to support special pool
          // for their allocations.
          //
  
          if ((MmVerifyDriverBufferLength != (ULONG)-1) ||
              ((MmSpecialPoolTag != 0) && (MmSpecialPoolTag != (ULONG)-1))) {
              MmNumberOfSystemPtes += MM_SPECIAL_POOL_PTES;
          }
  
          MmNumberOfSystemPtes += BBTPagesToReserve;
  
          //
          // Initialize the machine dependent portion of the hardware.
          //
  
          ExInitializeResource (&MmSystemWsLock);
  
          MiInitMachineDependent (LoaderBlock);
  
  #if PFN_CONSISTENCY
          MiPfnProtectionEnabled = TRUE;
  #endif
  
          MiReloadBootLoadedDrivers (LoaderBlock);
  
          MiInitializeDriverVerifierList (LoaderBlock);
  
          j = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
               (sizeof(PHYSICAL_MEMORY_RUN) *
                      (Memory->NumberOfRuns - 1)));
  
          MmPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPoolMustSucceed,
                                                         j,
                                                         '  mM');
  
          RtlCopyMemory (MmPhysicalMemoryBlock, Memory, j);
  
          //
          // Setup the system size as small, medium, or large depending
          // on memory available.
          //
          // For internal MM tuning, the following applies
          //
          // 12Mb  is small
          // 12-19 is medium
          // > 19 is large
          //
          //
          // For all other external tuning,
          // < 19 is small
          // 19 - 31 is medium for workstation
          // 19 - 63 is medium for server
          // >= 32 is large for workstation
          // >= 64 is large for server
          //
  
          MmReadClusterSize = 7;
          if (MmNumberOfPhysicalPages <= MM_SMALL_SYSTEM ) {
              MmSystemSize = MmSmallSystem;
              MmMaximumDeadKernelStacks = 0;
              MmModifiedPageMinimum = 40;
              MmModifiedPageMaximum = 100;
              MmDataClusterSize = 0;
              MmCodeClusterSize = 1;
              MmReadClusterSize = 2;
  
          } else if (MmNumberOfPhysicalPages <= MM_MEDIUM_SYSTEM ) {
              MmSystemSize = MmSmallSystem;
              MmMaximumDeadKernelStacks = 2;
              MmModifiedPageMinimum = 80;
              MmModifiedPageMaximum = 150;
              MmSystemCacheWsMinimum += 100;
              MmSystemCacheWsMaximum += 150;
              MmDataClusterSize = 1;
              MmCodeClusterSize = 2;
              MmReadClusterSize = 4;
  
          } else {
              MmSystemSize = MmMediumSystem;
              MmMaximumDeadKernelStacks = 5;
              MmModifiedPageMinimum = 150;
              MmModifiedPageMaximum = 300;
              MmSystemCacheWsMinimum += 400;
              MmSystemCacheWsMaximum += 800;
              MmDataClusterSize = 3;
              MmCodeClusterSize = 7;
          }
  
          if (MmNumberOfPhysicalPages < ((24*1024*1024)/PAGE_SIZE)) {
              MmSystemCacheWsMinimum = 32;
          }
  
          if (MmNumberOfPhysicalPages >= ((32*1024*1024)/PAGE_SIZE)) {
  
              //
              // If we are on a workstation, 32Mb and above are considered large systems
              //
              if ( MmProductType == 0x00690057 ) {
                  MmSystemSize = MmLargeSystem;
  
              } else {
  
                  //
                  // For servers, 64Mb and greater is a large system
                  //
  
                  if (MmNumberOfPhysicalPages >= ((64*1024*1024)/PAGE_SIZE)) {
                      MmSystemSize = MmLargeSystem;
                  }
              }
          }
  
          if (MmNumberOfPhysicalPages > ((33*1024*1024)/PAGE_SIZE)) {
              MmModifiedPageMinimum = 400;
              MmModifiedPageMaximum = 800;
              MmSystemCacheWsMinimum += 500;
              MmSystemCacheWsMaximum += 900;
          }
  
          //
          // determine if we are on an AS system ( Winnt is not AS)
          //
  
          if (MmProductType == 0x00690057) {
              SharedUserData->NtProductType = NtProductWinNt;
              MmProductType = 0;
              MmThrottleTop = 250;
              MmThrottleBottom = 30;
  
          } else {
              if ( MmProductType == 0x0061004c ) {
                  SharedUserData->NtProductType = NtProductLanManNt;
  
              } else {
                  SharedUserData->NtProductType = NtProductServer;
              }
  
              MmProductType = 1;
              MmThrottleTop = 450;
              MmThrottleBottom = 80;
              MmMinimumFreePages = 81;
          }
  
          MiAdjustWorkingSetManagerParameters((BOOLEAN)(MmProductType == 0 ? TRUE : FALSE));
  
          //
          // Set the ResidentAvailablePages to the number of available
          // pages minus the fluid value.
          //
  
          MmResidentAvailablePages = MmAvailablePages - MM_FLUID_PHYSICAL_PAGES;
  
          //
          // Subtract off the size of the system cache working set.
          //
  
          MmResidentAvailablePages -= MmSystemCacheWsMinimum;
          MmResidentAvailableAtInit = MmResidentAvailablePages;
  
  
          if (MmResidentAvailablePages < 0) {
  #if DBG
              DbgPrint("system cache working set too big\n");
  #endif
              return FALSE;
          }
  
          //
          // Initialize spin lock for charging and releasing page file
          // commitment.
          //
  
          KeInitializeSpinLock (&MmChargeCommitmentLock);
  
          MiInitializeIoTrackers ();
  
          //
          // Initialize spin lock for allowing working set expansion.
          //
  
          KeInitializeSpinLock (&MmExpansionLock);
  
          ExInitializeFastMutex (&MmPageFileCreationLock);
  
          //
          // Initialize resource for extending sections.
          //
  
          ExInitializeResource (&MmSectionExtendResource);
          ExInitializeResource (&MmSectionExtendSetResource);
  
          //
          // Build the system cache structures.
          //
  
          StartPde = MiGetPdeAddress (MmSystemCacheWorkingSetList);
          PointerPte = MiGetPteAddress (MmSystemCacheWorkingSetList);
  
  #if defined (_WIN64)
  
          StartPpe = MiGetPteAddress(StartPde);
  
          TempPte = ValidKernelPte;
  
          if (StartPpe->u.Hard.Valid == 0) {
  
              //
              // Map in a page directory page for the system cache working set.
              // Note that we only populate one page table for this.
              //
  
              DirectoryFrameIndex = MiRemoveAnyPage(
                  MI_GET_PAGE_COLOR_FROM_PTE (StartPpe));
              TempPte.u.Hard.PageFrameNumber = DirectoryFrameIndex;
              *StartPpe = TempPte;
  
  
              Pfn1 = MI_PFN_ELEMENT(DirectoryFrameIndex);
              Pfn1->PteFrame = MI_GET_PAGE_FRAME_FROM_PTE (
                                    MiGetPteAddress(PDE_KTBASE));
              Pfn1->PteAddress = StartPpe;
              Pfn1->u2.ShareCount += 1;
              Pfn1->u3.e2.ReferenceCount = 1;
              Pfn1->u3.e1.PageLocation = ActiveAndValid;
              Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
  
              MiFillMemoryPte (StartPde,
                               PAGE_SIZE,
                               ZeroKernelPte.u.Long);
          }
  
          //
          // Map in a page table page.
          //
  
          ASSERT (StartPde->u.Hard.Valid == 0);
  
          PageFrameIndex = MiRemoveAnyPage(
                                  MI_GET_PAGE_COLOR_FROM_PTE (StartPde));
          TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
          MI_WRITE_VALID_PTE (StartPde, TempPte);
  
          Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
          Pfn1->PteFrame = DirectoryFrameIndex;
          Pfn1->PteAddress = StartPde;
          Pfn1->u2.ShareCount += 1;
          Pfn1->u3.e2.ReferenceCount = 1;
          Pfn1->u3.e1.PageLocation = ActiveAndValid;
          Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
  
          MiFillMemoryPte (MiGetVirtualAddressMappedByPte (StartPde),
                           PAGE_SIZE,
                           ZeroKernelPte.u.Long);
  
          StartPpe = MiGetPpeAddress(MmSystemCacheStart);
          StartPde = MiGetPdeAddress(MmSystemCacheStart);
          PointerPte = MiGetVirtualAddressMappedByPte (StartPde);
  
  #else
  #if !defined(_X86PAE_)
          ASSERT ((StartPde + 1) == MiGetPdeAddress (MmSystemCacheStart));
  #endif
  #endif
  
          MaximumSystemCacheSizeTotal = MaximumSystemCacheSize;
  
  #if defined(_X86_)
          MaximumSystemCacheSizeTotal += MiMaximumSystemCacheSizeExtra;
  #endif
  
          //
          // Size the system cache based on the amount of physical memory.
          //
  
          i = (MmNumberOfPhysicalPages + 65) / 1024;
  
          if (i >= 4) {
  
              //
              // System has at least 4032 pages.  Make the system
              // cache 128mb + 64mb for each additional 1024 pages.
              //
  
              MmSizeOfSystemCacheInPages = (PFN_COUNT)(
                              ((128*1024*1024) >> PAGE_SHIFT) +
                              ((i - 4) * ((64*1024*1024) >> PAGE_SHIFT)));
              if (MmSizeOfSystemCacheInPages > MaximumSystemCacheSizeTotal) {
                  MmSizeOfSystemCacheInPages = MaximumSystemCacheSizeTotal;
              }
          }
  
          MmSystemCacheEnd = (PVOID)(((PCHAR)MmSystemCacheStart +
                      MmSizeOfSystemCacheInPages * PAGE_SIZE) - 1);
  
  #if defined(_X86_)
          if (MmSizeOfSystemCacheInPages > MaximumSystemCacheSize) {
              ASSERT (MiMaximumSystemCacheSizeExtra != 0);
              MmSystemCacheEnd = (PVOID)(((PCHAR)MmSystemCacheStart +
                          MaximumSystemCacheSize * PAGE_SIZE) - 1);
  
              MiSystemCacheStartExtra = (PVOID)MM_SYSTEM_CACHE_START_EXTRA;
              MiSystemCacheEndExtra = (PVOID)(((PCHAR)MiSystemCacheStartExtra +
                          (MmSizeOfSystemCacheInPages - MaximumSystemCacheSize) * PAGE_SIZE) - 1);
          }
  		else {
              MiSystemCacheStartExtra = MmSystemCacheStart;
              MiSystemCacheEndExtra = MmSystemCacheEnd;
  		}
  #endif
  
          EndPde = MiGetPdeAddress(MmSystemCacheEnd);
  
          TempPte = ValidKernelPte;
  
  #if defined(_WIN64)
          First = (StartPpe->u.Hard.Valid == 0) ? TRUE : FALSE;
  #endif
  
  #if !defined (_WIN64)
          DirectoryFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (MiGetPteAddress(PDE_BASE));
  #endif
  
          LOCK_PFN (OldIrql);
          while (StartPde <= EndPde) {
  
  #if defined (_WIN64)
              if (First == TRUE || MiIsPteOnPdeBoundary(StartPde)) {
                  First = FALSE;
                  StartPpe = MiGetPteAddress(StartPde);
  
                  //
                  // Map in a page directory page.
                  //
  
                  DirectoryFrameIndex = MiRemoveAnyPage(
                                          MI_GET_PAGE_COLOR_FROM_PTE (StartPpe));
                  TempPte.u.Hard.PageFrameNumber = DirectoryFrameIndex;
                  *StartPpe = TempPte;
  
                  Pfn1 = MI_PFN_ELEMENT(DirectoryFrameIndex);
                  Pfn1->PteFrame = MI_GET_PAGE_FRAME_FROM_PTE (
                                          MiGetPteAddress(PDE_KTBASE));
                  Pfn1->PteAddress = StartPpe;
                  Pfn1->u2.ShareCount += 1;
                  Pfn1->u3.e2.ReferenceCount = 1;
                  Pfn1->u3.e1.PageLocation = ActiveAndValid;
                  Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
  
                  MiFillMemoryPte (StartPde,
                                   PAGE_SIZE,
                                   ZeroKernelPte.u.Long);
              }
  #endif
  
              ASSERT (StartPde->u.Hard.Valid == 0);
  
              //
              // Map in a page table page.
              //
  
              PageFrameIndex = MiRemoveAnyPage(
                                      MI_GET_PAGE_COLOR_FROM_PTE (StartPde));
              TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
              MI_WRITE_VALID_PTE (StartPde, TempPte);
  
              Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
              Pfn1->PteFrame = DirectoryFrameIndex;
              Pfn1->PteAddress = StartPde;
              Pfn1->u2.ShareCount += 1;
              Pfn1->u3.e2.ReferenceCount = 1;
              Pfn1->u3.e1.PageLocation = ActiveAndValid;
              Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
  
              MiFillMemoryPte (PointerPte,
                               PAGE_SIZE,
                               ZeroKernelPte.u.Long);
  
              StartPde += 1;
              PointerPte += PTE_PER_PAGE;
          }
  
  #if defined(_X86_)
          if (MiSystemCacheEndExtra != MmSystemCacheEnd) {
  
              StartPde = MiGetPdeAddress (MiSystemCacheStartExtra);
  			EndPde = MiGetPdeAddress(MiSystemCacheEndExtra);
  
  			PointerPte = MiGetPteAddress (MiSystemCacheStartExtra);
  
  			while (StartPde <= EndPde) {
  
  				ASSERT (StartPde->u.Hard.Valid == 0);
  
                  //
                  // Map in a page directory page.
                  //
  
                  PageFrameIndex = MiRemoveAnyPage(
                                          MI_GET_PAGE_COLOR_FROM_PTE (StartPde));
                  TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
                  MI_WRITE_VALID_PTE (StartPde, TempPte);
  
                  Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
                  Pfn1->PteFrame = MI_GET_PAGE_FRAME_FROM_PTE (
                                          MiGetPdeAddress(PDE_BASE));
                  Pfn1->PteAddress = StartPde;
                  Pfn1->u2.ShareCount += 1;
                  Pfn1->u3.e2.ReferenceCount = 1;
                  Pfn1->u3.e1.PageLocation = ActiveAndValid;
                  Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
  
                  MiFillMemoryPte (PointerPte,
                                   PAGE_SIZE,
                                   ZeroKernelPte.u.Long);
  
                  StartPde += 1;
                  PointerPte += PTE_PER_PAGE;
              }
          }
  #endif
  
          UNLOCK_PFN (OldIrql);
  
          //
          // Initialize the system cache.  Only set the large system cache if
          // we have a large amount of physical memory.
          //
  
          if (MmLargeSystemCache != 0 && MmNumberOfPhysicalPages > 0x7FF0) {
              if ((MmAvailablePages >
                      MmSystemCacheWsMaximum + ((64*1024*1024) >> PAGE_SHIFT))) {
                  MmSystemCacheWsMaximum =
                              MmAvailablePages - ((32*1024*1024) >> PAGE_SHIFT);
                  ASSERT ((LONG)MmSystemCacheWsMaximum > (LONG)MmSystemCacheWsMinimum);
                  MmMoreThanEnoughFreePages = 256;
              }
          }
  
          if (MmSystemCacheWsMaximum > (MM_MAXIMUM_WORKING_SET - 5)) {
              MmSystemCacheWsMaximum = MM_MAXIMUM_WORKING_SET - 5;
          }
  
          if (MmSystemCacheWsMaximum > MmSizeOfSystemCacheInPages) {
              MmSystemCacheWsMaximum = MmSizeOfSystemCacheInPages;
              if ((MmSystemCacheWsMinimum + 500) > MmSystemCacheWsMaximum) {
                  MmSystemCacheWsMinimum = MmSystemCacheWsMaximum - 500;
              }
          }
  
          MiInitializeSystemCache ((ULONG)MmSystemCacheWsMinimum,
                                   (ULONG)MmSystemCacheWsMaximum);
  
          //
          // Set the commit page limit to four times the number of available
          // pages. This value is updated as paging files are created.
          //
  
          MmTotalCommitLimit = MmAvailablePages << 2;
          MmTotalCommitLimitMaximum = MmTotalCommitLimit;
  
          MmAttemptForCantExtend.Segment = NULL;
          MmAttemptForCantExtend.RequestedExpansionSize = 1;
          MmAttemptForCantExtend.ActualExpansion = 1;
          MmAttemptForCantExtend.InProgress = FALSE;
          MmAttemptForCantExtend.PageFileNumber = MI_EXTEND_ANY_PAGEFILE;
  
          KeInitializeEvent (&MmAttemptForCantExtend.Event,
                             NotificationEvent,
                             FALSE);
  
          if (MmOverCommit == 0) {
  
              // If this value was not set via the registry, set the
              // over commit value to the number of available pages
              // minus 1024 pages (4mb with 4k pages).
              //
  
              if (MmAvailablePages > 1024) {
                  MmOverCommit = MmAvailablePages - 1024;
              }
          }
  
          //
          // Set maximum working set size to 512 pages less total available
          // memory.  2mb on machine with 4k pages.
          //
  
          MmMaximumWorkingSetSize = (ULONG)(MmAvailablePages - 512);
  
          if (MmMaximumWorkingSetSize > (MM_MAXIMUM_WORKING_SET - 5)) {
              MmMaximumWorkingSetSize = MM_MAXIMUM_WORKING_SET - 5;
          }
  
          //
          // Create the modified page writer event.
          //
  
          KeInitializeEvent (&MmModifiedPageWriterEvent, NotificationEvent, FALSE);
  
          //
          // Build paged pool.
          //
  
          MiBuildPagedPool ();
  
          //
          // Initialize the loaded module list.  This cannot be done until
          // paged pool has been built.
          //
  
          if (MiInitializeLoadedModuleList (LoaderBlock) == FALSE) {
  #if DBG
              DbgPrint("Loaded module list initialization failed\n");
  #endif
              return FALSE;
          }
  
          //
          // Initialize the unused segment thresholds.  The assumption is made
          // that the filesystem will tack on approximately a 1024-byte paged
          // pool charge (regardless of file size) for each file in the cache.
          //
  
          if (MmUnusedSegmentTrimLevel < 5) {
              MmUnusedSegmentTrimLevel = 5;
          }
          else if (MmUnusedSegmentTrimLevel > 40) {
              MmUnusedSegmentTrimLevel = 40;
          }
  	
          MmMaxUnusedSegmentPagedPoolUsage = (MmSizeOfPagedPoolInBytes / 100) * (MmUnusedSegmentTrimLevel << 1);
          MmUnusedSegmentPagedPoolReduction = MmMaxUnusedSegmentPagedPoolUsage >> 2;
  
          MmMaxUnusedSegmentNonPagedPoolUsage = (MmMaximumNonPagedPoolInBytes / 100) * (MmUnusedSegmentTrimLevel << 1);
          MmUnusedSegmentNonPagedPoolReduction = MmMaxUnusedSegmentNonPagedPoolUsage >> 2;
  
          //
          // Add more system PTEs if this is a large memory system.
          // Note that 64 bit systems can determine the right value at the
          // beginning since there is no virtual address space crunch.
          //
  
  #if !defined (_WIN64)
          if (MmNumberOfPhysicalPages > ((127*1024*1024) >> PAGE_SHIFT)) {
  
              PointerPde = MiGetPdeAddress ((PCHAR)MmPagedPoolEnd + 1);
              StartingPte = MiGetPteAddress ((PCHAR)MmPagedPoolEnd + 1);
              j = 0;
  
              TempPte = ValidKernelPde;
              LOCK_PFN (OldIrql);
              while (PointerPde->u.Hard.Valid == 0) {
  
                  MiChargeCommitmentCantExpand (1, TRUE);
                  MM_TRACK_COMMIT (MM_DBG_COMMIT_EXTRA_SYSTEM_PTES, 1);
  
                  PageFrameIndex = MiRemoveZeroPage (
                                      MI_GET_PAGE_COLOR_FROM_PTE (PointerPde));
                  TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
                  MI_WRITE_VALID_PTE (PointerPde, TempPte);
                  MiInitializePfn (PageFrameIndex, PointerPde, 1);
                  PointerPde += 1;
                  StartingPte += PAGE_SIZE / sizeof(MMPTE);
                  j += PAGE_SIZE / sizeof(MMPTE);
              }
  
              UNLOCK_PFN (OldIrql);
  
              if (j != 0) {
                  StartingPte = MiGetPteAddress ((PCHAR)MmPagedPoolEnd + 1);
                  MmNonPagedSystemStart = MiGetVirtualAddressMappedByPte (StartingPte);
                  MmNumberOfSystemPtes += j;
                  MiAddSystemPtes (StartingPte, j, SystemPteSpace);
              }
          }
  #endif
  
  
  #if DBG
          if (MmDebug & MM_DBG_DUMP_BOOT_PTES) {
              MiDumpValidAddresses ();
              MiDumpPfn ();
          }
  #endif
  
          MmPageFaultNotifyRoutine = NULL;
          MmHardFaultNotifyRoutine = NULL;
  
          return TRUE;
      }
  
      if (Phase == 1) {
  
  #if DBG
          MmDebug |= MM_DBG_CHECK_PFN_LOCK;
  #endif
  
  #ifdef _X86_
          MiInitMachineDependent (LoaderBlock);
  #endif
          MiMapBBTMemory(LoaderBlock);
  
          if (!MiSectionInitialization ()) {
              return FALSE;
          }
  
          Process = PsGetCurrentProcess ();
          if (Process->PhysicalVadList.Flink == NULL) {
              KeInitializeSpinLock (&Process->AweLock);
              InitializeListHead (&Process->PhysicalVadList);
          }
  
  #if defined(MM_SHARED_USER_DATA_VA)
  
          //
          // Create double mapped page between kernel and user mode.
          //
  
          PointerPte = MiGetPteAddress(KI_USER_SHARED_DATA);
          ASSERT (PointerPte->u.Hard.Valid == 1);
          PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
  
          MI_MAKE_VALID_PTE (MmSharedUserDataPte,
                             PageFrameIndex,
                             MM_READONLY,
                             PointerPte);
          Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
  
          LOCK_PFN (OldIrql);
  
          Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
  
          UNLOCK_PFN (OldIrql);
  #endif
  
          if (MiHydra == TRUE) {
              MiSessionWideInitializeAddresses ();
              MiInitializeSessionWsSupport ();
              MiInitializeSessionIds ();
          }
  
          //
          // Set up system wide lock pages limit.
          //
  
          if ((MmLockPagesPercentage < 5) || (MmLockPagesPercentage >= 100)) {
  
              //
              // No (reasonable or max) registry override from the user
              // so default to allowing all available memory.
              //
  
              MmLockPagesLimit = (PFN_NUMBER)-1;
          }
          else {
  
              //
              // Use the registry value - note it is expressed as a percentage.
              //
  
              MmLockPagesLimit = (PFN_NUMBER)((MmAvailablePages * MmLockPagesPercentage) / 100);
          }
  
          //
          // Start the modified page writer.
          //
  
          InitializeObjectAttributes( &ObjectAttributes, NULL, 0, NULL, NULL );
  
          if (!NT_SUCCESS(PsCreateSystemThread(
                          &ThreadHandle,
                          THREAD_ALL_ACCESS,
                          &ObjectAttributes,
                          0L,
                          NULL,
                          MiModifiedPageWriter,
                          NULL
                          ))) {
              return FALSE;
          }
          ZwClose (ThreadHandle);
  
          //
          // Start the balance set manager.
          //
          // The balance set manager performs stack swapping and working
          // set management and requires two threads.
          //
  
          KeInitializeEvent (&MmWorkingSetManagerEvent,
                             SynchronizationEvent,
                             FALSE);
  
          InitializeObjectAttributes( &ObjectAttributes, NULL, 0, NULL, NULL );
  
          if (!NT_SUCCESS(PsCreateSystemThread(
                          &ThreadHandle,
                          THREAD_ALL_ACCESS,
                          &ObjectAttributes,
                          0L,
                          NULL,
                          KeBalanceSetManager,
                          NULL
                          ))) {
  
              return FALSE;
          }
          ZwClose (ThreadHandle);
  
          if (!NT_SUCCESS(PsCreateSystemThread(
                          &ThreadHandle,
                          THREAD_ALL_ACCESS,
                          &ObjectAttributes,
                          0L,
                          NULL,
                          KeSwapProcessOrStack,
                          NULL
                          ))) {
  
              return FALSE;
          }
          ZwClose (ThreadHandle);
  
  #ifndef NO_POOL_CHECKS
          MiInitializeSpecialPoolCriteria ();
  #endif
  
  #if defined(_X86_)
          MiEnableKernelVerifier ();
  #endif
  
          ExAcquireResourceExclusive (&PsLoadedModuleResource, TRUE);
  
          NextEntry = PsLoadedModuleList.Flink;
      
          for ( ; NextEntry != &PsLoadedModuleList; NextEntry = NextEntry->Flink) {
      
              DataTableEntry = CONTAINING_RECORD(NextEntry,
                                                 LDR_DATA_TABLE_ENTRY,
                                                 InLoadOrderLinks);
      
              NtHeaders = RtlImageNtHeader(DataTableEntry->DllBase);
  
              if ((NtHeaders->OptionalHeader.MajorOperatingSystemVersion >= 5) &&
                  (NtHeaders->OptionalHeader.MajorImageVersion >= 5)) {
                  DataTableEntry->Flags |= LDRP_ENTRY_NATIVE;
              }
  
              MiWriteProtectSystemImage (DataTableEntry->DllBase);
          }
          ExReleaseResource (&PsLoadedModuleResource);
  
          InterlockedDecrement (&MiTrimInProgressCount);
  
          return TRUE;
      }
  
      if (Phase == 2) {
          MiEnablePagingTheExecutive();
          return TRUE;
      }
  
      return FALSE;
  }
  
  VOID
  MiMapBBTMemory (
      IN PLOADER_PARAMETER_BLOCK LoaderBlock
      )
  
  /*++
  
  Routine Description:
  
      This function walks through the loader block's memory descriptor list
      and maps memory reserved for the BBT buffer into the system.
  
      The mapped PTEs are PDE-aligned and made user accessible.
  
  Arguments:
  
      LoaderBlock - Supplies a pointer to the system loader block.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  System initialization.
  
  --*/
  {
      PVOID Va;
      PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor;
      PLIST_ENTRY NextMd;
      PFN_NUMBER NumberOfPagesMapped;
      PFN_NUMBER NumberOfPages;
      PFN_NUMBER PageFrameIndex;
      PMMPTE PointerPte;
      PMMPTE PointerPde;
      PMMPTE LastPde;
      MMPTE TempPte;
  
      if (BBTPagesToReserve <= 0) {
          return;
      }
  
      //
      // Request enough PTEs such that protection can be applied to the PDEs.
      //
  
      NumberOfPages = (BBTPagesToReserve + (PTE_PER_PAGE - 1)) & ~(PTE_PER_PAGE - 1);
  
      PointerPte = MiReserveSystemPtes((ULONG)NumberOfPages,
                                       SystemPteSpace,
                                       MM_VA_MAPPED_BY_PDE,
                                       0,
                                       FALSE);
  
      if (PointerPte == NULL) {
          BBTPagesToReserve = 0;
          return;
      }
  
      //
      // Allow user access to the buffer.
      //
  
      PointerPde = MiGetPteAddress (PointerPte);
      LastPde = MiGetPteAddress (PointerPte + NumberOfPages);
  
      ASSERT (LastPde != PointerPde);
  
      do {
          TempPte = *PointerPde;
          TempPte.u.Long |= MM_PTE_OWNER_MASK;
          MI_WRITE_VALID_PTE (PointerPde, TempPte);
          PointerPde += 1;
      } while (PointerPde < LastPde);
  
      KeFlushEntireTb (TRUE, TRUE);
  
      Va = MiGetVirtualAddressMappedByPte (PointerPte);
  
      TempPte = ValidUserPte;
      NumberOfPagesMapped = 0;
  
      NextMd = LoaderBlock->MemoryDescriptorListHead.Flink;
  
      while (NextMd != &LoaderBlock->MemoryDescriptorListHead) {
  
          MemoryDescriptor = CONTAINING_RECORD(NextMd,
                                               MEMORY_ALLOCATION_DESCRIPTOR,
                                               ListEntry);
  
          if (MemoryDescriptor->MemoryType == LoaderBBTMemory) {
  
              PageFrameIndex = MemoryDescriptor->BasePage;
              NumberOfPages = MemoryDescriptor->PageCount;
  
              if (NumberOfPagesMapped + NumberOfPages > BBTPagesToReserve) {
                  NumberOfPages = BBTPagesToReserve - NumberOfPagesMapped;
              }
  
              NumberOfPagesMapped += NumberOfPages;
  
              do {
  
                  TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
                  MI_WRITE_VALID_PTE (PointerPte, TempPte);
  
                  PointerPte += 1;
                  PageFrameIndex += 1;
                  NumberOfPages -= 1;
              } while (NumberOfPages);
  
              if (NumberOfPagesMapped == BBTPagesToReserve) {
                  break;
              }
          }
  
          NextMd = MemoryDescriptor->ListEntry.Flink;
      }
  
      RtlZeroMemory(Va, BBTPagesToReserve << PAGE_SHIFT);
  
      //
      // Tell BBT_Init how many pages were allocated.
      //
  
      if (NumberOfPagesMapped < BBTPagesToReserve) {
          BBTPagesToReserve = (ULONG)NumberOfPagesMapped;
      }
      *(PULONG)Va = BBTPagesToReserve;
  
      //
      // At this point instrumentation code will detect the existence of
      // buffer and initialize the structures.
      //
  
      BBTBuffer = Va;
  
      PERFINFO_MMINIT_START();
  }
  
  
  VOID
  MmInitializeMemoryLimits (
      IN PLOADER_PARAMETER_BLOCK LoaderBlock,
      IN PBOOLEAN IncludeType,
      OUT PPHYSICAL_MEMORY_DESCRIPTOR Memory
      )
  
  /*++
  
  Routine Description:
  
      This function walks through the loader block's memory
      descriptor list and builds a list of contiguous physical
      memory blocks of the desired types.
  
  Arguments:
  
      LoaderBlock - Supplies a pointer the system loader block.
  
      IncludeType - Array of BOOLEANS of size LoaderMaximum.
                    TRUE means include this type of memory in return.
  
      Memory - Returns the physical memory blocks.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  System initialization.
  
  --*/
  {
  
      PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor;
      PLIST_ENTRY NextMd;
      PFN_NUMBER i;
      PFN_NUMBER LowestFound;
      PFN_NUMBER Found;
      PFN_NUMBER Merged;
      PFN_NUMBER NextPage;
      PFN_NUMBER TotalPages;
  
      TotalPages = 0;
  
      //
      // Walk through the memory descriptors and build the physical memory list.
      //
  
      LowestFound = 0;
      Memory->Run[0].BasePage = 0xffffffff;
      NextPage = 0xffffffff;
      Memory->Run[0].PageCount = 0;
      i = 0;
  
      do {
          Merged = FALSE;
          Found = FALSE;
          NextMd = LoaderBlock->MemoryDescriptorListHead.Flink;
  
          while (NextMd != &LoaderBlock->MemoryDescriptorListHead) {
  
              MemoryDescriptor = CONTAINING_RECORD(NextMd,
                                                   MEMORY_ALLOCATION_DESCRIPTOR,
                                                   ListEntry);
  
              if (MemoryDescriptor->MemoryType < LoaderMaximum &&
                  IncludeType [MemoryDescriptor->MemoryType] ) {
  
                  //
                  // Try to merge runs.
                  //
  
                  if (MemoryDescriptor->BasePage == NextPage) {
                      ASSERT (MemoryDescriptor->PageCount != 0);
                      Memory->Run[i - 1].PageCount += MemoryDescriptor->PageCount;
                      NextPage += MemoryDescriptor->PageCount;
                      TotalPages += MemoryDescriptor->PageCount;
                      Merged = TRUE;
                      Found = TRUE;
                      break;
                  }
  
                  if (MemoryDescriptor->BasePage >= LowestFound) {
                      if (Memory->Run[i].BasePage > MemoryDescriptor->BasePage) {
                          Memory->Run[i].BasePage = MemoryDescriptor->BasePage;
                          Memory->Run[i].PageCount = MemoryDescriptor->PageCount;
                      }
                      Found = TRUE;
                  }
              }
              NextMd = MemoryDescriptor->ListEntry.Flink;
          }
  
          if (!Merged && Found) {
              NextPage = Memory->Run[i].BasePage + Memory->Run[i].PageCount;
              TotalPages += Memory->Run[i].PageCount;
              i += 1;
          }
          Memory->Run[i].BasePage = 0xffffffff;
          LowestFound = NextPage;
  
      } while (Found);
      ASSERT (i <= Memory->NumberOfRuns);
      Memory->NumberOfRuns = (ULONG)i;
      Memory->NumberOfPages = TotalPages;
  
      return;
  }
  
  #if defined (_X86PAE_)
  
  static
  VOID
  MiCheckPaeLicense (
      IN PLOADER_PARAMETER_BLOCK LoaderBlock,
      IN PBOOLEAN IncludeType,
      OUT PPHYSICAL_MEMORY_DESCRIPTOR Memory
      )
  
  /*++
  
  Routine Description:
  
      This function walks through the loader block's memory descriptor list
      and removes descriptors above 4gb if the license does not allow them.
  
  Arguments:
  
      LoaderBlock - Supplies a pointer the system loader block.
  
      IncludeType - Array of BOOLEANS of size LoaderMaximum.
                    TRUE means include this type of memory in return.
  
      Memory - Returns the physical memory blocks.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  System initialization.
  
  --*/
  {
      PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor;
      PLIST_ENTRY NextMd;
      PFN_NUMBER i;
      PFN_NUMBER LowestFound;
      PFN_NUMBER Found;
      PFN_NUMBER Merged;
      PFN_NUMBER NextPage;
      PFN_NUMBER LastPage;
      PFN_NUMBER MaxPage;
      PFN_NUMBER TotalPages;
      static BOOLEAN BeenHere = FALSE;
  
      if (BeenHere == TRUE) {
          return;
      }
  
      BeenHere = TRUE;
  
      //
      // If properly licensed for PAE (ie: DataCenter) and booted without the
      // 3gb switch, then use all available physical memory.
      //
  
      if (ExVerifySuite(DataCenter) == TRUE) {
  
          //
          // Note MmVirtualBias has not yet been initialized at the time of the
          // first call to this routine, so use the LoaderBlock directly.
          //
  
          if (LoaderBlock->u.I386.VirtualBias == 0) {
  
              //
              // Limit the maximum physical memory to the amount we have
              // actually physically seen in a machine inhouse.
              //
  
              MaxPage = 8 * 1024 * 1024;
          }
          else {
  
              //
              // The system is booting /3gb, so don't use more than 16gb of
              // physical memory.  This ensures enough virtual space to map
              // the PFN database.
              //
  
              MaxPage = 4 * 1024 * 1024;
          }
      }
      else if ((MmProductType != 0x00690057) &&
               (ExVerifySuite(Enterprise) == TRUE)) {
  
          //
          // Advanced Server is permitted a maximum of 8gb physical memory.
          // The system continues to operate in 8-byte PTE mode.
          //
  
          MaxPage = 2 * 1024 * 1024;
      }
      else {
  
          //
          // All other configurations get a maximum of 4gb physical memory.
          // The system continues to operate in 8-byte PTE mode.
          //
  
          MaxPage = 1024 * 1024;
      }
  
      TotalPages = 0;
  
      //
      // Walk through the memory descriptors and make sure no descriptors
      // reach or exceed the physical addressing limit.
      //
  
      NextMd = LoaderBlock->MemoryDescriptorListHead.Flink;
      while (NextMd != &LoaderBlock->MemoryDescriptorListHead) {
  
          MemoryDescriptor = CONTAINING_RECORD(NextMd,
                                               MEMORY_ALLOCATION_DESCRIPTOR,
                                               ListEntry);
  
          if (MemoryDescriptor->BasePage >= MaxPage) {
              MemoryDescriptor->BasePage = 0;
              MemoryDescriptor->PageCount = 0;
          }
  
          LastPage = MemoryDescriptor->BasePage + MemoryDescriptor->PageCount;
  
          if (LastPage >= MaxPage || LastPage < MemoryDescriptor->BasePage) {
              MemoryDescriptor->PageCount = MaxPage - MemoryDescriptor->BasePage - 1;
          }
  
          NextMd = MemoryDescriptor->ListEntry.Flink;
      }
  
      //
      // Walk through the memory descriptors and truncate the physical memory
      // list.
      //
  
      LowestFound = 0;
      Memory->Run[0].BasePage = 0xffffffff;
      NextPage = 0xffffffff;
      Memory->Run[0].PageCount = 0;
      i = 0;
  
      do {
          Merged = FALSE;
          Found = FALSE;
          NextMd = LoaderBlock->MemoryDescriptorListHead.Flink;
  
          while (NextMd != &LoaderBlock->MemoryDescriptorListHead) {
  
              MemoryDescriptor = CONTAINING_RECORD(NextMd,
                                                   MEMORY_ALLOCATION_DESCRIPTOR,
                                                   ListEntry);
  
  
              if (MemoryDescriptor->BasePage || MemoryDescriptor->PageCount) {
  
                  if (MemoryDescriptor->MemoryType < LoaderMaximum &&
                      IncludeType [MemoryDescriptor->MemoryType] ) {
      
                      //
                      // Try to merge runs.
                      //
      
                      if (MemoryDescriptor->BasePage == NextPage) {
                          ASSERT (MemoryDescriptor->PageCount != 0);
                          Memory->Run[i - 1].PageCount += MemoryDescriptor->PageCount;
                          NextPage += MemoryDescriptor->PageCount;
                          TotalPages += MemoryDescriptor->PageCount;
                          Merged = TRUE;
                          Found = TRUE;
                          break;
                      }
      
                      if (MemoryDescriptor->BasePage >= LowestFound) {
                          if (Memory->Run[i].BasePage > MemoryDescriptor->BasePage) {
                              Memory->Run[i].BasePage = MemoryDescriptor->BasePage;
                              Memory->Run[i].PageCount = MemoryDescriptor->PageCount;
                          }
                          Found = TRUE;
                      }
                  }
              }
              NextMd = MemoryDescriptor->ListEntry.Flink;
          }
  
          if (!Merged && Found) {
              NextPage = Memory->Run[i].BasePage + Memory->Run[i].PageCount;
              TotalPages += Memory->Run[i].PageCount;
              i += 1;
          }
          Memory->Run[i].BasePage = 0xffffffff;
          LowestFound = NextPage;
  
      } while (Found);
      ASSERT (i <= Memory->NumberOfRuns);
      Memory->NumberOfRuns = (ULONG)i;
      Memory->NumberOfPages = TotalPages;
  
      return;
  }
  #endif
  
  
  VOID
  MiMergeMemoryLimit (
      IN OUT PPHYSICAL_MEMORY_DESCRIPTOR Memory,
      IN PFN_NUMBER StartPage,
      IN PFN_NUMBER NumberOfPages
      )
  /*++
  
  Routine Description:
  
      This function ensures the passed range is in the passed in Memory
      block adding any new data as needed.
  
      The passed memory block is assumed to be at least
      MAX_PHYSICAL_MEMORY_FRAGMENTS large
  
  Arguments:
  
      Memory - Supplies the memory block to verify run is present in.
  
      StartPage - Supplies the first page of the run.
  
      NumberOfPages - Supplies the number of pages in the run.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  System initialization.
  
  --*/
  {
      PFN_NUMBER EndPage, sp, ep, i;
  
      EndPage = StartPage + NumberOfPages;
  
      //
      // Clip range to area which is not already described
      //
  
      for (i=0; i < Memory->NumberOfRuns; i++) {
          sp = Memory->Run[i].BasePage;
          ep = sp + Memory->Run[i].PageCount;
  
          if (sp < StartPage) {
              if (ep > StartPage  &&  ep < EndPage) {
                  // bump beginning page of the target area
                  StartPage = ep;
              }
  
              if (ep > EndPage) {
                  //
                  // Target area is contained totally within this
                  // descriptor.  This range is fully accounted for.
                  //
  
                  StartPage = EndPage;
              }
  
          } else {
              // sp >= StartPage
  
              if (sp < EndPage) {
                  if (ep < EndPage) {
                      //
                      // This descriptor is totally within the target area -
                      // check the area on either side of this descriptor
                      //
  
                      MiMergeMemoryLimit (Memory, StartPage, sp - StartPage);
                      StartPage = ep;
  
                  }  else {
                      // clip the ending page of the target area
                      EndPage = sp;
                  }
              }
          }
  
          //
          // Anything left of target area?
          //
  
          if (StartPage == EndPage) {
              return ;
          }
      }   // next descriptor
  
      //
      // The range StartPage - EndPage is a missing. Add it.
      //
  
      if (Memory->NumberOfRuns == MAX_PHYSICAL_MEMORY_FRAGMENTS) {
          return ;
      }
  
      Memory->Run[Memory->NumberOfRuns].BasePage  = StartPage;
      Memory->Run[Memory->NumberOfRuns].PageCount = EndPage - StartPage;
      Memory->NumberOfPages += EndPage - StartPage;
      Memory->NumberOfRuns  += 1;
  }
  
  
  
  VOID
  MmFreeLoaderBlock (
      IN PLOADER_PARAMETER_BLOCK LoaderBlock
      )
  
  /*++
  
  Routine Description:
  
      This function is called as the last routine in phase 1 initialization.
      It frees memory used by the OsLoader.
  
  Arguments:
  
      LoadBlock - Supplies a pointer the system loader block.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  System initialization.
  
  --*/
  
  {
  
      PLIST_ENTRY NextMd;
      PMMPTE Pde;
      PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor;
      MEMORY_ALLOCATION_DESCRIPTOR SavedDescriptor[MM_MAX_LOADER_BLOCKS];
      PFN_NUMBER i;
      PFN_NUMBER NextPhysicalPage;
      PMMPFN Pfn1;
      LONG BlockNumber = -1;
      KIRQL OldIrql;
  
      //
      //
      // Walk through the memory descriptors and add pages to the
      // free list in the PFN database.
      //
  
      NextMd = LoaderBlock->MemoryDescriptorListHead.Flink;
  
      while (NextMd != &LoaderBlock->MemoryDescriptorListHead) {
  
          MemoryDescriptor = CONTAINING_RECORD(NextMd,
                                               MEMORY_ALLOCATION_DESCRIPTOR,
                                               ListEntry);
  
  
          switch (MemoryDescriptor->MemoryType) {
              case LoaderOsloaderHeap:
              case LoaderRegistryData:
              case LoaderNlsData:
              //case LoaderMemoryData:  //this has page table and other stuff.
  
                  //
                  // Capture the data to temporary storage so we won't
                  // free memory we are referencing.  Coalesce it if
                  // the blocks are adjacent and of the same type.
                  //
  
                  if (BlockNumber != -1 &&
                      MemoryDescriptor->MemoryType == SavedDescriptor[BlockNumber].MemoryType &&
                      MemoryDescriptor->BasePage == SavedDescriptor[BlockNumber].BasePage + SavedDescriptor[BlockNumber].PageCount) {
  
                          //
                          // these blocks are adjacent - merge them
                          //
  
                          SavedDescriptor[BlockNumber].PageCount += MemoryDescriptor->PageCount;
                  }
                  else {
                      BlockNumber += 1;
                      if (BlockNumber >= MM_MAX_LOADER_BLOCKS) {
                          KeBugCheckEx (MEMORY_MANAGEMENT, 0, 0, 0, 0);
                      }
                      SavedDescriptor[BlockNumber] = *MemoryDescriptor;
                  }
  
                  break;
  
              default:
  
                  break;
          }
  
          NextMd = MemoryDescriptor->ListEntry.Flink;
      }
  
      LOCK_PFN (OldIrql);
  
      while (BlockNumber >= 0) {
  
          i = SavedDescriptor[BlockNumber].PageCount;
          NextPhysicalPage = SavedDescriptor[BlockNumber].BasePage;
  
          Pfn1 = MI_PFN_ELEMENT (NextPhysicalPage);
          while (i != 0) {
  
              if (Pfn1->u3.e2.ReferenceCount == 0) {
                  if (Pfn1->u1.Flink == 0) {
  
                      //
                      // Set the PTE address to the physical page for
                      // virtual address alignment checking.
                      //
  
                      Pfn1->PteAddress =
                                 (PMMPTE)(NextPhysicalPage << PTE_SHIFT);
                      MiInsertPageInList (MmPageLocationList[FreePageList],
                                          NextPhysicalPage);
                  }
              } else {
  
                  if (NextPhysicalPage != 0) {
                      //
                      // Remove PTE and insert into the free list.  If it is
                      // a physical address within the PFN database, the PTE
                      // element does not exist and therefore cannot be updated.
                      //
  
                      if (!MI_IS_PHYSICAL_ADDRESS (
                              MiGetVirtualAddressMappedByPte (Pfn1->PteAddress))) {
  
                          //
                          // Not a physical address.
                          //
  
                          *(Pfn1->PteAddress) = ZeroPte;
                      }
  
                      MI_SET_PFN_DELETED (Pfn1);
                      MiDecrementShareCountOnly (NextPhysicalPage);
                  }
              }
  
              Pfn1++;
              i -= 1;
              NextPhysicalPage += 1;
          }
  
          BlockNumber -= 1;
      }
  
      //
      // If the kernel has been biased to allow for 3gb of user address space,
      // then the first 16mb of memory is doubly mapped to KSEG0_BASE and to
      // ALTERNATE_BASE. Therefore, the KSEG0_BASE entries must be unmapped.
      //
  
  #if defined(_X86_)
  
      if (MmVirtualBias != 0) {
          Pde = MiGetPdeAddress(KSEG0_BASE);
          MI_WRITE_INVALID_PTE (Pde, ZeroKernelPte);
          MI_WRITE_INVALID_PTE (Pde + 1, ZeroKernelPte);
          MI_WRITE_INVALID_PTE (Pde + 2, ZeroKernelPte);
          MI_WRITE_INVALID_PTE (Pde + 3, ZeroKernelPte);
  
  #if defined(_X86PAE_)
          MI_WRITE_INVALID_PTE (Pde + 4, ZeroKernelPte);
          MI_WRITE_INVALID_PTE (Pde + 5, ZeroKernelPte);
          MI_WRITE_INVALID_PTE (Pde + 6, ZeroKernelPte);
          MI_WRITE_INVALID_PTE (Pde + 7, ZeroKernelPte);
  #endif
  
      }
  
  #endif
  
      KeFlushEntireTb (TRUE, TRUE);
      UNLOCK_PFN (OldIrql);
      return;
  }
  
  VOID
  MiBuildPagedPool (
      VOID
      )
  
  /*++
  
  Routine Description:
  
      This function is called to build the structures required for paged
      pool and initialize the pool.  Once this routine is called, paged
      pool may be allocated.
  
  Arguments:
  
      None.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  System initialization.
  
  --*/
  
  {
      SIZE_T Size;
      PMMPTE PointerPte;
      PMMPTE PointerPde;
      PMMPTE PointerPpe;
      PMMPTE PointerPpeEnd;
      MMPTE TempPte;
      PMMPFN Pfn1;
      PFN_NUMBER PageFrameIndex;
      KIRQL OldIrql;
      ULONG i;
  
  #if !defined (_WIN64)
  
      //
      // Double map system page directory page.
      //
  
  #if defined (_X86PAE_)
  
      PointerPte = MiGetPteAddress(PDE_BASE);
  
      for (i = 0 ; i < PD_PER_SYSTEM; i += 1) {
          MmSystemPageDirectory[i] = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
          Pfn1 = MI_PFN_ELEMENT(MmSystemPageDirectory[i]);
          Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
          PointerPte += 1;
      }
  
      //
      // Was not mapped physically, map it virtually in system space.
      //
  
      PointerPte = MiReserveSystemPtes (
                                  PD_PER_SYSTEM,
                                  SystemPteSpace,
                                  MM_COLOR_ALIGNMENT,
                                  ((ULONG_PTR)PDE_BASE & MM_COLOR_MASK_VIRTUAL),
                                  TRUE);
  
      MmSystemPagePtes = (PMMPTE)MiGetVirtualAddressMappedByPte (PointerPte);
  
      TempPte = ValidKernelPde;
  
      for (i = 0 ; i < PD_PER_SYSTEM; i += 1) {
          TempPte.u.Hard.PageFrameNumber = MmSystemPageDirectory[i];
          MI_WRITE_VALID_PTE (PointerPte, TempPte);
          PointerPte += 1;
      }
  
  #else
  
      MmSystemPageDirectory = MI_GET_PAGE_FRAME_FROM_PTE (MiGetPteAddress(PDE_BASE));
  
      Pfn1 = MI_PFN_ELEMENT(MmSystemPageDirectory);
  
      Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
  
      MmSystemPagePtes = (PMMPTE)MiMapPageInHyperSpace (MmSystemPageDirectory,
                                                        &OldIrql);
      MiUnmapPageInHyperSpace (OldIrql);
  
      if (!MI_IS_PHYSICAL_ADDRESS(MmSystemPagePtes)) {
  
          //
          // Was not mapped physically, map it virtually in system space.
          //
  
          PointerPte = MiReserveSystemPtes (
                                      1,
                                      SystemPteSpace,
                                      MM_COLOR_ALIGNMENT,
                                      ((ULONG_PTR)PDE_BASE & MM_COLOR_MASK_VIRTUAL),
                                      TRUE);
          TempPte = ValidKernelPde;
          TempPte.u.Hard.PageFrameNumber = MmSystemPageDirectory;
          MI_WRITE_VALID_PTE (PointerPte, TempPte);
          MmSystemPagePtes = (PMMPTE)MiGetVirtualAddressMappedByPte (PointerPte);
      }
  #endif
  
  #endif
  
      if (MmPagedPoolMaximumDesired == TRUE) {
          MmSizeOfPagedPoolInBytes =
                      ((PCHAR)MmNonPagedSystemStart - (PCHAR)MmPagedPoolStart);
      }
  
      //
      // A size of 0 means size the pool based on physical memory.
      //
  
      if (MmSizeOfPagedPoolInBytes == 0) {
          MmSizeOfPagedPoolInBytes = 2 * MmMaximumNonPagedPoolInBytes;
      }
  
      if (MmIsThisAnNtAsSystem()) {
          if ((MmNumberOfPhysicalPages > ((24*1024*1024) >> PAGE_SHIFT)) &&
              (MmSizeOfPagedPoolInBytes < MM_MINIMUM_PAGED_POOL_NTAS)) {
  
              MmSizeOfPagedPoolInBytes = MM_MINIMUM_PAGED_POOL_NTAS;
          }
      }
  
      if (MmSizeOfPagedPoolInBytes >
                (ULONG_PTR)((PCHAR)MmNonPagedSystemStart - (PCHAR)MmPagedPoolStart)) {
          MmSizeOfPagedPoolInBytes =
                      ((PCHAR)MmNonPagedSystemStart - (PCHAR)MmPagedPoolStart);
      }
  
      Size = BYTES_TO_PAGES(MmSizeOfPagedPoolInBytes);
  
      if (Size < MM_MIN_INITIAL_PAGED_POOL) {
          Size = MM_MIN_INITIAL_PAGED_POOL;
      }
  
      if (Size > (MM_MAX_PAGED_POOL >> PAGE_SHIFT)) {
          Size = MM_MAX_PAGED_POOL >> PAGE_SHIFT;
      }
  
      Size = (Size + (PTE_PER_PAGE - 1)) / PTE_PER_PAGE;
      MmSizeOfPagedPoolInBytes = (ULONG_PTR)Size * PAGE_SIZE * PTE_PER_PAGE;
  
      ASSERT ((MmSizeOfPagedPoolInBytes + (PCHAR)MmPagedPoolStart) <=
              (PCHAR)MmNonPagedSystemStart);
  
      //
      // Set size to the number of pages in the pool.
      //
  
      Size = Size * PTE_PER_PAGE;
  
      MmPagedPoolEnd = (PVOID)(((PUCHAR)MmPagedPoolStart +
                              MmSizeOfPagedPoolInBytes) - 1);
  
      MmPageAlignedPoolBase[PagedPool] = MmPagedPoolStart;
  
      //
      // Build page table page for paged pool.
      //
  
      PointerPde = MiGetPdeAddress (MmPagedPoolStart);
      MmPagedPoolBasePde = PointerPde;
  
      TempPte = ValidKernelPde;
  
  #if defined (_WIN64)
  
      //
      // Map in all the page directory pages to span all of paged pool.
      // This removes the need for a system lookup directory.
      //
  
      PointerPpe = MiGetPpeAddress (MmPagedPoolStart);
      PointerPpeEnd = MiGetPpeAddress (MmPagedPoolEnd);
  
      while (PointerPpe <= PointerPpeEnd) {
  
          if (PointerPpe->u.Hard.Valid == 0) {
              PageFrameIndex = MiRemoveAnyPage(
                                       MI_GET_PAGE_COLOR_FROM_PTE (PointerPpe));
              TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
              *PointerPpe = TempPte;
  
              Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
              Pfn1->PteFrame = MI_GET_PAGE_FRAME_FROM_PTE (
                                       MiGetPteAddress(PDE_KTBASE));
              Pfn1->PteAddress = PointerPpe;
              Pfn1->u2.ShareCount = 1;
              Pfn1->u3.e2.ReferenceCount = 1;
              Pfn1->u3.e1.PageLocation = ActiveAndValid;
              Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
          }
  
          PointerPpe += 1;
      }
  
  #endif
  
      PointerPte = MiGetPteAddress (MmPagedPoolStart);
      MmPagedPoolInfo.FirstPteForPagedPool = PointerPte;
      MmPagedPoolInfo.LastPteForPagedPool = MiGetPteAddress (MmPagedPoolEnd);
  
      MiFillMemoryPte (PointerPde,
                       sizeof(MMPTE) *
                           (1 + MiGetPdeAddress (MmPagedPoolEnd) - PointerPde),
                       MM_KERNEL_NOACCESS_PTE);
  
      LOCK_PFN (OldIrql);
  
      //
      // Map in a page table page.
      //
  
      PageFrameIndex = MiRemoveAnyPage(
                              MI_GET_PAGE_COLOR_FROM_PTE (PointerPde));
      TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
      MI_WRITE_VALID_PTE (PointerPde, TempPte);
  
      Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
  #if defined (_WIN64)
      Pfn1->PteFrame = MI_GET_PAGE_FRAME_FROM_PTE(MiGetPpeAddress (MmPagedPoolStart));
  #else
  #if !defined (_X86PAE_)
      Pfn1->PteFrame = MmSystemPageDirectory;
  #else
      Pfn1->PteFrame = MmSystemPageDirectory[(PointerPde - MiGetPdeAddress(0)) / PDE_PER_PAGE];
  #endif
  #endif
      Pfn1->PteAddress = PointerPde;
      Pfn1->u2.ShareCount = 1;
      Pfn1->u3.e2.ReferenceCount = 1;
      Pfn1->u3.e1.PageLocation = ActiveAndValid;
      Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
      MiFillMemoryPte (PointerPte, PAGE_SIZE, MM_KERNEL_NOACCESS_PTE);
  
      UNLOCK_PFN (OldIrql);
  
      MmPagedPoolInfo.NextPdeForPagedPoolExpansion = PointerPde + 1;
  
      //
      // Build bitmaps for paged pool.
      //
  
      MiCreateBitMap (&MmPagedPoolInfo.PagedPoolAllocationMap, Size, NonPagedPool);
      RtlSetAllBits (MmPagedPoolInfo.PagedPoolAllocationMap);
  
      //
      // Indicate first page worth of PTEs are available.
      //
  
      RtlClearBits (MmPagedPoolInfo.PagedPoolAllocationMap, 0, PTE_PER_PAGE);
  
      MiCreateBitMap (&MmPagedPoolInfo.EndOfPagedPoolBitmap, Size, NonPagedPool);
      RtlClearAllBits (MmPagedPoolInfo.EndOfPagedPoolBitmap);
  
      //
      // If verifier is present then build the verifier paged pool bitmap.
      //
  
      if (MmVerifyDriverBufferLength != (ULONG)-1) {
          MiCreateBitMap (&VerifierLargePagedPoolMap, Size, NonPagedPool);
          RtlClearAllBits (VerifierLargePagedPoolMap);
      }
  
      //
      // Initialize paged pool.
      //
  
      InitializePool (PagedPool, 0L);
  
      MiInitializeSpecialPool();
  
      //
      // Allow mapping of views into system space.
      //
  
      MiInitializeSystemSpaceMap ((PVOID)0);
  
      return;
  }
  
  
  VOID
  MiFindInitializationCode (
      OUT PVOID *StartVa,
      OUT PVOID *EndVa
      )
  
  /*++
  
  Routine Description:
  
      This function locates the start and end of the kernel initialization
      code.  This code resides in the "init" section of the kernel image.
  
  Arguments:
  
      StartVa - Returns the starting address of the init section.
  
      EndVa - Returns the ending address of the init section.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  End of system initialization.
  
  --*/
  
  {
      PLDR_DATA_TABLE_ENTRY LdrDataTableEntry;
      PVOID CurrentBase;
      PVOID InitStart;
      PVOID InitEnd;
      PLIST_ENTRY Next;
      PIMAGE_NT_HEADERS NtHeader;
      PIMAGE_SECTION_HEADER SectionTableEntry;
      PIMAGE_SECTION_HEADER LastDiscard;
      LONG i;
      PVOID MiFindInitializationCodeAddress;
  
      MiFindInitializationCodeAddress = MmGetProcedureAddress((PVOID)&MiFindInitializationCode);
  
      *StartVa = NULL;
  
      //
      // Walk through the loader blocks looking for the base which
      // contains this routine.
      //
  
      KeEnterCriticalRegion();
      ExAcquireResourceExclusive (&PsLoadedModuleResource, TRUE);
      Next = PsLoadedModuleList.Flink;
  
      while ( Next != &PsLoadedModuleList ) {
          LdrDataTableEntry = CONTAINING_RECORD( Next,
                                                 LDR_DATA_TABLE_ENTRY,
                                                 InLoadOrderLinks
                                               );
          if (LdrDataTableEntry->SectionPointer != NULL) {
  
              //
              // This entry was loaded by MmLoadSystemImage so it's already
              // had its init section removed.
              //
  
              Next = Next->Flink;
              continue;
          }
  
          CurrentBase = (PVOID)LdrDataTableEntry->DllBase;
          NtHeader = RtlImageNtHeader(CurrentBase);
  
          SectionTableEntry = (PIMAGE_SECTION_HEADER)((PCHAR)NtHeader +
                                  sizeof(ULONG) +
                                  sizeof(IMAGE_FILE_HEADER) +
                                  NtHeader->FileHeader.SizeOfOptionalHeader);
  
          //
          // From the image header, locate the section named 'INIT'.
          //
  
          i = NtHeader->FileHeader.NumberOfSections;
  
          InitStart = NULL;
          while (i > 0) {
  
  #if DBG
              if ((*(PULONG)SectionTableEntry->Name == 'tini') ||
                  (*(PULONG)SectionTableEntry->Name == 'egap')) {
                  DbgPrint("driver %wZ has lower case sections (init or pagexxx)\n",
                      &LdrDataTableEntry->FullDllName);
              }
  #endif //DBG
  
              //
              // Free any INIT sections (or relocation sections that haven't
              // been already).  Note a driver may have a relocation section
              // but not have any INIT code.
              //
  
              if ((*(PULONG)SectionTableEntry->Name == 'TINI') ||
                  ((SectionTableEntry->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) != 0)) {
  
  
                  InitStart = (PVOID)((PCHAR)CurrentBase + SectionTableEntry->VirtualAddress);
                  InitEnd = (PVOID)((PCHAR)InitStart + SectionTableEntry->SizeOfRawData - 1);
                  InitEnd = (PVOID)((PCHAR)PAGE_ALIGN ((PCHAR)InitEnd +
                          (NtHeader->OptionalHeader.SectionAlignment - 1)) - 1);
                  InitStart = (PVOID)ROUND_TO_PAGES (InitStart);
  
                  //
                  // Check if more sections are discardable after this one so
                  // even small INIT sections can be discarded.
                  //
  
                  if (i == 1) {
                      LastDiscard = SectionTableEntry;
                  }
                  else {
                      LastDiscard = NULL;
                      do {
                          i -= 1;
                          SectionTableEntry += 1;
      
                          if ((SectionTableEntry->Characteristics &
                               IMAGE_SCN_MEM_DISCARDABLE) != 0) {
      
                              //
                              // Discard this too.
                              //
      
                              LastDiscard = SectionTableEntry;
                          } else {
                              break;
                          }
                      } while (i > 1);
                  }
  
                  if (LastDiscard) {
                      InitEnd = (PVOID)(((PCHAR)CurrentBase +
                                         LastDiscard->VirtualAddress) +
                                        (LastDiscard->SizeOfRawData - 1));
  
                      //
                      // If this isn't the last section in the driver then the
                      // the next section is not discardable.  So the last
                      // section is not rounded down, but all others must be.
                      //
  
                      if (i != 1) {
                          InitEnd = (PVOID)((PCHAR)PAGE_ALIGN ((PCHAR)InitEnd +
                                                               (NtHeader->OptionalHeader.SectionAlignment - 1)) - 1);
                      }
                  }
  
                  if (InitEnd > (PVOID)((PCHAR)CurrentBase +
                                        LdrDataTableEntry->SizeOfImage)) {
                      InitEnd = (PVOID)(((ULONG_PTR)CurrentBase +
                                         (LdrDataTableEntry->SizeOfImage - 1)) |
                                        (PAGE_SIZE - 1));
                  }
  
                  if (InitStart <= InitEnd) {
                      if ((MiFindInitializationCodeAddress >= InitStart) &&
                          (MiFindInitializationCodeAddress <= InitEnd)) {
  
                          //
                          // This init section is in the kernel, don't free it
                          // now as it would free this code!
                          //
  
                          *StartVa = InitStart;
                          *EndVa = InitEnd;
                      }
                      else {
                          MiFreeInitializationCode (InitStart, InitEnd);
                      }
                  }
              }
              i -= 1;
              SectionTableEntry += 1;
          }
          Next = Next->Flink;
      }
      ExReleaseResource (&PsLoadedModuleResource);
      KeLeaveCriticalRegion();
      return;
  }
  
  VOID
  MiFreeInitializationCode (
      IN PVOID StartVa,
      IN PVOID EndVa
      )
  
  /*++
  
  Routine Description:
  
      This function is called to delete the initialization code.
  
  Arguments:
  
      StartVa - Supplies the starting address of the range to delete.
  
      EndVa - Supplies the ending address of the range to delete.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  Runs after system initialization.
  
  --*/
  
  {
      PMMPFN Pfn1;
      PMMPTE PointerPte;
      PFN_NUMBER PageFrameIndex;
      KIRQL OldIrql;
      PVOID UnlockHandle;
  
      UnlockHandle = MmLockPagableCodeSection((PVOID)MiFreeInitializationCode);
      ASSERT(UnlockHandle);
  
      if (MI_IS_PHYSICAL_ADDRESS(StartVa)) {
          LOCK_PFN (OldIrql);
          while (StartVa < EndVa) {
  
              //
              // On certain architectures (e.g., MIPS) virtual addresses
              // may be physical and hence have no corresponding PTE.
              //
  
              PageFrameIndex = MI_CONVERT_PHYSICAL_TO_PFN (StartVa);
  
              Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
              Pfn1->u2.ShareCount = 0;
              Pfn1->u3.e2.ReferenceCount = 0;
              MI_SET_PFN_DELETED (Pfn1);
              MiInsertPageInList (MmPageLocationList[FreePageList], PageFrameIndex);
              StartVa = (PVOID)((PUCHAR)StartVa + PAGE_SIZE);
          }
          UNLOCK_PFN (OldIrql);
      } else {
          PointerPte = MiGetPteAddress (StartVa);
          MiDeleteSystemPagableVm (PointerPte,
                                   (PFN_NUMBER) (1 + MiGetPteAddress (EndVa) -
                                       PointerPte),
                                   ZeroKernelPte,
                                   FALSE,
                                   NULL);
      }
      MmUnlockPagableImageSection(UnlockHandle);
      return;
  }
  
  
  VOID
  MiEnablePagingTheExecutive (
      VOID
      )
  
  /*++
  
  Routine Description:
  
      This function locates the start and end of the kernel initialization
      code.  This code resides in the "init" section of the kernel image.
  
  Arguments:
  
      StartVa - Returns the starting address of the init section.
  
      EndVa - Returns the ending address of the init section.
  
  Return Value:
  
      None.
  
  Environment:
  
      Kernel Mode Only.  End of system initialization.
  
  --*/
  
  {
      PLDR_DATA_TABLE_ENTRY LdrDataTableEntry;
      PVOID CurrentBase;
      PLIST_ENTRY Next;
      PIMAGE_NT_HEADERS NtHeader;
      PIMAGE_SECTION_HEADER SectionTableEntry;
      LONG i;
      PMMPTE PointerPte;
      PMMPTE LastPte;
      BOOLEAN PageSection;
      PVOID SectionBaseAddress;
  
      //
      // Don't page kernel mode code if customer does not want it paged or if
      // this is a diskless remote boot client.
      //
  
      if (MmDisablePagingExecutive
  #if defined(REMOTE_BOOT)
          || (IoRemoteBootClient && IoCscInitializationFailed)
  #endif
          ) {
          return;
      }
  
      //
      // Walk through the loader blocks looking for the base which
      // contains this routine.
      //
  
      KeEnterCriticalRegion();
      ExAcquireResourceExclusive (&PsLoadedModuleResource, TRUE);
      Next = PsLoadedModuleList.Flink;
      while ( Next != &PsLoadedModuleList ) {
          LdrDataTableEntry = CONTAINING_RECORD( Next,
                                                 LDR_DATA_TABLE_ENTRY,
                                                 InLoadOrderLinks
                                               );
          if (LdrDataTableEntry->SectionPointer != NULL) {
  
              //
              // This entry was loaded by MmLoadSystemImage so it's already paged.
              //
  
              Next = Next->Flink;
              continue;
          }
  
          CurrentBase = (PVOID)LdrDataTableEntry->DllBase;
          NtHeader = RtlImageNtHeader(CurrentBase);
  
          SectionTableEntry = (PIMAGE_SECTION_HEADER)((PCHAR)NtHeader +
                                  sizeof(ULONG) +
                                  sizeof(IMAGE_FILE_HEADER) +
                                  NtHeader->FileHeader.SizeOfOptionalHeader);
  
          //
          // From the image header, locate the section named 'PAGE' or
          // '.edata'.
          //
  
          i = NtHeader->FileHeader.NumberOfSections;
  
          PointerPte = NULL;
  
          while (i > 0) {
  
              if (MI_IS_PHYSICAL_ADDRESS (CurrentBase)) {
  
                  //
                  // Mapped physically, can't be paged.
                  //
  
                  break;
              }
  
              SectionBaseAddress = SECTION_BASE_ADDRESS(SectionTableEntry);
  
              PageSection = ((*(PULONG)SectionTableEntry->Name == 'EGAP') ||
                            (*(PULONG)SectionTableEntry->Name == 'ade.')) &&
                             (SectionBaseAddress !=
                              ((PUCHAR)CurrentBase + SectionTableEntry->VirtualAddress));
  
              if (*(PULONG)SectionTableEntry->Name == 'EGAP' &&
                  SectionTableEntry->Name[4] == 'K'  &&
                  SectionTableEntry->Name[5] == 'D') {
  
                  //
                  // Only pageout PAGEKD if KdPitchDebugger is TRUE.
                  //
  
                  PageSection = KdPitchDebugger;
              }
  
              if ((*(PULONG)SectionTableEntry->Name == 'EGAP') &&
                  (*(PULONG)&SectionTableEntry->Name[4] == 'RDYH')) {
  
                  //
                  // Pageout PAGEHYDRA on non Hydra systems.
                  //
  
                  if (MiHydra == TRUE) {
                      PageSection = FALSE;
                  }
              }
  
              if ((*(PULONG)SectionTableEntry->Name == 'EGAP') &&
                  (*(PULONG)&SectionTableEntry->Name[4] == 'YFRV')) {
  
                  //
                  // Pageout PAGEVRFY if no drivers are being instrumented.
                  //
  
                  if (MmVerifyDriverBufferLength != (ULONG)-1) {
                      PageSection = FALSE;
                  }
              }
  
              if ((*(PULONG)SectionTableEntry->Name == 'EGAP') &&
                  (*(PULONG)&SectionTableEntry->Name[4] == 'CEPS')) {
  
                  //
                  // Pageout PAGESPEC special pool code if it's not enabled.
                  //
  
                  if (MiSpecialPoolPtes != 0) {
                      PageSection = FALSE;
                  }
              }
  
              if (PageSection) {
                   //
                   // This section is pagable, save away the start and end.
                   //
  
                   if (PointerPte == NULL) {
  
                       //
                       // Previous section was NOT pagable, get the start address.
                       //
  
                       PointerPte = MiGetPteAddress (ROUND_TO_PAGES (
                                    (ULONG_PTR)CurrentBase +
                                    SectionTableEntry->VirtualAddress));
                   }
                   LastPte = MiGetPteAddress ((ULONG_PTR)CurrentBase +
                               SectionTableEntry->VirtualAddress +
                               (NtHeader->OptionalHeader.SectionAlignment - 1) +
                               SectionTableEntry->SizeOfRawData -
                               PAGE_SIZE);
  
              } else {
  
                  //
                  // This section is not pagable, if the previous section was
                  // pagable, enable it.
                  //
  
                  if (PointerPte != NULL) {
                      MiEnablePagingOfDriverAtInit (PointerPte, LastPte);
                      PointerPte = NULL;
                  }
              }
              i -= 1;
              SectionTableEntry += 1;
          } //end while
  
          if (PointerPte != NULL) {
              MiEnablePagingOfDriverAtInit (PointerPte, LastPte);
          }
  
          Next = Next->Flink;
      } //end while
  
      ExReleaseResource (&PsLoadedModuleResource);
      KeLeaveCriticalRegion();
  
      return;
  }
  
  
  VOID
  MiEnablePagingOfDriverAtInit (
      IN PMMPTE PointerPte,
      IN PMMPTE LastPte
      )
  
  /*++
  
  Routine Description:
  
      This routine marks the specified range of PTEs as pagable.
  
  Arguments:
  
      PointerPte - Supplies the starting PTE.
  
      LastPte - Supplies the ending PTE.
  
  Return Value:
  
      None.
  
  --*/
  
  {
      PVOID Base;
      PFN_NUMBER PageFrameIndex;
      PMMPFN Pfn;
      MMPTE TempPte;
      KIRQL OldIrql;
      KIRQL OldIrqlWs;
      LOGICAL SessionAddress;
  
      Base = MiGetVirtualAddressMappedByPte (PointerPte);
      SessionAddress = MI_IS_SESSION_PTE (PointerPte);
  
      LOCK_SYSTEM_WS (OldIrqlWs);
      LOCK_PFN (OldIrql);
  
      while (PointerPte <= LastPte) {
  
          //
          // The PTE must be carefully checked as drivers may call MmPageEntire
          // during their DriverEntry yet faults may occur prior to this routine
          // running which cause pages to already be resident and in the working
          // set at this point.  So checks for validity and wsindex must be
          // applied.
          //
  
          if (PointerPte->u.Hard.Valid == 1) {
              PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
              Pfn = MI_PFN_ELEMENT (PageFrameIndex);
              ASSERT (Pfn->u2.ShareCount == 1);
      
              if (Pfn->u1.WsIndex == 0) {
  
                  //
                  // Set the working set index to zero.  This allows page table
                  // pages to be brought back in with the proper WSINDEX.
                  //
      
                  MI_ZERO_WSINDEX (Pfn);
  
                  //
                  // Original PTE may need to be set for drivers loaded via
                  // ntldr.
                  //
  
                  if (Pfn->OriginalPte.u.Long == 0) {
                      Pfn->OriginalPte.u.Long = MM_KERNEL_DEMAND_ZERO_PTE;
  #if defined(_IA64_)
                      Pfn->OriginalPte.u.Soft.Protection |= MM_EXECUTE;
  #endif
                  }
  
                  Pfn->u3.e1.Modified = 1;
                  TempPte = *PointerPte;
          
                  MI_MAKE_VALID_PTE_TRANSITION (TempPte,
                                                Pfn->OriginalPte.u.Soft.Protection);
          
                  KeFlushSingleTb (Base,
                                   TRUE,
                                   TRUE,
                                   (PHARDWARE_PTE)PointerPte,
                                   TempPte.u.Flush);
          
                  //
                  // Flush the translation buffer and decrement the number of valid
                  // PTEs within the containing page table page.  Note that for a
                  // private page, the page table page is still needed because the
                  // page is in transition.
                  //
          
                  MiDecrementShareCount (PageFrameIndex);
      
                  MmResidentAvailablePages += 1;
                  MmTotalSystemCodePages += 1;
              }
              else {
  
                  //
                  // This would need to be taken out of the WSLEs so skip it for
                  // now and let the normal paging algorithms remove it if we
                  // run into memory pressure.
                  //
              }
  
          }
          Base = (PVOID)((PCHAR)Base + PAGE_SIZE);
          PointerPte += 1;
      }
  
      UNLOCK_PFN (OldIrql);
      UNLOCK_SYSTEM_WS (OldIrqlWs);
  
      if (SessionAddress == TRUE) {
  
          //
          // Session space has no ASN - flush the entire TB.
          //
      
          MI_FLUSH_ENTIRE_SESSION_TB (TRUE, TRUE);
      }
  
      return;
  }
  
  
  MM_SYSTEMSIZE
  MmQuerySystemSize(
      VOID
      )
  {
      //
      // 12Mb  is small
      // 12-19 is medium
      // > 19 is large
      //
      return MmSystemSize;
  }
  
  NTKERNELAPI
  BOOLEAN
  MmIsThisAnNtAsSystem(
      VOID
      )
  {
      return (BOOLEAN)MmProductType;
  }
  
  NTKERNELAPI
  VOID
  FASTCALL
  MmSetPageFaultNotifyRoutine(
      PPAGE_FAULT_NOTIFY_ROUTINE NotifyRoutine
      )
  {
      MmPageFaultNotifyRoutine = NotifyRoutine;
  }
  
  NTKERNELAPI
  VOID
  FASTCALL
  MmSetHardFaultNotifyRoutine(
      PHARD_FAULT_NOTIFY_ROUTINE NotifyRoutine
      )
  {
      MmHardFaultNotifyRoutine = NotifyRoutine;
  }
  