https://git.reactos.org/?p=reactos.git;a=commitdiff;h=43378411fb258a02d3384acb3e301eb1774a119a
commit 43378411fb258a02d3384acb3e301eb1774a119a Author: Jérôme Gardou <[email protected]> AuthorDate: Mon Mar 22 09:52:11 2021 +0100 Commit: Jérôme Gardou <[email protected]> CommitDate: Thu Apr 8 15:40:37 2021 +0200 [NTOS:MM] Rewrite arch-specifics of the legacy Mm Properly handle PDE refcounting Clean-up of the internal API Enforce attaching to the process when modifying its memory layout, instead of making circonvoluted mappings which always end up being broken. --- ntoskrnl/include/internal/mm.h | 48 +-- ntoskrnl/mm/i386/page.c | 843 +++++++++++++++++++---------------------- ntoskrnl/mm/marea.c | 18 - ntoskrnl/mm/rmap.c | 30 +- ntoskrnl/mm/section.c | 17 +- 5 files changed, 429 insertions(+), 527 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 93c44e5bd05..d2cb0123bed 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1034,8 +1034,7 @@ MmCreateVirtualMapping( struct _EPROCESS* Process, PVOID Address, ULONG flProtect, - PPFN_NUMBER Pages, - ULONG PageCount + PFN_NUMBER Page ); NTSTATUS @@ -1044,8 +1043,7 @@ MmCreateVirtualMappingUnsafe( struct _EPROCESS* Process, PVOID Address, ULONG flProtect, - PPFN_NUMBER Pages, - ULONG PageCount + PFN_NUMBER Page ); ULONG @@ -1080,13 +1078,6 @@ VOID NTAPI MmInitGlobalKernelPageDirectory(VOID); -VOID -NTAPI -MmGetPageFileMapping( - struct _EPROCESS *Process, - PVOID Address, - SWAPENTRY* SwapEntry); - VOID NTAPI MmDeletePageFileMapping( @@ -1103,16 +1094,16 @@ MmCreatePageFileMapping( SWAPENTRY SwapEntry ); -BOOLEAN +VOID NTAPI -MmIsPageSwapEntry( - struct _EPROCESS *Process, - PVOID Address -); +MmGetPageFileMapping( + PEPROCESS Process, + PVOID Address, + SWAPENTRY *SwapEntry); -VOID +BOOLEAN NTAPI -MmSetDirtyPage( +MmIsPageSwapEntry( struct _EPROCESS *Process, PVOID Address ); @@ -1156,6 +1147,12 @@ MmSetCleanPage( PVOID Address ); +VOID +NTAPI +MmSetDirtyBit(PEPROCESS Process, PVOID Address, BOOLEAN Bit); +#define MmSetCleanPage(__P, __A) MmSetDirtyBit(__P, __A, FALSE) +#define MmSetDirtyPage(__P, __A) MmSetDirtyBit(__P, __A, TRUE) + VOID NTAPI MmDeletePageTable( @@ -1208,21 +1205,6 @@ MmDeleteVirtualMapping( PPFN_NUMBER Page ); -BOOLEAN -NTAPI -MmIsDirtyPage( - struct _EPROCESS *Process, - PVOID Address -); - -VOID -NTAPI -MmClearPageAccessedBit(PEPROCESS Process, PVOID Address); - -BOOLEAN -NTAPI -MmIsPageAccessed(PEPROCESS Process, PVOID Address); - /* wset.c ********************************************************************/ NTSTATUS diff --git a/ntoskrnl/mm/i386/page.c b/ntoskrnl/mm/i386/page.c index e1aac494dcb..35b0921d0ae 100644 --- a/ntoskrnl/mm/i386/page.c +++ b/ntoskrnl/mm/i386/page.c @@ -134,17 +134,6 @@ ULONG MmProtectToValue[32] = /* FUNCTIONS ***************************************************************/ -static BOOLEAN MmUnmapPageTable(PULONG Pt, KIRQL OldIrql); - -VOID -MiFlushTlb(PULONG Pt, PVOID Address, KIRQL OldIrql) -{ - if ((Pt && MmUnmapPageTable(Pt, OldIrql)) || Address >= MmSystemRangeStart) - { - KeInvalidateTlbEntry(Address); - } -} - static ULONG ProtectToPTE(ULONG flProtect) { @@ -191,120 +180,38 @@ NTAPI MiFillSystemPageDirectory(IN PVOID Base, IN SIZE_T NumberOfBytes); -static PULONG -MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create, PKIRQL OldIrql) +PFN_NUMBER +NTAPI +MmGetPfnForProcess(PEPROCESS Process, + PVOID Address) { - PFN_NUMBER Pfn; - PULONG Pt; - PMMPDE PointerPde; - - if (Address < MmSystemRangeStart) - { - /* We should have a process for user land addresses */ - ASSERT(Process != NULL); + PMMPTE PointerPte; + PFN_NUMBER Page; - if(Process != PsGetCurrentProcess()) - { - PMMPDE PdeBase; - ULONG PdeOffset = MiGetPdeOffset(Address); - - ASSERT(!Create); - - PdeBase = MiMapPageInHyperSpace(PsGetCurrentProcess(), - PTE_TO_PFN(Process->Pcb.DirectoryTableBase[0]), - OldIrql); - if (PdeBase == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - PointerPde = PdeBase + PdeOffset; - if (PointerPde->u.Hard.Valid == 0) - { - MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PdeBase, *OldIrql); - return NULL; - } - - Pfn = PointerPde->u.Hard.PageFrameNumber; - MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PdeBase, *OldIrql); - Pt = MiMapPageInHyperSpace(PsGetCurrentProcess(), Pfn, OldIrql); - if (Pt == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - return Pt + MiAddressToPteOffset(Address); - } + /* Must be called for user mode only */ + ASSERT(Process != NULL); + ASSERT(Address < MmSystemRangeStart); - /* This is for our process */ - PointerPde = MiAddressToPde(Address); - Pt = (PULONG)MiAddressToPte(Address); + /* And for our process */ + ASSERT(Process == PsGetCurrentProcess()); - if ((PointerPde->u.Hard.Valid == 0) && (Create == FALSE)) - { - /* Do not fault PDE in if not needed */ - return NULL; - } + /* Lock for reading */ + MiLockProcessWorkingSetShared(Process, PsGetCurrentThread()); - return (PULONG)MiAddressToPte(Address); - } - - /* This is for kernel land address */ - ASSERT(Process == NULL); - PointerPde = MiAddressToPde(Address); - Pt = (PULONG)MiAddressToPte(Address); - if (PointerPde->u.Hard.Valid == 0) + if (MiQueryPageTableReferences(Address) == 0) { - /* Let ARM3 synchronize the PDE */ - if(!MiSynchronizeSystemPde(PointerPde)) - { - /* PDE (still) not valid, let ARM3 allocate one if asked */ - if(Create == FALSE) - return NULL; - MiFillSystemPageDirectory(Address, PAGE_SIZE); - } - } - return Pt; -} - -static BOOLEAN MmUnmapPageTable(PULONG Pt, KIRQL OldIrql) -{ - if (!IS_HYPERSPACE(Pt)) - { - return TRUE; + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + return 0; } - MiUnmapPageInHyperSpace(PsGetCurrentProcess(), Pt, OldIrql); - - return FALSE; -} - -static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address) -{ - ULONG Pte; - PULONG Pt; - KIRQL OldIrql; + /* Make sure we can read the PTE */ + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); - if (Pt) - { - Pte = *Pt; - MmUnmapPageTable(Pt, OldIrql); - return Pte; - } - return 0; -} + PointerPte = MiAddressToPte(Address); + Page = PointerPte->u.Hard.Valid ? PFN_FROM_PTE(PointerPte) : 0; -PFN_NUMBER -NTAPI -MmGetPfnForProcess(PEPROCESS Process, - PVOID Address) -{ - ULONG Entry; - Entry = MmGetPageEntryForProcess(Process, Address); - if (!(Entry & PA_PRESENT)) - { - return 0; - } - return(PTE_TO_PFN(Entry)); + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + return Page; } VOID @@ -315,130 +222,150 @@ MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, * FUNCTION: Delete a virtual mapping */ { - BOOLEAN WasValid = FALSE; - PFN_NUMBER Pfn; + PMMPTE PointerPte; ULONG Pte; - PULONG Pt; - KIRQL OldIrql; - DPRINT("MmDeleteVirtualMapping(%p, %p, %p, %p)\n", - Process, Address, WasDirty, Page); + DPRINT("MmDeleteVirtualMapping(%p, %p, %p, %p)\n", Process, Address, WasDirty, Page); + + ASSERT(((ULONG_PTR)Address % PAGE_SIZE) == 0); - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); + /* And we should be at low IRQL */ + ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL); - if (Pt == NULL) + /* Make sure our PDE is valid, and that everything is going fine */ + if (Process == NULL) { - if (WasDirty != NULL) + if (Address < MmSystemRangeStart) { - *WasDirty = FALSE; + DPRINT1("NULL process given for user-mode mapping at %p\n", Address); + KeBugCheck(MEMORY_MANAGEMENT); } - if (Page != NULL) + + if (!MiSynchronizeSystemPde(MiAddressToPde(Address))) { - *Page = 0; + /* There can't be a page if there is no PDE */ + if (WasDirty) + *WasDirty = FALSE; + if (Page) + *Page = 0; + return; } - return; } - - /* - * Atomically set the entry to zero and get the old value. - */ - Pte = InterlockedExchangePte(Pt, 0); - - /* We count a mapping as valid if it's a present page, or it's a nonzero pfn with - * the swap bit unset, indicating a valid page protected to PAGE_NOACCESS. */ - WasValid = (Pte & PA_PRESENT) || ((Pte >> PAGE_SHIFT) && !(Pte & 0x800)); - if (WasValid) + else { - /* Flush the TLB since we transitioned this PTE - * from valid to invalid so any stale translations - * are removed from the cache */ - MiFlushTlb(Pt, Address, OldIrql); + if ((Address >= MmSystemRangeStart) || Add2Ptr(Address, PAGE_SIZE) >= MmSystemRangeStart) + { + DPRINT1("Process %p given for kernel-mode mapping at %p -- %lu pages starting at %Ix\n", Process, Address); + KeBugCheck(MEMORY_MANAGEMENT); + } - if (Address < MmSystemRangeStart) - { - /* Remove PDE reference */ - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_PER_PAGE); - } + /* Only for current process !!! */ + ASSERT(Process = PsGetCurrentProcess()); + MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); - Pfn = PTE_TO_PFN(Pte); - } - else - { - MmUnmapPageTable(Pt, OldIrql); - Pfn = 0; + /* No PDE --> No page */ + if (MiQueryPageTableReferences(Address) == 0) + { + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + if (WasDirty) + *WasDirty = 0; + if (Page) + *Page = 0; + return; + } + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); } - /* - * Return some information to the caller - */ - if (WasDirty != NULL) + PointerPte = MiAddressToPte(Address); + Pte = InterlockedExchangePte(PointerPte, 0); + + if (Pte == 0) { - *WasDirty = ((Pte & PA_DIRTY) && (Pte & PA_PRESENT)) ? TRUE : FALSE; + /* There was nothing here */ + if (Address < MmSystemRangeStart) + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + if (WasDirty) + *WasDirty = 0; + if (Page) + *Page = 0; + return; } - if (Page != NULL) + + /* It must have been present, or not a swap entry */ + ASSERT(FlagOn(Pte, PA_PRESENT) || !FlagOn(Pte, 0x800)); + + if (FlagOn(Pte, PA_PRESENT)) + KeInvalidateTlbEntry(Address); + + if (Address < MmSystemRangeStart) { - *Page = Pfn; + /* Remove PDE reference */ + MiDecrementPageTableReferences(Address); + if (MiQueryPageTableReferences(Address) == 0) + { + KIRQL OldIrql = MiAcquirePfnLock(); + MiDeletePte(MiAddressToPte(PointerPte), PointerPte, Process, NULL); + MiReleasePfnLock(OldIrql); + } + + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); } -} -VOID -NTAPI -MmGetPageFileMapping(PEPROCESS Process, PVOID Address, - SWAPENTRY* SwapEntry) -/* - * FUNCTION: Get a page file mapping - */ -{ - ULONG Entry = MmGetPageEntryForProcess(Process, Address); - *SwapEntry = Entry >> 1; + if (WasDirty) + *WasDirty = FlagOn(Pte, PA_DIRTY); + if (Page) + *Page = PTE_TO_PFN(Pte); } + VOID NTAPI -MmDeletePageFileMapping(PEPROCESS Process, PVOID Address, - SWAPENTRY* SwapEntry) -/* - * FUNCTION: Delete a virtual mapping - */ +MmDeletePageFileMapping( + PEPROCESS Process, + PVOID Address, + SWAPENTRY* SwapEntry) { + PMMPTE PointerPte; ULONG Pte; - PULONG Pt; - KIRQL OldIrql; - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); + /* This should not be called for kernel space anymore */ + ASSERT(Process != NULL); + ASSERT(Address < MmSystemRangeStart); - if (Pt == NULL) - { - *SwapEntry = 0; - return; - } + /* And we don't support deleting for other process */ + ASSERT(Process == PsGetCurrentProcess()); - /* - * Atomically set the entry to zero and get the old value. - */ - Pte = InterlockedExchangePte(Pt, 0); + /* And we should be at low IRQL */ + ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL); - if (Address < MmSystemRangeStart) - { - /* Remove PDE reference */ - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_PER_PAGE); - } + /* We are tinkering with the PDE here. Ensure it will be there */ + MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); - /* We don't need to flush here because page file entries - * are invalid translations, so the processor won't cache them */ - MmUnmapPageTable(Pt, OldIrql); + /* Callers must ensure there is actually something there */ + ASSERT(MiAddressToPde(Address)->u.Long != 0); - if ((Pte & PA_PRESENT) || !(Pte & 0x800)) + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + + PointerPte = MiAddressToPte(Address); + Pte = InterlockedExchangePte(PointerPte, 0); + if (!FlagOn(Pte, 0x800) || FlagOn(Pte, PA_PRESENT)) { - DPRINT1("Pte %x (want not 1 and 0x800)\n", Pte); - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheckEx(MEMORY_MANAGEMENT, Pte, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); } - /* - * Return some information to the caller - */ + /* This used to be a non-zero PTE, now we can let the PDE go. */ + MiDecrementPageTableReferences(Address); + if (MiQueryPageTableReferences(Address) == 0) + { + /* We can let it go */ + KIRQL OldIrql = MiAcquirePfnLock(); + MiDeletePte(MiAddressToPte(PointerPte), PointerPte, Process, NULL); + MiReleasePfnLock(OldIrql); + } + + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + *SwapEntry = Pte >> 1; } @@ -459,149 +386,160 @@ Mmi386MakeKernelPageTableGlobal(PVOID Address) BOOLEAN NTAPI -MmIsDirtyPage(PEPROCESS Process, PVOID Address) +MmIsPagePresent(PEPROCESS Process, PVOID Address) { - return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE; -} + BOOLEAN Ret; -VOID -NTAPI -MmSetCleanPage(PEPROCESS Process, PVOID Address) -{ - PULONG Pt; - ULONG Pte; - KIRQL OldIrql; - - if (Address < MmSystemRangeStart && Process == NULL) + if (Address >= MmSystemRangeStart) { - DPRINT1("MmSetCleanPage is called for user space without a process.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } + ASSERT(Process == NULL); + if (!MiSynchronizeSystemPde(MiAddressToPde(Address))) + { + /* It can't be present if there is no PDE */ + return FALSE; + } - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); - if (Pt == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); + return MiAddressToPte(Address)->u.Hard.Valid; } - do - { - Pte = *Pt; - } while (Pte != InterlockedCompareExchangePte(Pt, Pte & ~PA_DIRTY, Pte)); + ASSERT(Process != NULL); + ASSERT(Process == PsGetCurrentProcess()); - if (!(Pte & PA_PRESENT)) + MiLockProcessWorkingSetShared(Process, PsGetCurrentThread()); + + if (MiQueryPageTableReferences(Address) == 0) { - KeBugCheck(MEMORY_MANAGEMENT); + /* It can't be present if there is no PDE */ + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + return FALSE; } - else if (Pte & PA_DIRTY) + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + + Ret = MiAddressToPte(Address)->u.Hard.Valid; + + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + + return Ret; +} + +BOOLEAN +NTAPI +MmIsDisabledPage(PEPROCESS Process, PVOID Address) +{ + BOOLEAN Ret; + ULONG Pte; + + if (Address >= MmSystemRangeStart) { - MiFlushTlb(Pt, Address, OldIrql); + ASSERT(Process == NULL); + if (!MiSynchronizeSystemPde(MiAddressToPde(Address))) + { + /* It's not disabled if it's not present */ + return FALSE; + } + + Pte = MiAddressToPte(Address)->u.Long; } else { - MmUnmapPageTable(Pt, OldIrql); + ASSERT(Process != NULL); + ASSERT(Process == PsGetCurrentProcess()); + + MiLockProcessWorkingSetShared(Process, PsGetCurrentThread()); + + if (MiQueryPageTableReferences(Address) == 0) + { + /* It can't be disabled if there is no PDE */ + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + return FALSE; + } + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + + Pte = MiAddressToPte(Address)->u.Long; } + + Ret = !FlagOn(Pte, PA_PRESENT) && !FlagOn(Pte, 0x800) && (PAGE_MASK(Pte) != 0); + + if (Address < MmSystemRangeStart) + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + + return Ret; } -VOID +BOOLEAN NTAPI -MmSetDirtyPage(PEPROCESS Process, PVOID Address) +MmIsPageSwapEntry(PEPROCESS Process, PVOID Address) { - PULONG Pt; + BOOLEAN Ret; ULONG Pte; - KIRQL OldIrql; - if (Address < MmSystemRangeStart && Process == NULL) + /* We never set swap entries for kernel addresses */ + if (Address >= MmSystemRangeStart) { - DPRINT1("MmSetDirtyPage is called for user space without a process.\n"); - KeBugCheck(MEMORY_MANAGEMENT); + ASSERT(Process == NULL); + return FALSE; } - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); - if (Pt == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); - } + ASSERT(Process != NULL); + ASSERT(Process == PsGetCurrentProcess()); - do - { - Pte = *Pt; - } while (Pte != InterlockedCompareExchangePte(Pt, Pte | PA_DIRTY, Pte)); + MiLockProcessWorkingSetShared(Process, PsGetCurrentThread()); - if (!(Pte & PA_PRESENT)) + if (MiQueryPageTableReferences(Address) == 0) { - KeBugCheck(MEMORY_MANAGEMENT); - } - else - { - /* The processor will never clear this bit itself, therefore - * we do not need to flush the TLB here when setting it */ - MmUnmapPageTable(Pt, OldIrql); + /* There can't be a swap entry if there is no PDE */ + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + return FALSE; } + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + + Pte = MiAddressToPte(Address)->u.Long; + Ret = !FlagOn(Pte, PA_PRESENT) && FlagOn(Pte, 0x800); + + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + + return Ret; } VOID NTAPI -MmClearPageAccessedBit(PEPROCESS Process, PVOID Address) +MmGetPageFileMapping(PEPROCESS Process, PVOID Address, SWAPENTRY* SwapEntry) { - PULONG Pt; - LONG Pte; - KIRQL OldIrql; + ULONG Pte; - if (Address < MmSystemRangeStart && Process == NULL) + /* We never set swap entries for kernel addresses */ + if (Address >= MmSystemRangeStart) { - DPRINT1("MmClearPageAccessedBit is called for user space without a process.\n"); - KeBugCheck(MEMORY_MANAGEMENT); + ASSERT(Process == NULL); + *SwapEntry = 0; + return; } - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); - if (Pt == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); - } + ASSERT(Process != NULL); + ASSERT(Process == PsGetCurrentProcess()); - do - { - Pte = *Pt; - } while (Pte != InterlockedCompareExchangePte(Pt, Pte & ~PA_ACCESSED, Pte)); + MiLockProcessWorkingSetShared(Process, PsGetCurrentThread()); - if (!(Pte & PA_PRESENT)) + if (MiQueryPageTableReferences(Address) == 0) { - KeBugCheck(MEMORY_MANAGEMENT); + /* There can't be a swap entry if there is no PDE */ + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + *SwapEntry = 0; + return; } - MiFlushTlb(Pt, Address, OldIrql); -} + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); -BOOLEAN -NTAPI -MmIsPageAccessed(PEPROCESS Process, PVOID Address) -{ - return BooleanFlagOn(MmGetPageEntryForProcess(Process, Address), PA_ACCESSED); -} - -BOOLEAN -NTAPI -MmIsPagePresent(PEPROCESS Process, PVOID Address) -{ - return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT; -} - -BOOLEAN -NTAPI -MmIsDisabledPage(PEPROCESS Process, PVOID Address) -{ - ULONG_PTR Entry = MmGetPageEntryForProcess(Process, Address); - return !(Entry & PA_PRESENT) && !(Entry & 0x800) && (Entry >> PAGE_SHIFT); -} + Pte = MiAddressToPte(Address)->u.Long; + if (!FlagOn(Pte, PA_PRESENT) && FlagOn(Pte, 0x800)) + *SwapEntry = Pte >> 1; + else + *SwapEntry = 0; -BOOLEAN -NTAPI -MmIsPageSwapEntry(PEPROCESS Process, PVOID Address) -{ - ULONG Entry; - Entry = MmGetPageEntryForProcess(Process, Address); - return !(Entry & PA_PRESENT) && (Entry & 0x800); + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); } NTSTATUS @@ -610,56 +548,39 @@ MmCreatePageFileMapping(PEPROCESS Process, PVOID Address, SWAPENTRY SwapEntry) { - PULONG Pt; + PMMPTE PointerPte; ULONG Pte; - KIRQL OldIrql; - if (Process == NULL && Address < MmSystemRangeStart) - { - DPRINT1("No process\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - if (Process != NULL && Address >= MmSystemRangeStart) - { - DPRINT1("Setting kernel address with process context\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } + /* This should not be called for kernel space anymore */ + ASSERT(Process != NULL); + ASSERT(Address < MmSystemRangeStart); + + /* And we don't support creating for other process */ + ASSERT(Process == PsGetCurrentProcess()); if (SwapEntry & (1 << 31)) { KeBugCheck(MEMORY_MANAGEMENT); } - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); - if (Pt == NULL) - { - /* Nobody should page out an address that hasn't even been mapped */ - /* But we might place a wait entry first, requiring the page table */ - if (SwapEntry != MM_WAIT_ENTRY) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - Pt = MmGetPageTableForProcess(Process, Address, TRUE, &OldIrql); - } - Pte = InterlockedExchangePte(Pt, SwapEntry << 1); + /* We are tinkering with the PDE here. Ensure it will be there */ + ASSERT(Process == PsGetCurrentProcess()); + MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + + PointerPte = MiAddressToPte(Address); + Pte = InterlockedExchangePte(PointerPte, SwapEntry << 1); if (Pte != 0) { KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); } - if (Address < MmSystemRangeStart) - { - /* Add PDE reference */ - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_PER_PAGE); - } + /* This used to be a 0 PTE, now we need a valid PDE to keep it around */ + MiIncrementPageTableReferences(Address); + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); - /* We don't need to flush the TLB here because it - * only caches valid translations and a zero PTE - * is not a valid translation */ - MmUnmapPageTable(Pt, OldIrql); - - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } @@ -668,50 +589,41 @@ NTAPI MmCreateVirtualMappingUnsafe(PEPROCESS Process, PVOID Address, ULONG flProtect, - PPFN_NUMBER Pages, - ULONG PageCount) + PFN_NUMBER Page) { ULONG Attributes; - PVOID Addr; - ULONG i; - ULONG oldPdeOffset, PdeOffset; - PULONG Pt = NULL; + PMMPTE PointerPte; ULONG Pte; - KIRQL OldIrql; - DPRINT("MmCreateVirtualMappingUnsafe(%p, %p, %lu, %p (%x), %lu)\n", - Process, Address, flProtect, Pages, *Pages, PageCount); + DPRINT("MmCreateVirtualMappingUnsafe(%p, %p, %lu, %x)\n", + Process, Address, flProtect, Page); ASSERT(((ULONG_PTR)Address % PAGE_SIZE) == 0); + /* Make sure our PDE is valid, and that everything is going fine */ if (Process == NULL) { if (Address < MmSystemRangeStart) { - DPRINT1("NULL process given for user-mode mapping at %p -- %lu pages starting at %Ix\n", Address, PageCount, *Pages); - KeBugCheck(MEMORY_MANAGEMENT); - } - if (PageCount > 0x10000 || - (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000) - { - DPRINT1("Page count too large for kernel-mode mapping at %p -- %lu pages starting at %Ix\n", Address, PageCount, *Pages); + DPRINT1("NULL process given for user-mode mapping at %p\n", Address); KeBugCheck(MEMORY_MANAGEMENT); } + if (!MiSynchronizeSystemPde(MiAddressToPde(Address))) + MiFillSystemPageDirectory(Address, PAGE_SIZE); } else { - if (Address >= MmSystemRangeStart) + if ((Address >= MmSystemRangeStart) || Add2Ptr(Address, PAGE_SIZE) >= MmSystemRangeStart) { - DPRINT1("Process %p given for kernel-mode mapping at %p -- %lu pages starting at %Ix\n", Process, Address, PageCount, *Pages); - KeBugCheck(MEMORY_MANAGEMENT); - } - if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE || - (ULONG_PTR) Address / PAGE_SIZE + PageCount > - (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE) - { - DPRINT1("Page count too large for process %p user-mode mapping at %p -- %lu pages starting at %Ix\n", Process, Address, PageCount, *Pages); + DPRINT1("Process %p given for kernel-mode mapping at %p -- %lu pages starting at %Ix\n", Process, Address); KeBugCheck(MEMORY_MANAGEMENT); } + + /* Only for current process !!! */ + ASSERT(Process = PsGetCurrentProcess()); + MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); } Attributes = ProtectToPTE(flProtect); @@ -725,58 +637,28 @@ MmCreateVirtualMappingUnsafe(PEPROCESS Process, Attributes |= PA_USER; } - Addr = Address; - /* MmGetPageTableForProcess should be called on the first run, so - * let this trigger it */ - oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1; - for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) - { - if (!(Attributes & PA_PRESENT) && Pages[i] != 0) - { - DPRINT1("Setting physical address but not allowing access at address " - "0x%p with attributes %x/%x.\n", - Addr, Attributes, flProtect); - KeBugCheck(MEMORY_MANAGEMENT); - } - PdeOffset = ADDR_TO_PDE_OFFSET(Addr); - if (oldPdeOffset != PdeOffset) - { - if(Pt) MmUnmapPageTable(Pt, OldIrql); - Pt = MmGetPageTableForProcess(Process, Addr, TRUE, &OldIrql); - if (Pt == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - } - else - { - Pt++; - } - oldPdeOffset = PdeOffset; - - Pte = InterlockedExchangePte(Pt, PFN_TO_PTE(Pages[i]) | Attributes); + /* This must be for a valid address */ + ASSERT(FlagOn(Attributes, PA_PRESENT)); - /* There should not be anything valid here */ - if (Pte != 0) - { - DPRINT1("Bad PTE %lx at %p for %p + %lu\n", Pte, Pt, Address, i); - KeBugCheck(MEMORY_MANAGEMENT); - } + PointerPte = MiAddressToPte(Address); + Pte = InterlockedExchangePte(PointerPte, PFN_TO_PTE(Page) | Attributes); + /* There should not have been anything valid here */ + if (Pte != 0) + { + DPRINT1("Bad PTE %lx at %p for %p\n", Pte, PointerPte, Address); + KeBugCheck(MEMORY_MANAGEMENT); + } - /* We don't need to flush the TLB here because it only caches valid translations - * and we're moving this PTE from invalid to valid so it can't be cached right now */ + /* We don't need to flush the TLB here because it only caches valid translations + * and we're moving this PTE from invalid to valid so it can't be cached right now */ - if (Addr < MmSystemRangeStart) - { - /* Add PDE reference */ - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Addr)]++; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Addr)] <= PTE_PER_PAGE); - } + if (Address < MmSystemRangeStart) + { + /* Add PDE reference */ + MiIncrementPageTableReferences(Address); + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); } - ASSERT(Addr > Address); - MmUnmapPageTable(Pt, OldIrql); - return(STATUS_SUCCESS); } @@ -785,45 +667,60 @@ NTAPI MmCreateVirtualMapping(PEPROCESS Process, PVOID Address, ULONG flProtect, - PPFN_NUMBER Pages, - ULONG PageCount) + PFN_NUMBER Page) { - ULONG i; - ASSERT((ULONG_PTR)Address % PAGE_SIZE == 0); - for (i = 0; i < PageCount; i++) + if (!MmIsPageInUse(Page)) { - if (!MmIsPageInUse(Pages[i])) - { - DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages[i])); - KeBugCheck(MEMORY_MANAGEMENT); - } + DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Page)); + KeBugCheck(MEMORY_MANAGEMENT); } - return(MmCreateVirtualMappingUnsafe(Process, - Address, - flProtect, - Pages, - PageCount)); + return MmCreateVirtualMappingUnsafe(Process, Address, flProtect, Page); } ULONG NTAPI MmGetPageProtect(PEPROCESS Process, PVOID Address) { - ULONG Entry; + ULONG_PTR Pte; ULONG Protect; - Entry = MmGetPageEntryForProcess(Process, Address); + if (Address >= MmSystemRangeStart) + { + ASSERT(Process == NULL); + + if (!MiSynchronizeSystemPde(MiAddressToPde(Address))) + return PAGE_NOACCESS; + } + else + { + ASSERT(Address < MmSystemRangeStart); + ASSERT(Process != NULL); + + ASSERT(Process == PsGetCurrentProcess()); + MiLockProcessWorkingSetShared(Process, PsGetCurrentThread()); - if (!(Entry & PA_PRESENT)) + if (MiQueryPageTableReferences(Address) == 0) + { + /* It can't be present if there is no PDE */ + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + return PAGE_NOACCESS; + } + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + } + + Pte = MiAddressToPte(Address)->u.Long; + + if (!(Pte & PA_PRESENT)) { Protect = PAGE_NOACCESS; } else { - if (Entry & PA_READWRITE) + if (Pte & PA_READWRITE) { Protect = PAGE_READWRITE; } @@ -831,20 +728,23 @@ MmGetPageProtect(PEPROCESS Process, PVOID Address) { Protect = PAGE_EXECUTE_READ; } - if (Entry & PA_CD) + if (Pte & PA_CD) { Protect |= PAGE_NOCACHE; } - if (Entry & PA_WT) + if (Pte & PA_WT) { Protect |= PAGE_WRITETHROUGH; } - if (!(Entry & PA_USER)) + if (!(Pte & PA_USER)) { Protect |= PAGE_SYSTEM; } - } + + if (Address < MmSystemRangeStart) + MiUnlockProcessWorkingSetShared(Process, PsGetCurrentThread()); + return(Protect); } @@ -853,43 +753,80 @@ NTAPI MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect) { ULONG Attributes = 0; - PULONG Pt; + PMMPTE PointerPte; ULONG Pte; - KIRQL OldIrql; DPRINT("MmSetPageProtect(Process %p Address %p flProtect %x)\n", Process, Address, flProtect); + ASSERT(Process != NULL); + ASSERT(Address < MmSystemRangeStart); + + ASSERT(Process == PsGetCurrentProcess()); + + MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + Attributes = ProtectToPTE(flProtect); Attributes &= 0xfff; - if (Address >= MmSystemRangeStart) - { - Attributes &= ~PA_USER; - } - else - { - Attributes |= PA_USER; - } + Attributes |= PA_USER; + + PointerPte = MiAddressToPte(Address); + Pte = InterlockedExchangePte(PointerPte, PAGE_MASK(PointerPte->u.Long) | Attributes | (PointerPte->u.Long & (PA_ACCESSED|PA_DIRTY))); - Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); - if (Pt == NULL) + // We should be able to bring a page back from PAGE_NOACCESS + if ((Pte & 0x800) || !(Pte >> PAGE_SHIFT)) { + DPRINT1("Invalid Pte %lx\n", Pte); KeBugCheck(MEMORY_MANAGEMENT); } - Pte = InterlockedExchangePte(Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY))); - // We should be able to bring a page back from PAGE_NOACCESS + if ((Pte & 0xFFF) != Attributes) + KeInvalidateTlbEntry(Address); + + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); +} + +VOID +NTAPI +MmSetDirtyBit(PEPROCESS Process, PVOID Address, BOOLEAN Bit) +{ + PMMPTE PointerPte; + ULONG Pte; + + DPRINT("MmSetDirtyBit(Process %p Address %p Bit %x)\n", + Process, Address, Bit); + + ASSERT(Process != NULL); + ASSERT(Address < MmSystemRangeStart); + + ASSERT(Process == PsGetCurrentProcess()); + + MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + + MiMakePdeExistAndMakeValid(MiAddressToPde(Address), Process, MM_NOIRQL); + + PointerPte = MiAddressToPte(Address); + Pte = PointerPte->u.Long; + if (Bit) + Pte |= PA_DIRTY; + else + Pte &= ~PA_DIRTY; + Pte = InterlockedExchangePte(PointerPte, Pte); + + // We shouldnl't set dirty bit on non-mapped adresses if ((Pte & 0x800) || !(Pte >> PAGE_SHIFT)) { DPRINT1("Invalid Pte %lx\n", Pte); KeBugCheck(MEMORY_MANAGEMENT); } - if((Pte & Attributes) != Attributes) - MiFlushTlb(Pt, Address, OldIrql); - else - MmUnmapPageTable(Pt, OldIrql); + if (!Bit) + KeInvalidateTlbEntry(Address); + + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); } CODE_SEG("INIT") diff --git a/ntoskrnl/mm/marea.c b/ntoskrnl/mm/marea.c index 92eebc173cb..1942dd76b27 100644 --- a/ntoskrnl/mm/marea.c +++ b/ntoskrnl/mm/marea.c @@ -328,24 +328,6 @@ MmFreeMemoryArea( FreePage(FreePageContext, MemoryArea, (PVOID)Address, Page, SwapEntry, (BOOLEAN)Dirty); } -#if (_MI_PAGING_LEVELS == 2) - /* Remove page table reference */ - ASSERT(KeGetCurrentIrql() <= APC_LEVEL); - if ((SwapEntry || Page) && ((PVOID)Address < MmSystemRangeStart)) - { - ASSERT(AddressSpace != MmGetKernelAddressSpace()); - if (MiQueryPageTableReferences((PVOID)Address) == 0) - { - /* No PTE relies on this PDE. Release it */ - KIRQL OldIrql = MiAcquirePfnLock(); - PMMPDE PointerPde = MiAddressToPde(Address); - ASSERT(PointerPde->u.Hard.Valid == 1); - MiDeletePte(PointerPde, MiPdeToPte(PointerPde), Process, NULL); - ASSERT(PointerPde->u.Hard.Valid == 0); - MiReleasePfnLock(OldIrql); - } - } -#endif } if (Process != NULL && diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index c680a8f7a9f..64758cc5b89 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -102,18 +102,26 @@ GetEntry: MmLockAddressSpace(AddressSpace); - if (MmGetPfnForProcess(Process, Address) != Page) + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); + if (MemoryArea == NULL || MemoryArea->DeleteInProgress) { - /* This changed in the short window where we didn't have any locks */ MmUnlockAddressSpace(AddressSpace); ExReleaseRundownProtection(&Process->RundownProtect); ObDereferenceObject(Process); goto GetEntry; } - MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); - if (MemoryArea == NULL || MemoryArea->DeleteInProgress) + + /* Attach to it, if needed */ + ASSERT(PsGetCurrentProcess() == PsInitialSystemProcess); + if (Process != PsInitialSystemProcess) + KeAttachProcess(&Process->Pcb); + + if (MmGetPfnForProcess(Process, Address) != Page) { + /* This changed in the short window where we didn't have any locks */ + if (Process != PsInitialSystemProcess) + KeDetachProcess(); MmUnlockAddressSpace(AddressSpace); ExReleaseRundownProtection(&Process->RundownProtect); ObDereferenceObject(Process); @@ -140,6 +148,8 @@ GetEntry: { /* The segment is being read or something. Give up */ MmUnlockSectionSegment(Segment); + if (Process != PsInitialSystemProcess) + KeDetachProcess(); MmUnlockAddressSpace(AddressSpace); ExReleaseRundownProtection(&Process->RundownProtect); ObDereferenceObject(Process); @@ -160,11 +170,6 @@ GetEntry: /* This page is private to the process */ MmUnlockSectionSegment(Segment); - /* Attach to it, if needed */ - ASSERT(PsGetCurrentProcess() == PsInitialSystemProcess); - if (Process != PsInitialSystemProcess) - KeAttachProcess(&Process->Pcb); - /* Check if we should write it back to the page file */ SwapEntry = MmGetSavedSwapEntryPage(Page); @@ -179,7 +184,7 @@ GetEntry: Address, NULL); /* We can't, so let this page in the Process VM */ - MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmCreateVirtualMapping(Process, Address, Region->Protect, Page); MmInsertRmap(Page, Process, Address); MmSetDirtyPage(Process, Address); @@ -219,7 +224,7 @@ GetEntry: MmFreeSwapPage(SwapEntry); /* We can't, so let this page in the Process VM */ - MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmCreateVirtualMapping(Process, Address, Region->Protect, Page); MmInsertRmap(Page, Process, Address); MmSetDirtyPage(Process, Address); @@ -241,7 +246,6 @@ GetEntry: } /* We can finally let this page go */ - MmUnlockAddressSpace(AddressSpace); if (Process != PsInitialSystemProcess) KeDetachProcess(); @@ -262,6 +266,8 @@ GetEntry: Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL); MmUnlockSectionSegment(Segment); + if (Process != PsInitialSystemProcess) + KeDetachProcess(); MmUnlockAddressSpace(AddressSpace); ExReleaseRundownProtection(&Process->RundownProtect); diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 2485d5bb5f5..0b6b8e80416 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1636,8 +1636,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, Status = MmCreateVirtualMapping(Process, PAddress, Region->Protect, - &Page, - 1); + Page); if (!NT_SUCCESS(Status)) { DPRINT("MmCreateVirtualMapping failed, not out of memory\n"); @@ -1679,8 +1678,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, Status = MmCreateVirtualMappingUnsafe(Process, PAddress, Region->Protect, - &Page, - 1); + Page); if (!NT_SUCCESS(Status)) { DPRINT("MmCreateVirtualMappingUnsafe failed, not out of memory\n"); @@ -1729,7 +1727,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SSE(Page << PAGE_SHIFT, 1)); MmUnlockSectionSegment(Segment); - Status = MmCreateVirtualMapping(Process, PAddress, Attributes, &Page, 1); + Status = MmCreateVirtualMapping(Process, PAddress, Attributes, Page); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); @@ -1828,8 +1826,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, Status = MmCreateVirtualMapping(Process, PAddress, Attributes, - &Page, - 1); + Page); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); @@ -1857,8 +1854,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, Status = MmCreateVirtualMapping(Process, PAddress, Attributes, - &Page, - 1); + Page); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); @@ -1985,8 +1981,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, Status = MmCreateVirtualMapping(Process, PAddress, Region->Protect, - &NewPage, - 1); + NewPage); if (!NT_SUCCESS(Status)) { DPRINT1("MmCreateVirtualMapping failed, unable to create virtual mapping, not out of memory\n");
