diff options
author | gVisor bot <gvisor-bot@google.com> | 2021-01-13 04:47:27 +0000 |
---|---|---|
committer | gVisor bot <gvisor-bot@google.com> | 2021-01-13 04:47:27 +0000 |
commit | 43ca8a82cb55b2e82f479869a6eee019e91a3eff (patch) | |
tree | 4cb0aa42960543b8b41917617a9d821a7b1d2463 /pkg/sentry/platform/ring0/pagetables | |
parent | aec44e37bc7cec5e23479822a69726a4091613cd (diff) | |
parent | e74aa25e2289878fdbfc7affdd2e031be3a99b31 (diff) |
Merge release-20201216.0-94-ge74aa25e2 (automated)
Diffstat (limited to 'pkg/sentry/platform/ring0/pagetables')
-rw-r--r-- | pkg/sentry/platform/ring0/pagetables/walker_empty.go | 255 | ||||
-rw-r--r-- | pkg/sentry/platform/ring0/pagetables/walker_lookup.go | 255 | ||||
-rw-r--r-- | pkg/sentry/platform/ring0/pagetables/walker_map.go | 255 | ||||
-rw-r--r-- | pkg/sentry/platform/ring0/pagetables/walker_unmap.go | 255 |
4 files changed, 0 insertions, 1020 deletions
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_empty.go b/pkg/sentry/platform/ring0/pagetables/walker_empty.go deleted file mode 100644 index 417784e17..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_empty.go +++ /dev/null @@ -1,255 +0,0 @@ -package pagetables - -// Walker walks page tables. -type emptyWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor emptyVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// -// Precondition: start must be less than end. -// -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *emptyWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func emptynext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *emptyWalker) iterateRangeCanonical(start, end uintptr) { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = emptynext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = emptynext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - if pudEntry.Valid() { - start = emptynext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < emptynext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = emptynext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = emptynext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - if pmdEntry.Valid() { - start = emptynext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < emptynext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = emptynext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - w.visitor.visit(uintptr(start), pteEntry, pteSize-1) - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_lookup.go b/pkg/sentry/platform/ring0/pagetables/walker_lookup.go deleted file mode 100644 index 906c9c50f..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_lookup.go +++ /dev/null @@ -1,255 +0,0 @@ -package pagetables - -// Walker walks page tables. -type lookupWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor lookupVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// -// Precondition: start must be less than end. -// -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *lookupWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func lookupnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *lookupWalker) iterateRangeCanonical(start, end uintptr) { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = lookupnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = lookupnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - if pudEntry.Valid() { - start = lookupnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < lookupnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = lookupnext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = lookupnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - if pmdEntry.Valid() { - start = lookupnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < lookupnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = lookupnext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - w.visitor.visit(uintptr(start), pteEntry, pteSize-1) - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_map.go b/pkg/sentry/platform/ring0/pagetables/walker_map.go deleted file mode 100644 index 61ee3c825..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_map.go +++ /dev/null @@ -1,255 +0,0 @@ -package pagetables - -// Walker walks page tables. -type mapWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor mapVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// -// Precondition: start must be less than end. -// -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *mapWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func mapnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *mapWalker) iterateRangeCanonical(start, end uintptr) { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = mapnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = mapnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - if pudEntry.Valid() { - start = mapnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < mapnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = mapnext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = mapnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - if pmdEntry.Valid() { - start = mapnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < mapnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = mapnext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - w.visitor.visit(uintptr(start), pteEntry, pteSize-1) - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_unmap.go b/pkg/sentry/platform/ring0/pagetables/walker_unmap.go deleted file mode 100644 index be2aa0ce4..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_unmap.go +++ /dev/null @@ -1,255 +0,0 @@ -package pagetables - -// Walker walks page tables. -type unmapWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor unmapVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// -// Precondition: start must be less than end. -// -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *unmapWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func unmapnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *unmapWalker) iterateRangeCanonical(start, end uintptr) { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = unmapnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = unmapnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - if pudEntry.Valid() { - start = unmapnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < unmapnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - w.visitor.visit(uintptr(start), pudEntry, pudSize-1) - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = unmapnext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = unmapnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - if pmdEntry.Valid() { - start = unmapnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < unmapnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = unmapnext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - w.visitor.visit(uintptr(start), pteEntry, pteSize-1) - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } -} |