summaryrefslogtreecommitdiffhomepage
path: root/pkg/sentry/platform/ring0/pagetables
diff options
context:
space:
mode:
authorgVisor bot <gvisor-bot@google.com>2021-01-13 04:47:27 +0000
committergVisor bot <gvisor-bot@google.com>2021-01-13 04:47:27 +0000
commit43ca8a82cb55b2e82f479869a6eee019e91a3eff (patch)
tree4cb0aa42960543b8b41917617a9d821a7b1d2463 /pkg/sentry/platform/ring0/pagetables
parentaec44e37bc7cec5e23479822a69726a4091613cd (diff)
parente74aa25e2289878fdbfc7affdd2e031be3a99b31 (diff)
Merge release-20201216.0-94-ge74aa25e2 (automated)
Diffstat (limited to 'pkg/sentry/platform/ring0/pagetables')
-rw-r--r--pkg/sentry/platform/ring0/pagetables/walker_empty.go255
-rw-r--r--pkg/sentry/platform/ring0/pagetables/walker_lookup.go255
-rw-r--r--pkg/sentry/platform/ring0/pagetables/walker_map.go255
-rw-r--r--pkg/sentry/platform/ring0/pagetables/walker_unmap.go255
4 files changed, 0 insertions, 1020 deletions
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_empty.go b/pkg/sentry/platform/ring0/pagetables/walker_empty.go
deleted file mode 100644
index 417784e17..000000000
--- a/pkg/sentry/platform/ring0/pagetables/walker_empty.go
+++ /dev/null
@@ -1,255 +0,0 @@
-package pagetables
-
-// Walker walks page tables.
-type emptyWalker struct {
- // pageTables are the tables to walk.
- pageTables *PageTables
-
- // Visitor is the set of arguments.
- visitor emptyVisitor
-}
-
-// iterateRange iterates over all appropriate levels of page tables for the given range.
-//
-// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The
-// exception is super pages. If a valid super page (huge or jumbo) cannot be
-// installed, then the walk will continue to individual entries.
-//
-// This algorithm will attempt to maximize the use of super pages whenever
-// possible. Whether a super page is provided will be clear through the range
-// provided in the callback.
-//
-// Note that if requiresAlloc is true, then no gaps will be present. However,
-// if alloc is not set, then the iteration will likely be full of gaps.
-//
-// Note that this function should generally be avoided in favor of Map, Unmap,
-// etc. when not necessary.
-//
-// Precondition: start must be page-aligned.
-//
-// Precondition: start must be less than end.
-//
-// Precondition: If requiresAlloc is true, then start and end should not span
-// non-canonical ranges. If they do, a panic will result.
-//
-//go:nosplit
-func (w *emptyWalker) iterateRange(start, end uintptr) {
- if start%pteSize != 0 {
- panic("unaligned start")
- }
- if end < start {
- panic("start > end")
- }
- if start < lowerTop {
- if end <= lowerTop {
- w.iterateRangeCanonical(start, end)
- } else if end > lowerTop && end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else if start < upperBottom {
- if end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else {
- w.iterateRangeCanonical(start, end)
- }
-}
-
-// next returns the next address quantized by the given size.
-//
-//go:nosplit
-func emptynext(start uintptr, size uintptr) uintptr {
- start &= ^(size - 1)
- start += size
- return start
-}
-
-// iterateRangeCanonical walks a canonical range.
-//
-//go:nosplit
-func (w *emptyWalker) iterateRangeCanonical(start, end uintptr) {
- for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
- var (
- pgdEntry = &w.pageTables.root[pgdIndex]
- pudEntries *PTEs
- )
- if !pgdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- start = emptynext(start, pgdSize)
- continue
- }
-
- pudEntries = w.pageTables.Allocator.NewPTEs()
- pgdEntry.setPageTable(w.pageTables, pudEntries)
- } else {
- pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
- }
-
- clearPUDEntries := uint16(0)
-
- for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
- var (
- pudEntry = &pudEntries[pudIndex]
- pmdEntries *PTEs
- )
- if !pudEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPUDEntries++
- start = emptynext(start, pudSize)
- continue
- }
-
- if start&(pudSize-1) == 0 && end-start >= pudSize {
- pudEntry.SetSuper()
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
- if pudEntry.Valid() {
- start = emptynext(start, pudSize)
- continue
- }
- }
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- pudEntry.setPageTable(w.pageTables, pmdEntries)
-
- } else if pudEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < emptynext(start, pudSize)) {
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pmdEntries[index].SetSuper()
- pmdEntries[index].Set(
- pudEntry.Address()+(pmdSize*uintptr(index)),
- pudEntry.Opts())
- }
- pudEntry.setPageTable(w.pageTables, pmdEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
-
- if !pudEntry.Valid() {
- clearPUDEntries++
- }
-
- start = emptynext(start, pudSize)
- continue
- }
- } else {
- pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
- }
-
- clearPMDEntries := uint16(0)
-
- for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
- var (
- pmdEntry = &pmdEntries[pmdIndex]
- pteEntries *PTEs
- )
- if !pmdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPMDEntries++
- start = emptynext(start, pmdSize)
- continue
- }
-
- if start&(pmdSize-1) == 0 && end-start >= pmdSize {
- pmdEntry.SetSuper()
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
- if pmdEntry.Valid() {
- start = emptynext(start, pmdSize)
- continue
- }
- }
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- pmdEntry.setPageTable(w.pageTables, pteEntries)
-
- } else if pmdEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < emptynext(start, pmdSize)) {
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pteEntries[index].Set(
- pmdEntry.Address()+(pteSize*uintptr(index)),
- pmdEntry.Opts())
- }
- pmdEntry.setPageTable(w.pageTables, pteEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
-
- if !pmdEntry.Valid() {
- clearPMDEntries++
- }
-
- start = emptynext(start, pmdSize)
- continue
- }
- } else {
- pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
- }
-
- clearPTEEntries := uint16(0)
-
- for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
- var (
- pteEntry = &pteEntries[pteIndex]
- )
- if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
- clearPTEEntries++
- start += pteSize
- continue
- }
-
- w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
- if !pteEntry.Valid() {
- if w.visitor.requiresAlloc() {
- panic("PTE not set after iteration with requiresAlloc!")
- }
- clearPTEEntries++
- }
-
- start += pteSize
- continue
- }
-
- if clearPTEEntries == entriesPerPage {
- pmdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pteEntries)
- clearPMDEntries++
- }
- }
-
- if clearPMDEntries == entriesPerPage {
- pudEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pmdEntries)
- clearPUDEntries++
- }
- }
-
- if clearPUDEntries == entriesPerPage {
- pgdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pudEntries)
- }
- }
-}
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_lookup.go b/pkg/sentry/platform/ring0/pagetables/walker_lookup.go
deleted file mode 100644
index 906c9c50f..000000000
--- a/pkg/sentry/platform/ring0/pagetables/walker_lookup.go
+++ /dev/null
@@ -1,255 +0,0 @@
-package pagetables
-
-// Walker walks page tables.
-type lookupWalker struct {
- // pageTables are the tables to walk.
- pageTables *PageTables
-
- // Visitor is the set of arguments.
- visitor lookupVisitor
-}
-
-// iterateRange iterates over all appropriate levels of page tables for the given range.
-//
-// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The
-// exception is super pages. If a valid super page (huge or jumbo) cannot be
-// installed, then the walk will continue to individual entries.
-//
-// This algorithm will attempt to maximize the use of super pages whenever
-// possible. Whether a super page is provided will be clear through the range
-// provided in the callback.
-//
-// Note that if requiresAlloc is true, then no gaps will be present. However,
-// if alloc is not set, then the iteration will likely be full of gaps.
-//
-// Note that this function should generally be avoided in favor of Map, Unmap,
-// etc. when not necessary.
-//
-// Precondition: start must be page-aligned.
-//
-// Precondition: start must be less than end.
-//
-// Precondition: If requiresAlloc is true, then start and end should not span
-// non-canonical ranges. If they do, a panic will result.
-//
-//go:nosplit
-func (w *lookupWalker) iterateRange(start, end uintptr) {
- if start%pteSize != 0 {
- panic("unaligned start")
- }
- if end < start {
- panic("start > end")
- }
- if start < lowerTop {
- if end <= lowerTop {
- w.iterateRangeCanonical(start, end)
- } else if end > lowerTop && end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else if start < upperBottom {
- if end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else {
- w.iterateRangeCanonical(start, end)
- }
-}
-
-// next returns the next address quantized by the given size.
-//
-//go:nosplit
-func lookupnext(start uintptr, size uintptr) uintptr {
- start &= ^(size - 1)
- start += size
- return start
-}
-
-// iterateRangeCanonical walks a canonical range.
-//
-//go:nosplit
-func (w *lookupWalker) iterateRangeCanonical(start, end uintptr) {
- for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
- var (
- pgdEntry = &w.pageTables.root[pgdIndex]
- pudEntries *PTEs
- )
- if !pgdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- start = lookupnext(start, pgdSize)
- continue
- }
-
- pudEntries = w.pageTables.Allocator.NewPTEs()
- pgdEntry.setPageTable(w.pageTables, pudEntries)
- } else {
- pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
- }
-
- clearPUDEntries := uint16(0)
-
- for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
- var (
- pudEntry = &pudEntries[pudIndex]
- pmdEntries *PTEs
- )
- if !pudEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPUDEntries++
- start = lookupnext(start, pudSize)
- continue
- }
-
- if start&(pudSize-1) == 0 && end-start >= pudSize {
- pudEntry.SetSuper()
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
- if pudEntry.Valid() {
- start = lookupnext(start, pudSize)
- continue
- }
- }
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- pudEntry.setPageTable(w.pageTables, pmdEntries)
-
- } else if pudEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < lookupnext(start, pudSize)) {
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pmdEntries[index].SetSuper()
- pmdEntries[index].Set(
- pudEntry.Address()+(pmdSize*uintptr(index)),
- pudEntry.Opts())
- }
- pudEntry.setPageTable(w.pageTables, pmdEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
-
- if !pudEntry.Valid() {
- clearPUDEntries++
- }
-
- start = lookupnext(start, pudSize)
- continue
- }
- } else {
- pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
- }
-
- clearPMDEntries := uint16(0)
-
- for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
- var (
- pmdEntry = &pmdEntries[pmdIndex]
- pteEntries *PTEs
- )
- if !pmdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPMDEntries++
- start = lookupnext(start, pmdSize)
- continue
- }
-
- if start&(pmdSize-1) == 0 && end-start >= pmdSize {
- pmdEntry.SetSuper()
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
- if pmdEntry.Valid() {
- start = lookupnext(start, pmdSize)
- continue
- }
- }
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- pmdEntry.setPageTable(w.pageTables, pteEntries)
-
- } else if pmdEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < lookupnext(start, pmdSize)) {
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pteEntries[index].Set(
- pmdEntry.Address()+(pteSize*uintptr(index)),
- pmdEntry.Opts())
- }
- pmdEntry.setPageTable(w.pageTables, pteEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
-
- if !pmdEntry.Valid() {
- clearPMDEntries++
- }
-
- start = lookupnext(start, pmdSize)
- continue
- }
- } else {
- pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
- }
-
- clearPTEEntries := uint16(0)
-
- for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
- var (
- pteEntry = &pteEntries[pteIndex]
- )
- if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
- clearPTEEntries++
- start += pteSize
- continue
- }
-
- w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
- if !pteEntry.Valid() {
- if w.visitor.requiresAlloc() {
- panic("PTE not set after iteration with requiresAlloc!")
- }
- clearPTEEntries++
- }
-
- start += pteSize
- continue
- }
-
- if clearPTEEntries == entriesPerPage {
- pmdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pteEntries)
- clearPMDEntries++
- }
- }
-
- if clearPMDEntries == entriesPerPage {
- pudEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pmdEntries)
- clearPUDEntries++
- }
- }
-
- if clearPUDEntries == entriesPerPage {
- pgdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pudEntries)
- }
- }
-}
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_map.go b/pkg/sentry/platform/ring0/pagetables/walker_map.go
deleted file mode 100644
index 61ee3c825..000000000
--- a/pkg/sentry/platform/ring0/pagetables/walker_map.go
+++ /dev/null
@@ -1,255 +0,0 @@
-package pagetables
-
-// Walker walks page tables.
-type mapWalker struct {
- // pageTables are the tables to walk.
- pageTables *PageTables
-
- // Visitor is the set of arguments.
- visitor mapVisitor
-}
-
-// iterateRange iterates over all appropriate levels of page tables for the given range.
-//
-// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The
-// exception is super pages. If a valid super page (huge or jumbo) cannot be
-// installed, then the walk will continue to individual entries.
-//
-// This algorithm will attempt to maximize the use of super pages whenever
-// possible. Whether a super page is provided will be clear through the range
-// provided in the callback.
-//
-// Note that if requiresAlloc is true, then no gaps will be present. However,
-// if alloc is not set, then the iteration will likely be full of gaps.
-//
-// Note that this function should generally be avoided in favor of Map, Unmap,
-// etc. when not necessary.
-//
-// Precondition: start must be page-aligned.
-//
-// Precondition: start must be less than end.
-//
-// Precondition: If requiresAlloc is true, then start and end should not span
-// non-canonical ranges. If they do, a panic will result.
-//
-//go:nosplit
-func (w *mapWalker) iterateRange(start, end uintptr) {
- if start%pteSize != 0 {
- panic("unaligned start")
- }
- if end < start {
- panic("start > end")
- }
- if start < lowerTop {
- if end <= lowerTop {
- w.iterateRangeCanonical(start, end)
- } else if end > lowerTop && end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else if start < upperBottom {
- if end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else {
- w.iterateRangeCanonical(start, end)
- }
-}
-
-// next returns the next address quantized by the given size.
-//
-//go:nosplit
-func mapnext(start uintptr, size uintptr) uintptr {
- start &= ^(size - 1)
- start += size
- return start
-}
-
-// iterateRangeCanonical walks a canonical range.
-//
-//go:nosplit
-func (w *mapWalker) iterateRangeCanonical(start, end uintptr) {
- for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
- var (
- pgdEntry = &w.pageTables.root[pgdIndex]
- pudEntries *PTEs
- )
- if !pgdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- start = mapnext(start, pgdSize)
- continue
- }
-
- pudEntries = w.pageTables.Allocator.NewPTEs()
- pgdEntry.setPageTable(w.pageTables, pudEntries)
- } else {
- pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
- }
-
- clearPUDEntries := uint16(0)
-
- for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
- var (
- pudEntry = &pudEntries[pudIndex]
- pmdEntries *PTEs
- )
- if !pudEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPUDEntries++
- start = mapnext(start, pudSize)
- continue
- }
-
- if start&(pudSize-1) == 0 && end-start >= pudSize {
- pudEntry.SetSuper()
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
- if pudEntry.Valid() {
- start = mapnext(start, pudSize)
- continue
- }
- }
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- pudEntry.setPageTable(w.pageTables, pmdEntries)
-
- } else if pudEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < mapnext(start, pudSize)) {
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pmdEntries[index].SetSuper()
- pmdEntries[index].Set(
- pudEntry.Address()+(pmdSize*uintptr(index)),
- pudEntry.Opts())
- }
- pudEntry.setPageTable(w.pageTables, pmdEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
-
- if !pudEntry.Valid() {
- clearPUDEntries++
- }
-
- start = mapnext(start, pudSize)
- continue
- }
- } else {
- pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
- }
-
- clearPMDEntries := uint16(0)
-
- for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
- var (
- pmdEntry = &pmdEntries[pmdIndex]
- pteEntries *PTEs
- )
- if !pmdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPMDEntries++
- start = mapnext(start, pmdSize)
- continue
- }
-
- if start&(pmdSize-1) == 0 && end-start >= pmdSize {
- pmdEntry.SetSuper()
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
- if pmdEntry.Valid() {
- start = mapnext(start, pmdSize)
- continue
- }
- }
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- pmdEntry.setPageTable(w.pageTables, pteEntries)
-
- } else if pmdEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < mapnext(start, pmdSize)) {
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pteEntries[index].Set(
- pmdEntry.Address()+(pteSize*uintptr(index)),
- pmdEntry.Opts())
- }
- pmdEntry.setPageTable(w.pageTables, pteEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
-
- if !pmdEntry.Valid() {
- clearPMDEntries++
- }
-
- start = mapnext(start, pmdSize)
- continue
- }
- } else {
- pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
- }
-
- clearPTEEntries := uint16(0)
-
- for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
- var (
- pteEntry = &pteEntries[pteIndex]
- )
- if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
- clearPTEEntries++
- start += pteSize
- continue
- }
-
- w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
- if !pteEntry.Valid() {
- if w.visitor.requiresAlloc() {
- panic("PTE not set after iteration with requiresAlloc!")
- }
- clearPTEEntries++
- }
-
- start += pteSize
- continue
- }
-
- if clearPTEEntries == entriesPerPage {
- pmdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pteEntries)
- clearPMDEntries++
- }
- }
-
- if clearPMDEntries == entriesPerPage {
- pudEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pmdEntries)
- clearPUDEntries++
- }
- }
-
- if clearPUDEntries == entriesPerPage {
- pgdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pudEntries)
- }
- }
-}
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_unmap.go b/pkg/sentry/platform/ring0/pagetables/walker_unmap.go
deleted file mode 100644
index be2aa0ce4..000000000
--- a/pkg/sentry/platform/ring0/pagetables/walker_unmap.go
+++ /dev/null
@@ -1,255 +0,0 @@
-package pagetables
-
-// Walker walks page tables.
-type unmapWalker struct {
- // pageTables are the tables to walk.
- pageTables *PageTables
-
- // Visitor is the set of arguments.
- visitor unmapVisitor
-}
-
-// iterateRange iterates over all appropriate levels of page tables for the given range.
-//
-// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The
-// exception is super pages. If a valid super page (huge or jumbo) cannot be
-// installed, then the walk will continue to individual entries.
-//
-// This algorithm will attempt to maximize the use of super pages whenever
-// possible. Whether a super page is provided will be clear through the range
-// provided in the callback.
-//
-// Note that if requiresAlloc is true, then no gaps will be present. However,
-// if alloc is not set, then the iteration will likely be full of gaps.
-//
-// Note that this function should generally be avoided in favor of Map, Unmap,
-// etc. when not necessary.
-//
-// Precondition: start must be page-aligned.
-//
-// Precondition: start must be less than end.
-//
-// Precondition: If requiresAlloc is true, then start and end should not span
-// non-canonical ranges. If they do, a panic will result.
-//
-//go:nosplit
-func (w *unmapWalker) iterateRange(start, end uintptr) {
- if start%pteSize != 0 {
- panic("unaligned start")
- }
- if end < start {
- panic("start > end")
- }
- if start < lowerTop {
- if end <= lowerTop {
- w.iterateRangeCanonical(start, end)
- } else if end > lowerTop && end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(start, lowerTop)
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else if start < upperBottom {
- if end <= upperBottom {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- } else {
- if w.visitor.requiresAlloc() {
- panic("alloc spans non-canonical range")
- }
- w.iterateRangeCanonical(upperBottom, end)
- }
- } else {
- w.iterateRangeCanonical(start, end)
- }
-}
-
-// next returns the next address quantized by the given size.
-//
-//go:nosplit
-func unmapnext(start uintptr, size uintptr) uintptr {
- start &= ^(size - 1)
- start += size
- return start
-}
-
-// iterateRangeCanonical walks a canonical range.
-//
-//go:nosplit
-func (w *unmapWalker) iterateRangeCanonical(start, end uintptr) {
- for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
- var (
- pgdEntry = &w.pageTables.root[pgdIndex]
- pudEntries *PTEs
- )
- if !pgdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- start = unmapnext(start, pgdSize)
- continue
- }
-
- pudEntries = w.pageTables.Allocator.NewPTEs()
- pgdEntry.setPageTable(w.pageTables, pudEntries)
- } else {
- pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
- }
-
- clearPUDEntries := uint16(0)
-
- for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
- var (
- pudEntry = &pudEntries[pudIndex]
- pmdEntries *PTEs
- )
- if !pudEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPUDEntries++
- start = unmapnext(start, pudSize)
- continue
- }
-
- if start&(pudSize-1) == 0 && end-start >= pudSize {
- pudEntry.SetSuper()
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
- if pudEntry.Valid() {
- start = unmapnext(start, pudSize)
- continue
- }
- }
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- pudEntry.setPageTable(w.pageTables, pmdEntries)
-
- } else if pudEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < unmapnext(start, pudSize)) {
-
- pmdEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pmdEntries[index].SetSuper()
- pmdEntries[index].Set(
- pudEntry.Address()+(pmdSize*uintptr(index)),
- pudEntry.Opts())
- }
- pudEntry.setPageTable(w.pageTables, pmdEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
-
- if !pudEntry.Valid() {
- clearPUDEntries++
- }
-
- start = unmapnext(start, pudSize)
- continue
- }
- } else {
- pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
- }
-
- clearPMDEntries := uint16(0)
-
- for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
- var (
- pmdEntry = &pmdEntries[pmdIndex]
- pteEntries *PTEs
- )
- if !pmdEntry.Valid() {
- if !w.visitor.requiresAlloc() {
-
- clearPMDEntries++
- start = unmapnext(start, pmdSize)
- continue
- }
-
- if start&(pmdSize-1) == 0 && end-start >= pmdSize {
- pmdEntry.SetSuper()
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
- if pmdEntry.Valid() {
- start = unmapnext(start, pmdSize)
- continue
- }
- }
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- pmdEntry.setPageTable(w.pageTables, pteEntries)
-
- } else if pmdEntry.IsSuper() {
-
- if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < unmapnext(start, pmdSize)) {
-
- pteEntries = w.pageTables.Allocator.NewPTEs()
- for index := uint16(0); index < entriesPerPage; index++ {
- pteEntries[index].Set(
- pmdEntry.Address()+(pteSize*uintptr(index)),
- pmdEntry.Opts())
- }
- pmdEntry.setPageTable(w.pageTables, pteEntries)
- } else {
-
- w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
-
- if !pmdEntry.Valid() {
- clearPMDEntries++
- }
-
- start = unmapnext(start, pmdSize)
- continue
- }
- } else {
- pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
- }
-
- clearPTEEntries := uint16(0)
-
- for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
- var (
- pteEntry = &pteEntries[pteIndex]
- )
- if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
- clearPTEEntries++
- start += pteSize
- continue
- }
-
- w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
- if !pteEntry.Valid() {
- if w.visitor.requiresAlloc() {
- panic("PTE not set after iteration with requiresAlloc!")
- }
- clearPTEEntries++
- }
-
- start += pteSize
- continue
- }
-
- if clearPTEEntries == entriesPerPage {
- pmdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pteEntries)
- clearPMDEntries++
- }
- }
-
- if clearPMDEntries == entriesPerPage {
- pudEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pmdEntries)
- clearPUDEntries++
- }
- }
-
- if clearPUDEntries == entriesPerPage {
- pgdEntry.Clear()
- w.pageTables.Allocator.FreePTEs(pudEntries)
- }
- }
-}