Change ring0 & page tables arguments to structs.

This is a refactor of ring0 and ring0/pagetables that changes from
individual arguments to opts structures. This should involve no
functional changes, but sets the stage for subsequent changes.

PiperOrigin-RevId: 198627556
Change-Id: Id4460340f6a73f0c793cd879324398139cd58ae9

6 files changed, 186 insertions, 150 deletions

diff --git a/pkg/sentry/platform/ring0/pagetables/BUILD b/pkg/sentry/platform/ring0/pagetables/BUILD
index c0c481ab3..1a8b7931e 100644
--- a/pkg/sentry/platform/ring0/pagetables/BUILD
+++ b/pkg/sentry/platform/ring0/pagetables/BUILD
@@ -23,8 +23,8 @@ go_test(
     name = "pagetables_test",
     size = "small",
     srcs = [
+        "pagetables_amd64_test.go",
         "pagetables_test.go",
-        "pagetables_x86_test.go",
         "pcids_x86_test.go",
     ],
     embed = [":pagetables"],
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables.go b/pkg/sentry/platform/ring0/pagetables/pagetables.go
index ee7f27601..2df6792f7 100644
--- a/pkg/sentry/platform/ring0/pagetables/pagetables.go
+++ b/pkg/sentry/platform/ring0/pagetables/pagetables.go
@@ -117,8 +117,8 @@ func (p *PageTables) getPageTable(n *Node, index int) *Node {
 // True is returned iff there was a previous mapping in the range.
 //
 // Precondition: addr & length must be aligned, their sum must not overflow.
-func (p *PageTables) Map(addr usermem.Addr, length uintptr, user bool, at usermem.AccessType, physical uintptr) bool {
-	if at == usermem.NoAccess {
+func (p *PageTables) Map(addr usermem.Addr, length uintptr, opts MapOpts, physical uintptr) bool {
+	if !opts.AccessType.Any() {
 		return p.Unmap(addr, length)
 	}
 	prev := false
@@ -129,7 +129,7 @@ func (p *PageTables) Map(addr usermem.Addr, length uintptr, user bool, at userme
 	}
 	p.iterateRange(uintptr(addr), uintptr(end), true, func(s, e uintptr, pte *PTE, align uintptr) {
 		p := physical + (s - uintptr(addr))
-		prev = prev || (pte.Valid() && (p != pte.Address() || at.Write != pte.Writeable() || at.Execute != pte.Executable()))
+		prev = prev || (pte.Valid() && (p != pte.Address() || opts != pte.Opts()))
 		if p&align != 0 {
 			// We will install entries at a smaller granulaity if
 			// we don't install a valid entry here, however we must
@@ -137,7 +137,7 @@ func (p *PageTables) Map(addr usermem.Addr, length uintptr, user bool, at userme
 			pte.Clear()
 			return
 		}
-		pte.Set(p, at.Write, at.Execute, user)
+		pte.Set(p, opts)
 	})
 	p.mu.Unlock()
 	return prev
@@ -167,7 +167,7 @@ func (p *PageTables) Release() {
 }
 
 // Lookup returns the physical address for the given virtual address.
-func (p *PageTables) Lookup(addr usermem.Addr) (physical uintptr, accessType usermem.AccessType) {
+func (p *PageTables) Lookup(addr usermem.Addr) (physical uintptr, opts MapOpts) {
 	mask := uintptr(usermem.PageSize - 1)
 	off := uintptr(addr) & mask
 	addr = addr &^ usermem.Addr(mask)
@@ -176,13 +176,9 @@ func (p *PageTables) Lookup(addr usermem.Addr) (physical uintptr, accessType use
 			return
 		}
 		physical = pte.Address() + (s - uintptr(addr)) + off
-		accessType = usermem.AccessType{
-			Read:    true,
-			Write:   pte.Writeable(),
-			Execute: pte.Executable(),
-		}
+		opts = pte.Opts()
 	})
-	return physical, accessType
+	return
 }
 
 // allocNode allocates a new page.
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64.go b/pkg/sentry/platform/ring0/pagetables/pagetables_amd64.go
index a2050b99c..8dc50f9dd 100644
--- a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64.go
+++ b/pkg/sentry/platform/ring0/pagetables/pagetables_amd64.go
@@ -18,7 +18,6 @@ package pagetables
 
 import (
 	"fmt"
-	"sync/atomic"
 )
 
 // Address constraints.
@@ -43,98 +42,11 @@ const (
 	pmdSize = 1 << pmdShift
 	pudSize = 1 << pudShift
 	pgdSize = 1 << pgdShift
-)
 
-// Bits in page table entries.
-const (
-	present        = 0x001
-	writable       = 0x002
-	user           = 0x004
-	writeThrough   = 0x008
-	cacheDisable   = 0x010
-	accessed       = 0x020
-	dirty          = 0x040
-	super          = 0x080
 	executeDisable = 1 << 63
+	entriesPerPage = 512
 )
 
-// PTE is a page table entry.
-type PTE uint64
-
-// Clear clears this PTE, including super page information.
-func (p *PTE) Clear() {
-	atomic.StoreUint64((*uint64)(p), 0)
-}
-
-// Valid returns true iff this entry is valid.
-func (p *PTE) Valid() bool {
-	return atomic.LoadUint64((*uint64)(p))&present != 0
-}
-
-// Writeable returns true iff the page is writable.
-func (p *PTE) Writeable() bool {
-	return atomic.LoadUint64((*uint64)(p))&writable != 0
-}
-
-// User returns true iff the page is user-accessible.
-func (p *PTE) User() bool {
-	return atomic.LoadUint64((*uint64)(p))&user != 0
-}
-
-// Executable returns true iff the page is executable.
-func (p *PTE) Executable() bool {
-	return atomic.LoadUint64((*uint64)(p))&executeDisable == 0
-}
-
-// SetSuper sets this page as a super page.
-//
-// The page must not be valid or a panic will result.
-func (p *PTE) SetSuper() {
-	if p.Valid() {
-		// This is not allowed.
-		panic("SetSuper called on valid page!")
-	}
-	atomic.StoreUint64((*uint64)(p), super)
-}
-
-// IsSuper returns true iff this page is a super page.
-func (p *PTE) IsSuper() bool {
-	return atomic.LoadUint64((*uint64)(p))&super != 0
-}
-
-// Set sets this PTE value.
-func (p *PTE) Set(addr uintptr, write, execute bool, userAccessible bool) {
-	v := uint64(addr)&^uint64(0xfff) | present | accessed
-	if userAccessible {
-		v |= user
-	}
-	if !execute {
-		v |= executeDisable
-	}
-	if write {
-		v |= writable | dirty
-	}
-	if p.IsSuper() {
-		v |= super
-	}
-	atomic.StoreUint64((*uint64)(p), v)
-}
-
-// setPageTable sets this PTE value and forces the write bit and super bit to
-// be cleared. This is used explicitly for breaking super pages.
-func (p *PTE) setPageTable(addr uintptr) {
-	v := uint64(addr)&^uint64(0xfff) | present | user | writable | accessed | dirty
-	atomic.StoreUint64((*uint64)(p), v)
-}
-
-// Address extracts the address. This should only be used if Valid returns true.
-func (p *PTE) Address() uintptr {
-	return uintptr(atomic.LoadUint64((*uint64)(p)) & ^uint64(executeDisable|0xfff))
-}
-
-// entriesPerPage is the number of PTEs per page.
-const entriesPerPage = 512
-
 // PTEs is a collection of entries.
 type PTEs [entriesPerPage]PTE
 
@@ -255,9 +167,6 @@ func (p *PageTables) iterateRange(startAddr, endAddr uintptr, alloc bool, fn fun
 				// Does this page need to be split?
 				if start&(pudSize-1) != 0 || end < next(start, pudSize) {
 					currentAddr := uint64(pudEntry.Address())
-					writeable := pudEntry.Writeable()
-					executable := pudEntry.Executable()
-					user := pudEntry.User()
 
 					// Install the relevant entries.
 					pmdNode := p.allocNode()
@@ -265,7 +174,7 @@ func (p *PageTables) iterateRange(startAddr, endAddr uintptr, alloc bool, fn fun
 					for index := 0; index < entriesPerPage; index++ {
 						pmdEntry := &pmdEntries[index]
 						pmdEntry.SetSuper()
-						pmdEntry.Set(uintptr(currentAddr), writeable, executable, user)
+						pmdEntry.Set(uintptr(currentAddr), pudEntry.Opts())
 						currentAddr += pmdSize
 					}
 
@@ -319,16 +228,13 @@ func (p *PageTables) iterateRange(startAddr, endAddr uintptr, alloc bool, fn fun
 					// Does this page need to be split?
 					if start&(pmdSize-1) != 0 || end < next(start, pmdSize) {
 						currentAddr := uint64(pmdEntry.Address())
-						writeable := pmdEntry.Writeable()
-						executable := pmdEntry.Executable()
-						user := pmdEntry.User()
 
 						// Install the relevant entries.
 						pteNode := p.allocNode()
 						pteEntries := pteNode.PTEs()
 						for index := 0; index < entriesPerPage; index++ {
 							pteEntry := &pteEntries[index]
-							pteEntry.Set(uintptr(currentAddr), writeable, executable, user)
+							pteEntry.Set(uintptr(currentAddr), pmdEntry.Opts())
 							currentAddr += pteSize
 						}
 
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_x86_test.go b/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_test.go
index 1fc403c48..4f15c6b58 100644
--- a/pkg/sentry/platform/ring0/pagetables/pagetables_x86_test.go
+++ b/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build i386 amd64
+// +build amd64
 
 package pagetables
 
@@ -26,12 +26,12 @@ func Test2MAnd4K(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map a small page and a huge page.
-	pt.Map(0x400000, pteSize, true, usermem.ReadWrite, pteSize*42)
-	pt.Map(0x00007f0000000000, 1<<21, true, usermem.Read, pmdSize*47)
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
+	pt.Map(0x00007f0000000000, pmdSize, MapOpts{AccessType: usermem.Read}, pmdSize*47)
 
 	checkMappings(t, pt, []mapping{
-		{0x400000, pteSize, pteSize * 42, true},
-		{0x00007f0000000000, pmdSize, pmdSize * 47, false},
+		{0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
+		{0x00007f0000000000, pmdSize, pmdSize * 47, MapOpts{AccessType: usermem.Read}},
 	})
 	pt.Release()
 }
@@ -40,12 +40,12 @@ func Test1GAnd4K(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map a small page and a super page.
-	pt.Map(0x400000, pteSize, true, usermem.ReadWrite, pteSize*42)
-	pt.Map(0x00007f0000000000, pudSize, true, usermem.Read, pudSize*47)
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
+	pt.Map(0x00007f0000000000, pudSize, MapOpts{AccessType: usermem.Read}, pudSize*47)
 
 	checkMappings(t, pt, []mapping{
-		{0x400000, pteSize, pteSize * 42, true},
-		{0x00007f0000000000, pudSize, pudSize * 47, false},
+		{0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
+		{0x00007f0000000000, pudSize, pudSize * 47, MapOpts{AccessType: usermem.Read}},
 	})
 	pt.Release()
 }
@@ -54,12 +54,12 @@ func TestSplit1GPage(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map a super page and knock out the middle.
-	pt.Map(0x00007f0000000000, pudSize, true, usermem.Read, pudSize*42)
+	pt.Map(0x00007f0000000000, pudSize, MapOpts{AccessType: usermem.Read}, pudSize*42)
 	pt.Unmap(usermem.Addr(0x00007f0000000000+pteSize), pudSize-(2*pteSize))
 
 	checkMappings(t, pt, []mapping{
-		{0x00007f0000000000, pteSize, pudSize * 42, false},
-		{0x00007f0000000000 + pudSize - pteSize, pteSize, pudSize*42 + pudSize - pteSize, false},
+		{0x00007f0000000000, pteSize, pudSize * 42, MapOpts{AccessType: usermem.Read}},
+		{0x00007f0000000000 + pudSize - pteSize, pteSize, pudSize*42 + pudSize - pteSize, MapOpts{AccessType: usermem.Read}},
 	})
 	pt.Release()
 }
@@ -68,12 +68,12 @@ func TestSplit2MPage(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map a huge page and knock out the middle.
-	pt.Map(0x00007f0000000000, pmdSize, true, usermem.Read, pmdSize*42)
+	pt.Map(0x00007f0000000000, pmdSize, MapOpts{AccessType: usermem.Read}, pmdSize*42)
 	pt.Unmap(usermem.Addr(0x00007f0000000000+pteSize), pmdSize-(2*pteSize))
 
 	checkMappings(t, pt, []mapping{
-		{0x00007f0000000000, pteSize, pmdSize * 42, false},
-		{0x00007f0000000000 + pmdSize - pteSize, pteSize, pmdSize*42 + pmdSize - pteSize, false},
+		{0x00007f0000000000, pteSize, pmdSize * 42, MapOpts{AccessType: usermem.Read}},
+		{0x00007f0000000000 + pmdSize - pteSize, pteSize, pmdSize*42 + pmdSize - pteSize, MapOpts{AccessType: usermem.Read}},
 	})
 	pt.Release()
 }
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_test.go b/pkg/sentry/platform/ring0/pagetables/pagetables_test.go
index 9cbc0e3b0..a4f684af2 100644
--- a/pkg/sentry/platform/ring0/pagetables/pagetables_test.go
+++ b/pkg/sentry/platform/ring0/pagetables/pagetables_test.go
@@ -28,10 +28,10 @@ func (r reflectTranslater) TranslateToPhysical(ptes *PTEs) uintptr {
 }
 
 type mapping struct {
-	start     uintptr
-	length    uintptr
-	addr      uintptr
-	writeable bool
+	start  uintptr
+	length uintptr
+	addr   uintptr
+	opts   MapOpts
 }
 
 func checkMappings(t *testing.T, pt *PageTables, m []mapping) {
@@ -44,10 +44,10 @@ func checkMappings(t *testing.T, pt *PageTables, m []mapping) {
 	// Iterate over all the mappings.
 	pt.iterateRange(0, ^uintptr(0), false, func(s, e uintptr, pte *PTE, align uintptr) {
 		found = append(found, mapping{
-			start:     s,
-			length:    e - s,
-			addr:      pte.Address(),
-			writeable: pte.Writeable(),
+			start:  s,
+			length: e - s,
+			addr:   pte.Address(),
+			opts:   pte.Opts(),
 		})
 		if failed != "" {
 			// Don't keep looking for errors.
@@ -62,8 +62,8 @@ func checkMappings(t *testing.T, pt *PageTables, m []mapping) {
 			failed = "end didn't match expected"
 		} else if m[current].addr != pte.Address() {
 			failed = "address didn't match expected"
-		} else if m[current].writeable != pte.Writeable() {
-			failed = "writeable didn't match"
+		} else if m[current].opts != pte.Opts() {
+			failed = "opts didn't match"
 		}
 		current++
 	})
@@ -88,7 +88,7 @@ func TestUnmap(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map and unmap one entry.
-	pt.Map(0x400000, pteSize, true, usermem.ReadWrite, pteSize*42)
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
 	pt.Unmap(0x400000, pteSize)
 
 	checkMappings(t, pt, nil)
@@ -99,10 +99,10 @@ func TestReadOnly(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map one entry.
-	pt.Map(0x400000, pteSize, true, usermem.Read, pteSize*42)
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.Read}, pteSize*42)
 
 	checkMappings(t, pt, []mapping{
-		{0x400000, pteSize, pteSize * 42, false},
+		{0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.Read}},
 	})
 	pt.Release()
 }
@@ -111,10 +111,10 @@ func TestReadWrite(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map one entry.
-	pt.Map(0x400000, pteSize, true, usermem.ReadWrite, pteSize*42)
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
 
 	checkMappings(t, pt, []mapping{
-		{0x400000, pteSize, pteSize * 42, true},
+		{0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
 	})
 	pt.Release()
 }
@@ -123,12 +123,12 @@ func TestSerialEntries(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map two sequential entries.
-	pt.Map(0x400000, pteSize, true, usermem.ReadWrite, pteSize*42)
-	pt.Map(0x401000, pteSize, true, usermem.ReadWrite, pteSize*47)
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
+	pt.Map(0x401000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*47)
 
 	checkMappings(t, pt, []mapping{
-		{0x400000, pteSize, pteSize * 42, true},
-		{0x401000, pteSize, pteSize * 47, true},
+		{0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
+		{0x401000, pteSize, pteSize * 47, MapOpts{AccessType: usermem.ReadWrite}},
 	})
 	pt.Release()
 }
@@ -137,11 +137,11 @@ func TestSpanningEntries(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Span a pgd with two pages.
-	pt.Map(0x00007efffffff000, 2*pteSize, true, usermem.Read, pteSize*42)
+	pt.Map(0x00007efffffff000, 2*pteSize, MapOpts{AccessType: usermem.Read}, pteSize*42)
 
 	checkMappings(t, pt, []mapping{
-		{0x00007efffffff000, pteSize, pteSize * 42, false},
-		{0x00007f0000000000, pteSize, pteSize * 43, false},
+		{0x00007efffffff000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.Read}},
+		{0x00007f0000000000, pteSize, pteSize * 43, MapOpts{AccessType: usermem.Read}},
 	})
 	pt.Release()
 }
@@ -150,12 +150,12 @@ func TestSparseEntries(t *testing.T) {
 	pt := New(reflectTranslater{}, Opts{})
 
 	// Map two entries in different pgds.
-	pt.Map(0x400000, pteSize, true, usermem.ReadWrite, pteSize*42)
-	pt.Map(0x00007f0000000000, pteSize, true, usermem.Read, pteSize*47)
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
+	pt.Map(0x00007f0000000000, pteSize, MapOpts{AccessType: usermem.Read}, pteSize*47)
 
 	checkMappings(t, pt, []mapping{
-		{0x400000, pteSize, pteSize * 42, true},
-		{0x00007f0000000000, pteSize, pteSize * 47, false},
+		{0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
+		{0x00007f0000000000, pteSize, pteSize * 47, MapOpts{AccessType: usermem.Read}},
 	})
 	pt.Release()
 }
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_x86.go b/pkg/sentry/platform/ring0/pagetables/pagetables_x86.go
index dac66373f..8ba78ed0d 100644
--- a/pkg/sentry/platform/ring0/pagetables/pagetables_x86.go
+++ b/pkg/sentry/platform/ring0/pagetables/pagetables_x86.go
@@ -16,6 +16,12 @@
 
 package pagetables
 
+import (
+	"sync/atomic"
+
+	"gvisor.googlesource.com/gvisor/pkg/sentry/usermem"
+)
+
 // Opts are pagetable options.
 type Opts struct {
 	EnablePCID bool
@@ -77,3 +83,131 @@ func (p *PageTables) CR3() uint64 {
 func (p *PageTables) FlushCR3() uint64 {
 	return uint64(p.root.physical) | uint64(p.pcid)
 }
+
+// Bits in page table entries.
+const (
+	present      = 0x001
+	writable     = 0x002
+	user         = 0x004
+	writeThrough = 0x008
+	cacheDisable = 0x010
+	accessed     = 0x020
+	dirty        = 0x040
+	super        = 0x080
+	global       = 0x100
+	optionMask   = executeDisable | 0xfff
+)
+
+// MapOpts are x86 options.
+type MapOpts struct {
+	// AccessType defines permissions.
+	AccessType usermem.AccessType
+
+	// Global indicates the page is globally accessible.
+	Global bool
+
+	// User indicates the page is a user page.
+	User bool
+}
+
+// PTE is a page table entry.
+type PTE uintptr
+
+// Clear clears this PTE, including super page information.
+//
+//go:nosplit
+func (p *PTE) Clear() {
+	atomic.StoreUintptr((*uintptr)(p), 0)
+}
+
+// Valid returns true iff this entry is valid.
+//
+//go:nosplit
+func (p *PTE) Valid() bool {
+	return atomic.LoadUintptr((*uintptr)(p))&present != 0
+}
+
+// Opts returns the PTE options.
+//
+// These are all options except Valid and Super.
+//
+//go:nosplit
+func (p *PTE) Opts() MapOpts {
+	v := atomic.LoadUintptr((*uintptr)(p))
+	return MapOpts{
+		AccessType: usermem.AccessType{
+			Read:    v&present != 0,
+			Write:   v&writable != 0,
+			Execute: v&executeDisable == 0,
+		},
+		Global: v&global != 0,
+		User:   v&user != 0,
+	}
+}
+
+// SetSuper sets this page as a super page.
+//
+// The page must not be valid or a panic will result.
+//
+//go:nosplit
+func (p *PTE) SetSuper() {
+	if p.Valid() {
+		// This is not allowed.
+		panic("SetSuper called on valid page!")
+	}
+	atomic.StoreUintptr((*uintptr)(p), super)
+}
+
+// IsSuper returns true iff this page is a super page.
+//
+//go:nosplit
+func (p *PTE) IsSuper() bool {
+	return atomic.LoadUintptr((*uintptr)(p))&super != 0
+}
+
+// Set sets this PTE value.
+//
+// This does not change the super page property.
+//
+//go:nosplit
+func (p *PTE) Set(addr uintptr, opts MapOpts) {
+	if !opts.AccessType.Any() {
+		p.Clear()
+		return
+	}
+	v := (addr &^ optionMask) | present | accessed
+	if opts.User {
+		v |= user
+	}
+	if opts.Global {
+		v |= global
+	}
+	if !opts.AccessType.Execute {
+		v |= executeDisable
+	}
+	if opts.AccessType.Write {
+		v |= writable | dirty
+	}
+	if p.IsSuper() {
+		// Note that this is inherited from the previous instance. Set
+		// does not change the value of Super. See above.
+		v |= super
+	}
+	atomic.StoreUintptr((*uintptr)(p), v)
+}
+
+// setPageTable sets this PTE value and forces the write bit and super bit to
+// be cleared. This is used explicitly for breaking super pages.
+//
+//go:nosplit
+func (p *PTE) setPageTable(addr uintptr) {
+	v := (addr &^ optionMask) | present | user | writable | accessed | dirty
+	atomic.StoreUintptr((*uintptr)(p), v)
+}
+
+// Address extracts the address. This should only be used if Valid returns true.
+//
+//go:nosplit
+func (p *PTE) Address() uintptr {
+	return atomic.LoadUintptr((*uintptr)(p)) &^ optionMask
+}