summaryrefslogtreecommitdiffhomepage
path: root/pkg/sentry/platform
diff options
context:
space:
mode:
Diffstat (limited to 'pkg/sentry/platform')
-rw-r--r--pkg/sentry/platform/BUILD40
-rwxr-xr-xpkg/sentry/platform/file_range.go62
-rw-r--r--pkg/sentry/platform/interrupt/BUILD19
-rwxr-xr-xpkg/sentry/platform/interrupt/interrupt_state_autogen.go4
-rw-r--r--pkg/sentry/platform/interrupt/interrupt_test.go99
-rw-r--r--pkg/sentry/platform/kvm/BUILD68
-rwxr-xr-xpkg/sentry/platform/kvm/kvm_state_autogen.go4
-rw-r--r--pkg/sentry/platform/kvm/kvm_test.go533
-rw-r--r--pkg/sentry/platform/kvm/testutil/BUILD15
-rw-r--r--pkg/sentry/platform/kvm/testutil/testutil.go75
-rw-r--r--pkg/sentry/platform/kvm/testutil/testutil_amd64.go135
-rw-r--r--pkg/sentry/platform/kvm/testutil/testutil_amd64.s98
-rw-r--r--pkg/sentry/platform/kvm/virtual_map_test.go93
-rwxr-xr-xpkg/sentry/platform/platform_state_autogen.go24
-rw-r--r--pkg/sentry/platform/ptrace/BUILD33
-rwxr-xr-xpkg/sentry/platform/ptrace/ptrace_state_autogen.go4
-rw-r--r--pkg/sentry/platform/ring0/BUILD53
-rw-r--r--pkg/sentry/platform/ring0/defs.go121
-rw-r--r--pkg/sentry/platform/ring0/defs_amd64.go136
-rwxr-xr-xpkg/sentry/platform/ring0/defs_impl.go538
-rwxr-xr-x[-rw-r--r--]pkg/sentry/platform/ring0/entry_impl_amd64.s (renamed from pkg/sentry/platform/ring0/entry_amd64.s)64
-rw-r--r--pkg/sentry/platform/ring0/gen_offsets/BUILD26
-rw-r--r--pkg/sentry/platform/ring0/gen_offsets/main.go24
-rw-r--r--pkg/sentry/platform/ring0/offsets_amd64.go92
-rw-r--r--pkg/sentry/platform/ring0/pagetables/BUILD105
-rw-r--r--pkg/sentry/platform/ring0/pagetables/pagetables_amd64_test.go75
-rwxr-xr-xpkg/sentry/platform/ring0/pagetables/pagetables_state_autogen.go4
-rw-r--r--pkg/sentry/platform/ring0/pagetables/pagetables_test.go156
-rwxr-xr-x[-rw-r--r--]pkg/sentry/platform/ring0/pagetables/walker_empty.go (renamed from pkg/sentry/platform/ring0/pagetables/walker_amd64.go)98
-rwxr-xr-xpkg/sentry/platform/ring0/pagetables/walker_lookup.go255
-rwxr-xr-xpkg/sentry/platform/ring0/pagetables/walker_map.go255
-rwxr-xr-xpkg/sentry/platform/ring0/pagetables/walker_unmap.go255
-rwxr-xr-xpkg/sentry/platform/ring0/ring0_state_autogen.go4
-rw-r--r--pkg/sentry/platform/ring0/x86.go264
-rw-r--r--pkg/sentry/platform/safecopy/BUILD30
-rw-r--r--pkg/sentry/platform/safecopy/LICENSE27
-rwxr-xr-xpkg/sentry/platform/safecopy/safecopy_state_autogen.go4
-rw-r--r--pkg/sentry/platform/safecopy/safecopy_test.go617
38 files changed, 1500 insertions, 3009 deletions
diff --git a/pkg/sentry/platform/BUILD b/pkg/sentry/platform/BUILD
deleted file mode 100644
index 9aa6ec507..000000000
--- a/pkg/sentry/platform/BUILD
+++ /dev/null
@@ -1,40 +0,0 @@
-package(licenses = ["notice"])
-
-load("//tools/go_generics:defs.bzl", "go_template_instance")
-load("//tools/go_stateify:defs.bzl", "go_library")
-
-go_template_instance(
- name = "file_range",
- out = "file_range.go",
- package = "platform",
- prefix = "File",
- template = "//pkg/segment:generic_range",
- types = {
- "T": "uint64",
- },
-)
-
-go_library(
- name = "platform",
- srcs = [
- "context.go",
- "file_range.go",
- "mmap_min_addr.go",
- "platform.go",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform",
- visibility = ["//pkg/sentry:internal"],
- deps = [
- "//pkg/abi/linux",
- "//pkg/atomicbitops",
- "//pkg/log",
- "//pkg/seccomp",
- "//pkg/sentry/arch",
- "//pkg/sentry/context",
- "//pkg/sentry/platform/safecopy",
- "//pkg/sentry/safemem",
- "//pkg/sentry/usage",
- "//pkg/sentry/usermem",
- "//pkg/syserror",
- ],
-)
diff --git a/pkg/sentry/platform/file_range.go b/pkg/sentry/platform/file_range.go
new file mode 100755
index 000000000..685d360e3
--- /dev/null
+++ b/pkg/sentry/platform/file_range.go
@@ -0,0 +1,62 @@
+package platform
+
+// A Range represents a contiguous range of T.
+//
+// +stateify savable
+type FileRange struct {
+ // Start is the inclusive start of the range.
+ Start uint64
+
+ // End is the exclusive end of the range.
+ End uint64
+}
+
+// WellFormed returns true if r.Start <= r.End. All other methods on a Range
+// require that the Range is well-formed.
+func (r FileRange) WellFormed() bool {
+ return r.Start <= r.End
+}
+
+// Length returns the length of the range.
+func (r FileRange) Length() uint64 {
+ return r.End - r.Start
+}
+
+// Contains returns true if r contains x.
+func (r FileRange) Contains(x uint64) bool {
+ return r.Start <= x && x < r.End
+}
+
+// Overlaps returns true if r and r2 overlap.
+func (r FileRange) Overlaps(r2 FileRange) bool {
+ return r.Start < r2.End && r2.Start < r.End
+}
+
+// IsSupersetOf returns true if r is a superset of r2; that is, the range r2 is
+// contained within r.
+func (r FileRange) IsSupersetOf(r2 FileRange) bool {
+ return r.Start <= r2.Start && r.End >= r2.End
+}
+
+// Intersect returns a range consisting of the intersection between r and r2.
+// If r and r2 do not overlap, Intersect returns a range with unspecified
+// bounds, but for which Length() == 0.
+func (r FileRange) Intersect(r2 FileRange) FileRange {
+ if r.Start < r2.Start {
+ r.Start = r2.Start
+ }
+ if r.End > r2.End {
+ r.End = r2.End
+ }
+ if r.End < r.Start {
+ r.End = r.Start
+ }
+ return r
+}
+
+// CanSplitAt returns true if it is legal to split a segment spanning the range
+// r at x; that is, splitting at x would produce two ranges, both of which have
+// non-zero length.
+func (r FileRange) CanSplitAt(x uint64) bool {
+ return r.Contains(x) && r.Start < x
+}
diff --git a/pkg/sentry/platform/interrupt/BUILD b/pkg/sentry/platform/interrupt/BUILD
deleted file mode 100644
index eeb634644..000000000
--- a/pkg/sentry/platform/interrupt/BUILD
+++ /dev/null
@@ -1,19 +0,0 @@
-load("//tools/go_stateify:defs.bzl", "go_library", "go_test")
-
-package(licenses = ["notice"])
-
-go_library(
- name = "interrupt",
- srcs = [
- "interrupt.go",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform/interrupt",
- visibility = ["//pkg/sentry:internal"],
-)
-
-go_test(
- name = "interrupt_test",
- size = "small",
- srcs = ["interrupt_test.go"],
- embed = [":interrupt"],
-)
diff --git a/pkg/sentry/platform/interrupt/interrupt_state_autogen.go b/pkg/sentry/platform/interrupt/interrupt_state_autogen.go
new file mode 100755
index 000000000..15e8bacdf
--- /dev/null
+++ b/pkg/sentry/platform/interrupt/interrupt_state_autogen.go
@@ -0,0 +1,4 @@
+// automatically generated by stateify.
+
+package interrupt
+
diff --git a/pkg/sentry/platform/interrupt/interrupt_test.go b/pkg/sentry/platform/interrupt/interrupt_test.go
deleted file mode 100644
index 0ecdf6e7a..000000000
--- a/pkg/sentry/platform/interrupt/interrupt_test.go
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package interrupt
-
-import (
- "testing"
-)
-
-type countingReceiver struct {
- interrupts int
-}
-
-// NotifyInterrupt implements Receiver.NotifyInterrupt.
-func (r *countingReceiver) NotifyInterrupt() {
- r.interrupts++
-}
-
-func TestSingleInterruptBeforeEnable(t *testing.T) {
- var (
- f Forwarder
- r countingReceiver
- )
- f.NotifyInterrupt()
- // The interrupt should cause the first Enable to fail.
- if f.Enable(&r) {
- f.Disable()
- t.Fatalf("Enable: got true, wanted false")
- }
- // The failing Enable "acknowledges" the interrupt, allowing future Enables
- // to succeed.
- if !f.Enable(&r) {
- t.Fatalf("Enable: got false, wanted true")
- }
- f.Disable()
-}
-
-func TestMultipleInterruptsBeforeEnable(t *testing.T) {
- var (
- f Forwarder
- r countingReceiver
- )
- f.NotifyInterrupt()
- f.NotifyInterrupt()
- // The interrupts should cause the first Enable to fail.
- if f.Enable(&r) {
- f.Disable()
- t.Fatalf("Enable: got true, wanted false")
- }
- // Interrupts are deduplicated while the Forwarder is disabled, so the
- // failing Enable "acknowledges" all interrupts, allowing future Enables to
- // succeed.
- if !f.Enable(&r) {
- t.Fatalf("Enable: got false, wanted true")
- }
- f.Disable()
-}
-
-func TestSingleInterruptAfterEnable(t *testing.T) {
- var (
- f Forwarder
- r countingReceiver
- )
- if !f.Enable(&r) {
- t.Fatalf("Enable: got false, wanted true")
- }
- defer f.Disable()
- f.NotifyInterrupt()
- if r.interrupts != 1 {
- t.Errorf("interrupts: got %d, wanted 1", r.interrupts)
- }
-}
-
-func TestMultipleInterruptsAfterEnable(t *testing.T) {
- var (
- f Forwarder
- r countingReceiver
- )
- if !f.Enable(&r) {
- t.Fatalf("Enable: got false, wanted true")
- }
- defer f.Disable()
- f.NotifyInterrupt()
- f.NotifyInterrupt()
- if r.interrupts != 2 {
- t.Errorf("interrupts: got %d, wanted 2", r.interrupts)
- }
-}
diff --git a/pkg/sentry/platform/kvm/BUILD b/pkg/sentry/platform/kvm/BUILD
deleted file mode 100644
index ad8b95744..000000000
--- a/pkg/sentry/platform/kvm/BUILD
+++ /dev/null
@@ -1,68 +0,0 @@
-load("//tools/go_stateify:defs.bzl", "go_library", "go_test")
-
-package(licenses = ["notice"])
-
-go_library(
- name = "kvm",
- srcs = [
- "address_space.go",
- "allocator.go",
- "bluepill.go",
- "bluepill_amd64.go",
- "bluepill_amd64.s",
- "bluepill_amd64_unsafe.go",
- "bluepill_fault.go",
- "bluepill_unsafe.go",
- "context.go",
- "filters.go",
- "kvm.go",
- "kvm_amd64.go",
- "kvm_amd64_unsafe.go",
- "kvm_const.go",
- "machine.go",
- "machine_amd64.go",
- "machine_amd64_unsafe.go",
- "machine_unsafe.go",
- "physical_map.go",
- "virtual_map.go",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform/kvm",
- visibility = ["//pkg/sentry:internal"],
- deps = [
- "//pkg/abi/linux",
- "//pkg/atomicbitops",
- "//pkg/cpuid",
- "//pkg/log",
- "//pkg/procid",
- "//pkg/seccomp",
- "//pkg/sentry/arch",
- "//pkg/sentry/platform",
- "//pkg/sentry/platform/interrupt",
- "//pkg/sentry/platform/ring0",
- "//pkg/sentry/platform/ring0/pagetables",
- "//pkg/sentry/platform/safecopy",
- "//pkg/sentry/time",
- "//pkg/sentry/usermem",
- ],
-)
-
-go_test(
- name = "kvm_test",
- srcs = [
- "kvm_test.go",
- "virtual_map_test.go",
- ],
- embed = [":kvm"],
- tags = [
- "nogotsan",
- "requires-kvm",
- ],
- deps = [
- "//pkg/sentry/arch",
- "//pkg/sentry/platform",
- "//pkg/sentry/platform/kvm/testutil",
- "//pkg/sentry/platform/ring0",
- "//pkg/sentry/platform/ring0/pagetables",
- "//pkg/sentry/usermem",
- ],
-)
diff --git a/pkg/sentry/platform/kvm/kvm_state_autogen.go b/pkg/sentry/platform/kvm/kvm_state_autogen.go
new file mode 100755
index 000000000..5ab0e0735
--- /dev/null
+++ b/pkg/sentry/platform/kvm/kvm_state_autogen.go
@@ -0,0 +1,4 @@
+// automatically generated by stateify.
+
+package kvm
+
diff --git a/pkg/sentry/platform/kvm/kvm_test.go b/pkg/sentry/platform/kvm/kvm_test.go
deleted file mode 100644
index 30df725d4..000000000
--- a/pkg/sentry/platform/kvm/kvm_test.go
+++ /dev/null
@@ -1,533 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package kvm
-
-import (
- "math/rand"
- "reflect"
- "sync/atomic"
- "syscall"
- "testing"
- "time"
-
- "gvisor.dev/gvisor/pkg/sentry/arch"
- "gvisor.dev/gvisor/pkg/sentry/platform"
- "gvisor.dev/gvisor/pkg/sentry/platform/kvm/testutil"
- "gvisor.dev/gvisor/pkg/sentry/platform/ring0"
- "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables"
- "gvisor.dev/gvisor/pkg/sentry/usermem"
-)
-
-var dummyFPState = (*byte)(arch.NewFloatingPointData())
-
-type testHarness interface {
- Errorf(format string, args ...interface{})
- Fatalf(format string, args ...interface{})
-}
-
-func kvmTest(t testHarness, setup func(*KVM), fn func(*vCPU) bool) {
- // Create the machine.
- deviceFile, err := OpenDevice()
- if err != nil {
- t.Fatalf("error opening device file: %v", err)
- }
- k, err := New(deviceFile)
- if err != nil {
- t.Fatalf("error creating KVM instance: %v", err)
- }
- defer k.machine.Destroy()
-
- // Call additional setup.
- if setup != nil {
- setup(k)
- }
-
- var c *vCPU // For recovery.
- defer func() {
- redpill()
- if c != nil {
- k.machine.Put(c)
- }
- }()
- for {
- c = k.machine.Get()
- if !fn(c) {
- break
- }
-
- // We put the vCPU here and clear the value so that the
- // deferred recovery will not re-put it above.
- k.machine.Put(c)
- c = nil
- }
-}
-
-func bluepillTest(t testHarness, fn func(*vCPU)) {
- kvmTest(t, nil, func(c *vCPU) bool {
- bluepill(c)
- fn(c)
- return false
- })
-}
-
-func TestKernelSyscall(t *testing.T) {
- bluepillTest(t, func(c *vCPU) {
- redpill() // Leave guest mode.
- if got := atomic.LoadUint32(&c.state); got != vCPUUser {
- t.Errorf("vCPU not in ready state: got %v", got)
- }
- })
-}
-
-func hostFault() {
- defer func() {
- recover()
- }()
- var foo *int
- *foo = 0
-}
-
-func TestKernelFault(t *testing.T) {
- hostFault() // Ensure recovery works.
- bluepillTest(t, func(c *vCPU) {
- hostFault()
- if got := atomic.LoadUint32(&c.state); got != vCPUUser {
- t.Errorf("vCPU not in ready state: got %v", got)
- }
- })
-}
-
-func TestKernelFloatingPoint(t *testing.T) {
- bluepillTest(t, func(c *vCPU) {
- if !testutil.FloatingPointWorks() {
- t.Errorf("floating point does not work, and it should!")
- }
- })
-}
-
-func applicationTest(t testHarness, useHostMappings bool, target func(), fn func(*vCPU, *syscall.PtraceRegs, *pagetables.PageTables) bool) {
- // Initialize registers & page tables.
- var (
- regs syscall.PtraceRegs
- pt *pagetables.PageTables
- )
- testutil.SetTestTarget(&regs, target)
-
- kvmTest(t, func(k *KVM) {
- // Create new page tables.
- as, _, err := k.NewAddressSpace(nil /* invalidator */)
- if err != nil {
- t.Fatalf("can't create new address space: %v", err)
- }
- pt = as.(*addressSpace).pageTables
-
- if useHostMappings {
- // Apply the physical mappings to these page tables.
- // (This is normally dangerous, since they point to
- // physical pages that may not exist. This shouldn't be
- // done for regular user code, but is fine for test
- // purposes.)
- applyPhysicalRegions(func(pr physicalRegion) bool {
- pt.Map(usermem.Addr(pr.virtual), pr.length, pagetables.MapOpts{
- AccessType: usermem.AnyAccess,
- User: true,
- }, pr.physical)
- return true // Keep iterating.
- })
- }
- }, func(c *vCPU) bool {
- // Invoke the function with the extra data.
- return fn(c, &regs, pt)
- })
-}
-
-func TestApplicationSyscall(t *testing.T) {
- applicationTest(t, true, testutil.SyscallLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- FullRestore: true,
- }, &si); err == platform.ErrContextInterrupt {
- return true // Retry.
- } else if err != nil {
- t.Errorf("application syscall with full restore failed: %v", err)
- }
- return false
- })
- applicationTest(t, true, testutil.SyscallLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err == platform.ErrContextInterrupt {
- return true // Retry.
- } else if err != nil {
- t.Errorf("application syscall with partial restore failed: %v", err)
- }
- return false
- })
-}
-
-func TestApplicationFault(t *testing.T) {
- applicationTest(t, true, testutil.Touch, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- testutil.SetTouchTarget(regs, nil) // Cause fault.
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- FullRestore: true,
- }, &si); err == platform.ErrContextInterrupt {
- return true // Retry.
- } else if err != platform.ErrContextSignal || si.Signo != int32(syscall.SIGSEGV) {
- t.Errorf("application fault with full restore got (%v, %v), expected (%v, SIGSEGV)", err, si, platform.ErrContextSignal)
- }
- return false
- })
- applicationTest(t, true, testutil.Touch, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- testutil.SetTouchTarget(regs, nil) // Cause fault.
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err == platform.ErrContextInterrupt {
- return true // Retry.
- } else if err != platform.ErrContextSignal || si.Signo != int32(syscall.SIGSEGV) {
- t.Errorf("application fault with partial restore got (%v, %v), expected (%v, SIGSEGV)", err, si, platform.ErrContextSignal)
- }
- return false
- })
-}
-
-func TestRegistersSyscall(t *testing.T) {
- applicationTest(t, true, testutil.TwiddleRegsSyscall, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- testutil.SetTestRegs(regs) // Fill values for all registers.
- for {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err == platform.ErrContextInterrupt {
- continue // Retry.
- } else if err != nil {
- t.Errorf("application register check with partial restore got unexpected error: %v", err)
- }
- if err := testutil.CheckTestRegs(regs, false); err != nil {
- t.Errorf("application register check with partial restore failed: %v", err)
- }
- break // Done.
- }
- return false
- })
-}
-
-func TestRegistersFault(t *testing.T) {
- applicationTest(t, true, testutil.TwiddleRegsFault, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- testutil.SetTestRegs(regs) // Fill values for all registers.
- for {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- FullRestore: true,
- }, &si); err == platform.ErrContextInterrupt {
- continue // Retry.
- } else if err != platform.ErrContextSignal || si.Signo != int32(syscall.SIGSEGV) {
- t.Errorf("application register check with full restore got unexpected error: %v", err)
- }
- if err := testutil.CheckTestRegs(regs, true); err != nil {
- t.Errorf("application register check with full restore failed: %v", err)
- }
- break // Done.
- }
- return false
- })
-}
-
-func TestSegments(t *testing.T) {
- applicationTest(t, true, testutil.TwiddleSegments, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- testutil.SetTestSegments(regs)
- for {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- FullRestore: true,
- }, &si); err == platform.ErrContextInterrupt {
- continue // Retry.
- } else if err != nil {
- t.Errorf("application segment check with full restore got unexpected error: %v", err)
- }
- if err := testutil.CheckTestSegments(regs); err != nil {
- t.Errorf("application segment check with full restore failed: %v", err)
- }
- break // Done.
- }
- return false
- })
-}
-
-func TestBounce(t *testing.T) {
- applicationTest(t, true, testutil.SpinLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- go func() {
- time.Sleep(time.Millisecond)
- c.BounceToKernel()
- }()
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err != platform.ErrContextInterrupt {
- t.Errorf("application partial restore: got %v, wanted %v", err, platform.ErrContextInterrupt)
- }
- return false
- })
- applicationTest(t, true, testutil.SpinLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- go func() {
- time.Sleep(time.Millisecond)
- c.BounceToKernel()
- }()
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- FullRestore: true,
- }, &si); err != platform.ErrContextInterrupt {
- t.Errorf("application full restore: got %v, wanted %v", err, platform.ErrContextInterrupt)
- }
- return false
- })
-}
-
-func TestBounceStress(t *testing.T) {
- applicationTest(t, true, testutil.SpinLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- randomSleep := func() {
- // O(hundreds of microseconds) is appropriate to ensure
- // different overlaps and different schedules.
- if n := rand.Intn(1000); n > 100 {
- time.Sleep(time.Duration(n) * time.Microsecond)
- }
- }
- for i := 0; i < 1000; i++ {
- // Start an asynchronously executing goroutine that
- // calls Bounce at pseudo-random point in time.
- // This should wind up calling Bounce when the
- // kernel is in various stages of the switch.
- go func() {
- randomSleep()
- c.BounceToKernel()
- }()
- randomSleep()
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err != platform.ErrContextInterrupt {
- t.Errorf("application partial restore: got %v, wanted %v", err, platform.ErrContextInterrupt)
- }
- c.unlock()
- randomSleep()
- c.lock()
- }
- return false
- })
-}
-
-func TestInvalidate(t *testing.T) {
- var data uintptr // Used below.
- applicationTest(t, true, testutil.Touch, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- testutil.SetTouchTarget(regs, &data) // Read legitimate value.
- for {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err == platform.ErrContextInterrupt {
- continue // Retry.
- } else if err != nil {
- t.Errorf("application partial restore: got %v, wanted nil", err)
- }
- break // Done.
- }
- // Unmap the page containing data & invalidate.
- pt.Unmap(usermem.Addr(reflect.ValueOf(&data).Pointer() & ^uintptr(usermem.PageSize-1)), usermem.PageSize)
- for {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- Flush: true,
- }, &si); err == platform.ErrContextInterrupt {
- continue // Retry.
- } else if err != platform.ErrContextSignal {
- t.Errorf("application partial restore: got %v, wanted %v", err, platform.ErrContextSignal)
- }
- break // Success.
- }
- return false
- })
-}
-
-// IsFault returns true iff the given signal represents a fault.
-func IsFault(err error, si *arch.SignalInfo) bool {
- return err == platform.ErrContextSignal && si.Signo == int32(syscall.SIGSEGV)
-}
-
-func TestEmptyAddressSpace(t *testing.T) {
- applicationTest(t, false, testutil.SyscallLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err == platform.ErrContextInterrupt {
- return true // Retry.
- } else if !IsFault(err, &si) {
- t.Errorf("first fault with partial restore failed got %v", err)
- t.Logf("registers: %#v", &regs)
- }
- return false
- })
- applicationTest(t, false, testutil.SyscallLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- FullRestore: true,
- }, &si); err == platform.ErrContextInterrupt {
- return true // Retry.
- } else if !IsFault(err, &si) {
- t.Errorf("first fault with full restore failed got %v", err)
- t.Logf("registers: %#v", &regs)
- }
- return false
- })
-}
-
-func TestWrongVCPU(t *testing.T) {
- kvmTest(t, nil, func(c1 *vCPU) bool {
- kvmTest(t, nil, func(c2 *vCPU) bool {
- // Basic test, one then the other.
- bluepill(c1)
- bluepill(c2)
- if c2.switches == 0 {
- // Don't allow the test to proceed if this fails.
- t.Fatalf("wrong vCPU#2 switches: vCPU1=%+v,vCPU2=%+v", c1, c2)
- }
-
- // Alternate vCPUs; we expect to need to trigger the
- // wrong vCPU path on each switch.
- for i := 0; i < 100; i++ {
- bluepill(c1)
- bluepill(c2)
- }
- if count := c1.switches; count < 90 {
- t.Errorf("wrong vCPU#1 switches: vCPU1=%+v,vCPU2=%+v", c1, c2)
- }
- if count := c2.switches; count < 90 {
- t.Errorf("wrong vCPU#2 switches: vCPU1=%+v,vCPU2=%+v", c1, c2)
- }
- return false
- })
- return false
- })
- kvmTest(t, nil, func(c1 *vCPU) bool {
- kvmTest(t, nil, func(c2 *vCPU) bool {
- bluepill(c1)
- bluepill(c2)
- return false
- })
- return false
- })
-}
-
-func BenchmarkApplicationSyscall(b *testing.B) {
- var (
- i int // Iteration includes machine.Get() / machine.Put().
- a int // Count for ErrContextInterrupt.
- )
- applicationTest(b, true, testutil.SyscallLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err == platform.ErrContextInterrupt {
- a++
- return true // Ignore.
- } else if err != nil {
- b.Fatalf("benchmark failed: %v", err)
- }
- i++
- return i < b.N
- })
- if a != 0 {
- b.Logf("ErrContextInterrupt occurred %d times (in %d iterations).", a, a+i)
- }
-}
-
-func BenchmarkKernelSyscall(b *testing.B) {
- // Note that the target passed here is irrelevant, we never execute SwitchToUser.
- applicationTest(b, true, testutil.Getpid, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- // iteration does not include machine.Get() / machine.Put().
- for i := 0; i < b.N; i++ {
- testutil.Getpid()
- }
- return false
- })
-}
-
-func BenchmarkWorldSwitchToUserRoundtrip(b *testing.B) {
- // see BenchmarkApplicationSyscall.
- var (
- i int
- a int
- )
- applicationTest(b, true, testutil.SyscallLoop, func(c *vCPU, regs *syscall.PtraceRegs, pt *pagetables.PageTables) bool {
- var si arch.SignalInfo
- if _, err := c.SwitchToUser(ring0.SwitchOpts{
- Registers: regs,
- FloatingPointState: dummyFPState,
- PageTables: pt,
- }, &si); err == platform.ErrContextInterrupt {
- a++
- return true // Ignore.
- } else if err != nil {
- b.Fatalf("benchmark failed: %v", err)
- }
- // This will intentionally cause the world switch. By executing
- // a host syscall here, we force the transition between guest
- // and host mode.
- testutil.Getpid()
- i++
- return i < b.N
- })
- if a != 0 {
- b.Logf("ErrContextInterrupt occurred %d times (in %d iterations).", a, a+i)
- }
-}
diff --git a/pkg/sentry/platform/kvm/testutil/BUILD b/pkg/sentry/platform/kvm/testutil/BUILD
deleted file mode 100644
index 77a449a8b..000000000
--- a/pkg/sentry/platform/kvm/testutil/BUILD
+++ /dev/null
@@ -1,15 +0,0 @@
-load("//tools/go_stateify:defs.bzl", "go_library")
-
-package(licenses = ["notice"])
-
-go_library(
- name = "testutil",
- testonly = 1,
- srcs = [
- "testutil.go",
- "testutil_amd64.go",
- "testutil_amd64.s",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform/kvm/testutil",
- visibility = ["//pkg/sentry/platform/kvm:__pkg__"],
-)
diff --git a/pkg/sentry/platform/kvm/testutil/testutil.go b/pkg/sentry/platform/kvm/testutil/testutil.go
deleted file mode 100644
index 6cf2359a3..000000000
--- a/pkg/sentry/platform/kvm/testutil/testutil.go
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package testutil provides common assembly stubs for testing.
-package testutil
-
-import (
- "fmt"
- "strings"
-)
-
-// Getpid executes a trivial system call.
-func Getpid()
-
-// Touch touches the value in the first register.
-func Touch()
-
-// SyscallLoop executes a syscall and loops.
-func SyscallLoop()
-
-// SpinLoop spins on the CPU.
-func SpinLoop()
-
-// HaltLoop immediately halts and loops.
-func HaltLoop()
-
-// TwiddleRegsFault twiddles registers then faults.
-func TwiddleRegsFault()
-
-// TwiddleRegsSyscall twiddles registers then executes a syscall.
-func TwiddleRegsSyscall()
-
-// TwiddleSegments reads segments into known registers.
-func TwiddleSegments()
-
-// FloatingPointWorks is a floating point test.
-//
-// It returns true or false.
-func FloatingPointWorks() bool
-
-// RegisterMismatchError is used for checking registers.
-type RegisterMismatchError []string
-
-// Error returns a human-readable error.
-func (r RegisterMismatchError) Error() string {
- return strings.Join([]string(r), ";")
-}
-
-// addRegisterMisatch allows simple chaining of register mismatches.
-func addRegisterMismatch(err error, reg string, got, expected interface{}) error {
- errStr := fmt.Sprintf("%s got %08x, expected %08x", reg, got, expected)
- switch r := err.(type) {
- case nil:
- // Return a new register mismatch.
- return RegisterMismatchError{errStr}
- case RegisterMismatchError:
- // Append the error.
- r = append(r, errStr)
- return r
- default:
- // Leave as is.
- return err
- }
-}
diff --git a/pkg/sentry/platform/kvm/testutil/testutil_amd64.go b/pkg/sentry/platform/kvm/testutil/testutil_amd64.go
deleted file mode 100644
index 203d71528..000000000
--- a/pkg/sentry/platform/kvm/testutil/testutil_amd64.go
+++ /dev/null
@@ -1,135 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build amd64
-
-package testutil
-
-import (
- "reflect"
- "syscall"
-)
-
-// SetTestTarget sets the rip appropriately.
-func SetTestTarget(regs *syscall.PtraceRegs, fn func()) {
- regs.Rip = uint64(reflect.ValueOf(fn).Pointer())
-}
-
-// SetTouchTarget sets rax appropriately.
-func SetTouchTarget(regs *syscall.PtraceRegs, target *uintptr) {
- if target != nil {
- regs.Rax = uint64(reflect.ValueOf(target).Pointer())
- } else {
- regs.Rax = 0
- }
-}
-
-// RewindSyscall rewinds a syscall RIP.
-func RewindSyscall(regs *syscall.PtraceRegs) {
- regs.Rip -= 2
-}
-
-// SetTestRegs initializes registers to known values.
-func SetTestRegs(regs *syscall.PtraceRegs) {
- regs.R15 = 0x15
- regs.R14 = 0x14
- regs.R13 = 0x13
- regs.R12 = 0x12
- regs.Rbp = 0xb9
- regs.Rbx = 0xb4
- regs.R11 = 0x11
- regs.R10 = 0x10
- regs.R9 = 0x09
- regs.R8 = 0x08
- regs.Rax = 0x44
- regs.Rcx = 0xc4
- regs.Rdx = 0xd4
- regs.Rsi = 0x51
- regs.Rdi = 0xd1
- regs.Rsp = 0x59
-}
-
-// CheckTestRegs checks that registers were twiddled per TwiddleRegs.
-func CheckTestRegs(regs *syscall.PtraceRegs, full bool) (err error) {
- if need := ^uint64(0x15); regs.R15 != need {
- err = addRegisterMismatch(err, "R15", regs.R15, need)
- }
- if need := ^uint64(0x14); regs.R14 != need {
- err = addRegisterMismatch(err, "R14", regs.R14, need)
- }
- if need := ^uint64(0x13); regs.R13 != need {
- err = addRegisterMismatch(err, "R13", regs.R13, need)
- }
- if need := ^uint64(0x12); regs.R12 != need {
- err = addRegisterMismatch(err, "R12", regs.R12, need)
- }
- if need := ^uint64(0xb9); regs.Rbp != need {
- err = addRegisterMismatch(err, "Rbp", regs.Rbp, need)
- }
- if need := ^uint64(0xb4); regs.Rbx != need {
- err = addRegisterMismatch(err, "Rbx", regs.Rbx, need)
- }
- if need := ^uint64(0x10); regs.R10 != need {
- err = addRegisterMismatch(err, "R10", regs.R10, need)
- }
- if need := ^uint64(0x09); regs.R9 != need {
- err = addRegisterMismatch(err, "R9", regs.R9, need)
- }
- if need := ^uint64(0x08); regs.R8 != need {
- err = addRegisterMismatch(err, "R8", regs.R8, need)
- }
- if need := ^uint64(0x44); regs.Rax != need {
- err = addRegisterMismatch(err, "Rax", regs.Rax, need)
- }
- if need := ^uint64(0xd4); regs.Rdx != need {
- err = addRegisterMismatch(err, "Rdx", regs.Rdx, need)
- }
- if need := ^uint64(0x51); regs.Rsi != need {
- err = addRegisterMismatch(err, "Rsi", regs.Rsi, need)
- }
- if need := ^uint64(0xd1); regs.Rdi != need {
- err = addRegisterMismatch(err, "Rdi", regs.Rdi, need)
- }
- if need := ^uint64(0x59); regs.Rsp != need {
- err = addRegisterMismatch(err, "Rsp", regs.Rsp, need)
- }
- // Rcx & R11 are ignored if !full is set.
- if need := ^uint64(0x11); full && regs.R11 != need {
- err = addRegisterMismatch(err, "R11", regs.R11, need)
- }
- if need := ^uint64(0xc4); full && regs.Rcx != need {
- err = addRegisterMismatch(err, "Rcx", regs.Rcx, need)
- }
- return
-}
-
-var fsData uint64 = 0x55
-var gsData uint64 = 0x85
-
-// SetTestSegments initializes segments to known values.
-func SetTestSegments(regs *syscall.PtraceRegs) {
- regs.Fs_base = uint64(reflect.ValueOf(&fsData).Pointer())
- regs.Gs_base = uint64(reflect.ValueOf(&gsData).Pointer())
-}
-
-// CheckTestSegments checks that registers were twiddled per TwiddleSegments.
-func CheckTestSegments(regs *syscall.PtraceRegs) (err error) {
- if regs.Rax != fsData {
- err = addRegisterMismatch(err, "Rax", regs.Rax, fsData)
- }
- if regs.Rbx != gsData {
- err = addRegisterMismatch(err, "Rbx", regs.Rcx, gsData)
- }
- return
-}
diff --git a/pkg/sentry/platform/kvm/testutil/testutil_amd64.s b/pkg/sentry/platform/kvm/testutil/testutil_amd64.s
deleted file mode 100644
index 491ec0c2a..000000000
--- a/pkg/sentry/platform/kvm/testutil/testutil_amd64.s
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build amd64
-
-// test_util_amd64.s provides AMD64 test functions.
-
-#include "funcdata.h"
-#include "textflag.h"
-
-TEXT ·Getpid(SB),NOSPLIT,$0
- NO_LOCAL_POINTERS
- MOVQ $39, AX // getpid
- SYSCALL
- RET
-
-TEXT ·Touch(SB),NOSPLIT,$0
-start:
- MOVQ 0(AX), BX // deref AX
- MOVQ $39, AX // getpid
- SYSCALL
- JMP start
-
-TEXT ·HaltLoop(SB),NOSPLIT,$0
-start:
- HLT
- JMP start
-
-TEXT ·SyscallLoop(SB),NOSPLIT,$0
-start:
- SYSCALL
- JMP start
-
-TEXT ·SpinLoop(SB),NOSPLIT,$0
-start:
- JMP start
-
-TEXT ·FloatingPointWorks(SB),NOSPLIT,$0-8
- NO_LOCAL_POINTERS
- MOVQ $1, AX
- MOVQ AX, X0
- MOVQ $39, AX // getpid
- SYSCALL
- MOVQ X0, AX
- CMPQ AX, $1
- SETEQ ret+0(FP)
- RET
-
-#define TWIDDLE_REGS() \
- NOTQ R15; \
- NOTQ R14; \
- NOTQ R13; \
- NOTQ R12; \
- NOTQ BP; \
- NOTQ BX; \
- NOTQ R11; \
- NOTQ R10; \
- NOTQ R9; \
- NOTQ R8; \
- NOTQ AX; \
- NOTQ CX; \
- NOTQ DX; \
- NOTQ SI; \
- NOTQ DI; \
- NOTQ SP;
-
-TEXT ·TwiddleRegsSyscall(SB),NOSPLIT,$0
- TWIDDLE_REGS()
- SYSCALL
- RET // never reached
-
-TEXT ·TwiddleRegsFault(SB),NOSPLIT,$0
- TWIDDLE_REGS()
- JMP AX // must fault
- RET // never reached
-
-#define READ_FS() BYTE $0x64; BYTE $0x48; BYTE $0x8b; BYTE $0x00;
-#define READ_GS() BYTE $0x65; BYTE $0x48; BYTE $0x8b; BYTE $0x00;
-
-TEXT ·TwiddleSegments(SB),NOSPLIT,$0
- MOVQ $0x0, AX
- READ_GS()
- MOVQ AX, BX
- MOVQ $0x0, AX
- READ_FS()
- SYSCALL
- RET // never reached
diff --git a/pkg/sentry/platform/kvm/virtual_map_test.go b/pkg/sentry/platform/kvm/virtual_map_test.go
deleted file mode 100644
index 6a2f145be..000000000
--- a/pkg/sentry/platform/kvm/virtual_map_test.go
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package kvm
-
-import (
- "syscall"
- "testing"
-
- "gvisor.dev/gvisor/pkg/sentry/usermem"
-)
-
-type checker struct {
- ok bool
- accessType usermem.AccessType
-}
-
-func (c *checker) Containing(addr uintptr) func(virtualRegion) {
- c.ok = false // Reset for below calls.
- return func(vr virtualRegion) {
- if vr.virtual <= addr && addr < vr.virtual+vr.length {
- c.ok = true
- c.accessType = vr.accessType
- }
- }
-}
-
-func TestParseMaps(t *testing.T) {
- c := new(checker)
-
- // Simple test.
- if err := applyVirtualRegions(c.Containing(0)); err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
-
- // MMap a new page.
- addr, _, errno := syscall.RawSyscall6(
- syscall.SYS_MMAP, 0, usermem.PageSize,
- syscall.PROT_READ|syscall.PROT_WRITE,
- syscall.MAP_ANONYMOUS|syscall.MAP_PRIVATE, 0, 0)
- if errno != 0 {
- t.Fatalf("unexpected map error: %v", errno)
- }
-
- // Re-parse maps.
- if err := applyVirtualRegions(c.Containing(addr)); err != nil {
- syscall.RawSyscall(syscall.SYS_MUNMAP, addr, usermem.PageSize, 0)
- t.Fatalf("unexpected error: %v", err)
- }
-
- // Assert that it now does contain the region.
- if !c.ok {
- syscall.RawSyscall(syscall.SYS_MUNMAP, addr, usermem.PageSize, 0)
- t.Fatalf("updated map does not contain 0x%08x, expected true", addr)
- }
-
- // Map the region as PROT_NONE.
- newAddr, _, errno := syscall.RawSyscall6(
- syscall.SYS_MMAP, addr, usermem.PageSize,
- syscall.PROT_NONE,
- syscall.MAP_ANONYMOUS|syscall.MAP_FIXED|syscall.MAP_PRIVATE, 0, 0)
- if errno != 0 {
- t.Fatalf("unexpected map error: %v", errno)
- }
- if newAddr != addr {
- t.Fatalf("unable to remap address: got 0x%08x, wanted 0x%08x", newAddr, addr)
- }
-
- // Re-parse maps.
- if err := applyVirtualRegions(c.Containing(addr)); err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if !c.ok {
- t.Fatalf("final map does not contain 0x%08x, expected true", addr)
- }
- if c.accessType.Read || c.accessType.Write || c.accessType.Execute {
- t.Fatalf("final map has incorrect permissions for 0x%08x", addr)
- }
-
- // Unmap the region.
- syscall.RawSyscall(syscall.SYS_MUNMAP, addr, usermem.PageSize, 0)
-}
diff --git a/pkg/sentry/platform/platform_state_autogen.go b/pkg/sentry/platform/platform_state_autogen.go
new file mode 100755
index 000000000..28dcd1764
--- /dev/null
+++ b/pkg/sentry/platform/platform_state_autogen.go
@@ -0,0 +1,24 @@
+// automatically generated by stateify.
+
+package platform
+
+import (
+ "gvisor.dev/gvisor/pkg/state"
+)
+
+func (x *FileRange) beforeSave() {}
+func (x *FileRange) save(m state.Map) {
+ x.beforeSave()
+ m.Save("Start", &x.Start)
+ m.Save("End", &x.End)
+}
+
+func (x *FileRange) afterLoad() {}
+func (x *FileRange) load(m state.Map) {
+ m.Load("Start", &x.Start)
+ m.Load("End", &x.End)
+}
+
+func init() {
+ state.Register("platform.FileRange", (*FileRange)(nil), state.Fns{Save: (*FileRange).save, Load: (*FileRange).load})
+}
diff --git a/pkg/sentry/platform/ptrace/BUILD b/pkg/sentry/platform/ptrace/BUILD
deleted file mode 100644
index 1b6c54e96..000000000
--- a/pkg/sentry/platform/ptrace/BUILD
+++ /dev/null
@@ -1,33 +0,0 @@
-load("//tools/go_stateify:defs.bzl", "go_library")
-
-package(licenses = ["notice"])
-
-go_library(
- name = "ptrace",
- srcs = [
- "filters.go",
- "ptrace.go",
- "ptrace_unsafe.go",
- "stub_amd64.s",
- "stub_unsafe.go",
- "subprocess.go",
- "subprocess_amd64.go",
- "subprocess_linux.go",
- "subprocess_linux_amd64_unsafe.go",
- "subprocess_unsafe.go",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform/ptrace",
- visibility = ["//:sandbox"],
- deps = [
- "//pkg/abi/linux",
- "//pkg/log",
- "//pkg/procid",
- "//pkg/seccomp",
- "//pkg/sentry/arch",
- "//pkg/sentry/platform",
- "//pkg/sentry/platform/interrupt",
- "//pkg/sentry/platform/safecopy",
- "//pkg/sentry/usermem",
- "@org_golang_x_sys//unix:go_default_library",
- ],
-)
diff --git a/pkg/sentry/platform/ptrace/ptrace_state_autogen.go b/pkg/sentry/platform/ptrace/ptrace_state_autogen.go
new file mode 100755
index 000000000..ac83a71e7
--- /dev/null
+++ b/pkg/sentry/platform/ptrace/ptrace_state_autogen.go
@@ -0,0 +1,4 @@
+// automatically generated by stateify.
+
+package ptrace
+
diff --git a/pkg/sentry/platform/ring0/BUILD b/pkg/sentry/platform/ring0/BUILD
deleted file mode 100644
index 8ed6c7652..000000000
--- a/pkg/sentry/platform/ring0/BUILD
+++ /dev/null
@@ -1,53 +0,0 @@
-load("//tools/go_stateify:defs.bzl", "go_library")
-
-package(licenses = ["notice"])
-
-load("//tools/go_generics:defs.bzl", "go_template", "go_template_instance")
-
-go_template(
- name = "defs",
- srcs = [
- "defs.go",
- "defs_amd64.go",
- "offsets_amd64.go",
- "x86.go",
- ],
- visibility = [":__subpackages__"],
-)
-
-go_template_instance(
- name = "defs_impl",
- out = "defs_impl.go",
- package = "ring0",
- template = ":defs",
-)
-
-genrule(
- name = "entry_impl_amd64",
- srcs = ["entry_amd64.s"],
- outs = ["entry_impl_amd64.s"],
- cmd = "(echo -e '// build +amd64\\n' && $(location //pkg/sentry/platform/ring0/gen_offsets) && cat $(SRCS)) > $@",
- tools = ["//pkg/sentry/platform/ring0/gen_offsets"],
-)
-
-go_library(
- name = "ring0",
- srcs = [
- "defs_impl.go",
- "entry_amd64.go",
- "entry_impl_amd64.s",
- "kernel.go",
- "kernel_amd64.go",
- "kernel_unsafe.go",
- "lib_amd64.go",
- "lib_amd64.s",
- "ring0.go",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform/ring0",
- visibility = ["//pkg/sentry:internal"],
- deps = [
- "//pkg/cpuid",
- "//pkg/sentry/platform/ring0/pagetables",
- "//pkg/sentry/usermem",
- ],
-)
diff --git a/pkg/sentry/platform/ring0/defs.go b/pkg/sentry/platform/ring0/defs.go
deleted file mode 100644
index 076063f85..000000000
--- a/pkg/sentry/platform/ring0/defs.go
+++ /dev/null
@@ -1,121 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ring0
-
-import (
- "syscall"
-
- "gvisor.dev/gvisor/pkg/sentry/usermem"
-)
-
-var (
- // UserspaceSize is the total size of userspace.
- UserspaceSize = uintptr(1) << (VirtualAddressBits() - 1)
-
- // MaximumUserAddress is the largest possible user address.
- MaximumUserAddress = (UserspaceSize - 1) & ^uintptr(usermem.PageSize-1)
-
- // KernelStartAddress is the starting kernel address.
- KernelStartAddress = ^uintptr(0) - (UserspaceSize - 1)
-)
-
-// Kernel is a global kernel object.
-//
-// This contains global state, shared by multiple CPUs.
-type Kernel struct {
- KernelArchState
-}
-
-// Hooks are hooks for kernel functions.
-type Hooks interface {
- // KernelSyscall is called for kernel system calls.
- //
- // Return from this call will restore registers and return to the kernel: the
- // registers must be modified directly.
- //
- // If this function is not provided, a kernel exception results in halt.
- //
- // This must be go:nosplit, as this will be on the interrupt stack.
- // Closures are permitted, as the pointer to the closure frame is not
- // passed on the stack.
- KernelSyscall()
-
- // KernelException handles an exception during kernel execution.
- //
- // Return from this call will restore registers and return to the kernel: the
- // registers must be modified directly.
- //
- // If this function is not provided, a kernel exception results in halt.
- //
- // This must be go:nosplit, as this will be on the interrupt stack.
- // Closures are permitted, as the pointer to the closure frame is not
- // passed on the stack.
- KernelException(Vector)
-}
-
-// CPU is the per-CPU struct.
-type CPU struct {
- // self is a self reference.
- //
- // This is always guaranteed to be at offset zero.
- self *CPU
-
- // kernel is reference to the kernel that this CPU was initialized
- // with. This reference is kept for garbage collection purposes: CPU
- // registers may refer to objects within the Kernel object that cannot
- // be safely freed.
- kernel *Kernel
-
- // CPUArchState is architecture-specific state.
- CPUArchState
-
- // registers is a set of registers; these may be used on kernel system
- // calls and exceptions via the Registers function.
- registers syscall.PtraceRegs
-
- // hooks are kernel hooks.
- hooks Hooks
-}
-
-// Registers returns a modifiable-copy of the kernel registers.
-//
-// This is explicitly safe to call during KernelException and KernelSyscall.
-//
-//go:nosplit
-func (c *CPU) Registers() *syscall.PtraceRegs {
- return &c.registers
-}
-
-// SwitchOpts are passed to the Switch function.
-type SwitchOpts struct {
- // Registers are the user register state.
- Registers *syscall.PtraceRegs
-
- // FloatingPointState is a byte pointer where floating point state is
- // saved and restored.
- FloatingPointState *byte
-
- // PageTables are the application page tables.
- PageTables *pagetables.PageTables
-
- // Flush indicates that a TLB flush should be forced on switch.
- Flush bool
-
- // FullRestore indicates that an iret-based restore should be used.
- FullRestore bool
-
- // SwitchArchOpts are architecture-specific options.
- SwitchArchOpts
-}
diff --git a/pkg/sentry/platform/ring0/defs_amd64.go b/pkg/sentry/platform/ring0/defs_amd64.go
deleted file mode 100644
index 7206322b1..000000000
--- a/pkg/sentry/platform/ring0/defs_amd64.go
+++ /dev/null
@@ -1,136 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build amd64
-
-package ring0
-
-import (
- "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables"
-)
-
-// Segment indices and Selectors.
-const (
- // Index into GDT array.
- _ = iota // Null descriptor first.
- _ // Reserved (Linux is kernel 32).
- segKcode // Kernel code (64-bit).
- segKdata // Kernel data.
- segUcode32 // User code (32-bit).
- segUdata // User data.
- segUcode64 // User code (64-bit).
- segTss // Task segment descriptor.
- segTssHi // Upper bits for TSS.
- segLast // Last segment (terminal, not included).
-)
-
-// Selectors.
-const (
- Kcode Selector = segKcode << 3
- Kdata Selector = segKdata << 3
- Ucode32 Selector = (segUcode32 << 3) | 3
- Udata Selector = (segUdata << 3) | 3
- Ucode64 Selector = (segUcode64 << 3) | 3
- Tss Selector = segTss << 3
-)
-
-// Standard segments.
-var (
- UserCodeSegment32 SegmentDescriptor
- UserDataSegment SegmentDescriptor
- UserCodeSegment64 SegmentDescriptor
- KernelCodeSegment SegmentDescriptor
- KernelDataSegment SegmentDescriptor
-)
-
-// KernelOpts has initialization options for the kernel.
-type KernelOpts struct {
- // PageTables are the kernel pagetables; this must be provided.
- PageTables *pagetables.PageTables
-}
-
-// KernelArchState contains architecture-specific state.
-type KernelArchState struct {
- KernelOpts
-
- // globalIDT is our set of interrupt gates.
- globalIDT idt64
-}
-
-// CPUArchState contains CPU-specific arch state.
-type CPUArchState struct {
- // stack is the stack used for interrupts on this CPU.
- stack [256]byte
-
- // errorCode is the error code from the last exception.
- errorCode uintptr
-
- // errorType indicates the type of error code here, it is always set
- // along with the errorCode value above.
- //
- // It will either by 1, which indicates a user error, or 0 indicating a
- // kernel error. If the error code below returns false (kernel error),
- // then it cannot provide relevant information about the last
- // exception.
- errorType uintptr
-
- // gdt is the CPU's descriptor table.
- gdt descriptorTable
-
- // tss is the CPU's task state.
- tss TaskState64
-}
-
-// ErrorCode returns the last error code.
-//
-// The returned boolean indicates whether the error code corresponds to the
-// last user error or not. If it does not, then fault information must be
-// ignored. This is generally the result of a kernel fault while servicing a
-// user fault.
-//
-//go:nosplit
-func (c *CPU) ErrorCode() (value uintptr, user bool) {
- return c.errorCode, c.errorType != 0
-}
-
-// ClearErrorCode resets the error code.
-//
-//go:nosplit
-func (c *CPU) ClearErrorCode() {
- c.errorCode = 0 // No code.
- c.errorType = 1 // User mode.
-}
-
-// SwitchArchOpts are embedded in SwitchOpts.
-type SwitchArchOpts struct {
- // UserPCID indicates that the application PCID to be used on switch,
- // assuming that PCIDs are supported.
- //
- // Per pagetables_x86.go, a zero PCID implies a flush.
- UserPCID uint16
-
- // KernelPCID indicates that the kernel PCID to be used on return,
- // assuming that PCIDs are supported.
- //
- // Per pagetables_x86.go, a zero PCID implies a flush.
- KernelPCID uint16
-}
-
-func init() {
- KernelCodeSegment.setCode64(0, 0, 0)
- KernelDataSegment.setData(0, 0xffffffff, 0)
- UserCodeSegment32.setCode64(0, 0, 3)
- UserDataSegment.setData(0, 0xffffffff, 3)
- UserCodeSegment64.setCode64(0, 0, 3)
-}
diff --git a/pkg/sentry/platform/ring0/defs_impl.go b/pkg/sentry/platform/ring0/defs_impl.go
new file mode 100755
index 000000000..d4bfc5a4a
--- /dev/null
+++ b/pkg/sentry/platform/ring0/defs_impl.go
@@ -0,0 +1,538 @@
+package ring0
+
+import (
+ "syscall"
+
+ "fmt"
+ "gvisor.dev/gvisor/pkg/cpuid"
+ "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables"
+ "gvisor.dev/gvisor/pkg/sentry/usermem"
+ "io"
+ "reflect"
+)
+
+var (
+ // UserspaceSize is the total size of userspace.
+ UserspaceSize = uintptr(1) << (VirtualAddressBits() - 1)
+
+ // MaximumUserAddress is the largest possible user address.
+ MaximumUserAddress = (UserspaceSize - 1) & ^uintptr(usermem.PageSize-1)
+
+ // KernelStartAddress is the starting kernel address.
+ KernelStartAddress = ^uintptr(0) - (UserspaceSize - 1)
+)
+
+// Kernel is a global kernel object.
+//
+// This contains global state, shared by multiple CPUs.
+type Kernel struct {
+ KernelArchState
+}
+
+// Hooks are hooks for kernel functions.
+type Hooks interface {
+ // KernelSyscall is called for kernel system calls.
+ //
+ // Return from this call will restore registers and return to the kernel: the
+ // registers must be modified directly.
+ //
+ // If this function is not provided, a kernel exception results in halt.
+ //
+ // This must be go:nosplit, as this will be on the interrupt stack.
+ // Closures are permitted, as the pointer to the closure frame is not
+ // passed on the stack.
+ KernelSyscall()
+
+ // KernelException handles an exception during kernel execution.
+ //
+ // Return from this call will restore registers and return to the kernel: the
+ // registers must be modified directly.
+ //
+ // If this function is not provided, a kernel exception results in halt.
+ //
+ // This must be go:nosplit, as this will be on the interrupt stack.
+ // Closures are permitted, as the pointer to the closure frame is not
+ // passed on the stack.
+ KernelException(Vector)
+}
+
+// CPU is the per-CPU struct.
+type CPU struct {
+ // self is a self reference.
+ //
+ // This is always guaranteed to be at offset zero.
+ self *CPU
+
+ // kernel is reference to the kernel that this CPU was initialized
+ // with. This reference is kept for garbage collection purposes: CPU
+ // registers may refer to objects within the Kernel object that cannot
+ // be safely freed.
+ kernel *Kernel
+
+ // CPUArchState is architecture-specific state.
+ CPUArchState
+
+ // registers is a set of registers; these may be used on kernel system
+ // calls and exceptions via the Registers function.
+ registers syscall.PtraceRegs
+
+ // hooks are kernel hooks.
+ hooks Hooks
+}
+
+// Registers returns a modifiable-copy of the kernel registers.
+//
+// This is explicitly safe to call during KernelException and KernelSyscall.
+//
+//go:nosplit
+func (c *CPU) Registers() *syscall.PtraceRegs {
+ return &c.registers
+}
+
+// SwitchOpts are passed to the Switch function.
+type SwitchOpts struct {
+ // Registers are the user register state.
+ Registers *syscall.PtraceRegs
+
+ // FloatingPointState is a byte pointer where floating point state is
+ // saved and restored.
+ FloatingPointState *byte
+
+ // PageTables are the application page tables.
+ PageTables *pagetables.PageTables
+
+ // Flush indicates that a TLB flush should be forced on switch.
+ Flush bool
+
+ // FullRestore indicates that an iret-based restore should be used.
+ FullRestore bool
+
+ // SwitchArchOpts are architecture-specific options.
+ SwitchArchOpts
+}
+
+// Segment indices and Selectors.
+const (
+ // Index into GDT array.
+ _ = iota // Null descriptor first.
+ _ // Reserved (Linux is kernel 32).
+ segKcode // Kernel code (64-bit).
+ segKdata // Kernel data.
+ segUcode32 // User code (32-bit).
+ segUdata // User data.
+ segUcode64 // User code (64-bit).
+ segTss // Task segment descriptor.
+ segTssHi // Upper bits for TSS.
+ segLast // Last segment (terminal, not included).
+)
+
+// Selectors.
+const (
+ Kcode Selector = segKcode << 3
+ Kdata Selector = segKdata << 3
+ Ucode32 Selector = (segUcode32 << 3) | 3
+ Udata Selector = (segUdata << 3) | 3
+ Ucode64 Selector = (segUcode64 << 3) | 3
+ Tss Selector = segTss << 3
+)
+
+// Standard segments.
+var (
+ UserCodeSegment32 SegmentDescriptor
+ UserDataSegment SegmentDescriptor
+ UserCodeSegment64 SegmentDescriptor
+ KernelCodeSegment SegmentDescriptor
+ KernelDataSegment SegmentDescriptor
+)
+
+// KernelOpts has initialization options for the kernel.
+type KernelOpts struct {
+ // PageTables are the kernel pagetables; this must be provided.
+ PageTables *pagetables.PageTables
+}
+
+// KernelArchState contains architecture-specific state.
+type KernelArchState struct {
+ KernelOpts
+
+ // globalIDT is our set of interrupt gates.
+ globalIDT idt64
+}
+
+// CPUArchState contains CPU-specific arch state.
+type CPUArchState struct {
+ // stack is the stack used for interrupts on this CPU.
+ stack [256]byte
+
+ // errorCode is the error code from the last exception.
+ errorCode uintptr
+
+ // errorType indicates the type of error code here, it is always set
+ // along with the errorCode value above.
+ //
+ // It will either by 1, which indicates a user error, or 0 indicating a
+ // kernel error. If the error code below returns false (kernel error),
+ // then it cannot provide relevant information about the last
+ // exception.
+ errorType uintptr
+
+ // gdt is the CPU's descriptor table.
+ gdt descriptorTable
+
+ // tss is the CPU's task state.
+ tss TaskState64
+}
+
+// ErrorCode returns the last error code.
+//
+// The returned boolean indicates whether the error code corresponds to the
+// last user error or not. If it does not, then fault information must be
+// ignored. This is generally the result of a kernel fault while servicing a
+// user fault.
+//
+//go:nosplit
+func (c *CPU) ErrorCode() (value uintptr, user bool) {
+ return c.errorCode, c.errorType != 0
+}
+
+// ClearErrorCode resets the error code.
+//
+//go:nosplit
+func (c *CPU) ClearErrorCode() {
+ c.errorCode = 0
+ c.errorType = 1
+}
+
+// SwitchArchOpts are embedded in SwitchOpts.
+type SwitchArchOpts struct {
+ // UserPCID indicates that the application PCID to be used on switch,
+ // assuming that PCIDs are supported.
+ //
+ // Per pagetables_x86.go, a zero PCID implies a flush.
+ UserPCID uint16
+
+ // KernelPCID indicates that the kernel PCID to be used on return,
+ // assuming that PCIDs are supported.
+ //
+ // Per pagetables_x86.go, a zero PCID implies a flush.
+ KernelPCID uint16
+}
+
+func init() {
+ KernelCodeSegment.setCode64(0, 0, 0)
+ KernelDataSegment.setData(0, 0xffffffff, 0)
+ UserCodeSegment32.setCode64(0, 0, 3)
+ UserDataSegment.setData(0, 0xffffffff, 3)
+ UserCodeSegment64.setCode64(0, 0, 3)
+}
+
+// Emit prints architecture-specific offsets.
+func Emit(w io.Writer) {
+ fmt.Fprintf(w, "// Automatically generated, do not edit.\n")
+
+ c := &CPU{}
+ fmt.Fprintf(w, "\n// CPU offsets.\n")
+ fmt.Fprintf(w, "#define CPU_SELF 0x%02x\n", reflect.ValueOf(&c.self).Pointer()-reflect.ValueOf(c).Pointer())
+ fmt.Fprintf(w, "#define CPU_REGISTERS 0x%02x\n", reflect.ValueOf(&c.registers).Pointer()-reflect.ValueOf(c).Pointer())
+ fmt.Fprintf(w, "#define CPU_STACK_TOP 0x%02x\n", reflect.ValueOf(&c.stack[0]).Pointer()-reflect.ValueOf(c).Pointer()+uintptr(len(c.stack)))
+ fmt.Fprintf(w, "#define CPU_ERROR_CODE 0x%02x\n", reflect.ValueOf(&c.errorCode).Pointer()-reflect.ValueOf(c).Pointer())
+ fmt.Fprintf(w, "#define CPU_ERROR_TYPE 0x%02x\n", reflect.ValueOf(&c.errorType).Pointer()-reflect.ValueOf(c).Pointer())
+
+ fmt.Fprintf(w, "\n// Bits.\n")
+ fmt.Fprintf(w, "#define _RFLAGS_IF 0x%02x\n", _RFLAGS_IF)
+ fmt.Fprintf(w, "#define _KERNEL_FLAGS 0x%02x\n", KernelFlagsSet)
+
+ fmt.Fprintf(w, "\n// Vectors.\n")
+ fmt.Fprintf(w, "#define DivideByZero 0x%02x\n", DivideByZero)
+ fmt.Fprintf(w, "#define Debug 0x%02x\n", Debug)
+ fmt.Fprintf(w, "#define NMI 0x%02x\n", NMI)
+ fmt.Fprintf(w, "#define Breakpoint 0x%02x\n", Breakpoint)
+ fmt.Fprintf(w, "#define Overflow 0x%02x\n", Overflow)
+ fmt.Fprintf(w, "#define BoundRangeExceeded 0x%02x\n", BoundRangeExceeded)
+ fmt.Fprintf(w, "#define InvalidOpcode 0x%02x\n", InvalidOpcode)
+ fmt.Fprintf(w, "#define DeviceNotAvailable 0x%02x\n", DeviceNotAvailable)
+ fmt.Fprintf(w, "#define DoubleFault 0x%02x\n", DoubleFault)
+ fmt.Fprintf(w, "#define CoprocessorSegmentOverrun 0x%02x\n", CoprocessorSegmentOverrun)
+ fmt.Fprintf(w, "#define InvalidTSS 0x%02x\n", InvalidTSS)
+ fmt.Fprintf(w, "#define SegmentNotPresent 0x%02x\n", SegmentNotPresent)
+ fmt.Fprintf(w, "#define StackSegmentFault 0x%02x\n", StackSegmentFault)
+ fmt.Fprintf(w, "#define GeneralProtectionFault 0x%02x\n", GeneralProtectionFault)
+ fmt.Fprintf(w, "#define PageFault 0x%02x\n", PageFault)
+ fmt.Fprintf(w, "#define X87FloatingPointException 0x%02x\n", X87FloatingPointException)
+ fmt.Fprintf(w, "#define AlignmentCheck 0x%02x\n", AlignmentCheck)
+ fmt.Fprintf(w, "#define MachineCheck 0x%02x\n", MachineCheck)
+ fmt.Fprintf(w, "#define SIMDFloatingPointException 0x%02x\n", SIMDFloatingPointException)
+ fmt.Fprintf(w, "#define VirtualizationException 0x%02x\n", VirtualizationException)
+ fmt.Fprintf(w, "#define SecurityException 0x%02x\n", SecurityException)
+ fmt.Fprintf(w, "#define SyscallInt80 0x%02x\n", SyscallInt80)
+ fmt.Fprintf(w, "#define Syscall 0x%02x\n", Syscall)
+
+ p := &syscall.PtraceRegs{}
+ fmt.Fprintf(w, "\n// Ptrace registers.\n")
+ fmt.Fprintf(w, "#define PTRACE_R15 0x%02x\n", reflect.ValueOf(&p.R15).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_R14 0x%02x\n", reflect.ValueOf(&p.R14).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_R13 0x%02x\n", reflect.ValueOf(&p.R13).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_R12 0x%02x\n", reflect.ValueOf(&p.R12).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RBP 0x%02x\n", reflect.ValueOf(&p.Rbp).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RBX 0x%02x\n", reflect.ValueOf(&p.Rbx).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_R11 0x%02x\n", reflect.ValueOf(&p.R11).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_R10 0x%02x\n", reflect.ValueOf(&p.R10).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_R9 0x%02x\n", reflect.ValueOf(&p.R9).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_R8 0x%02x\n", reflect.ValueOf(&p.R8).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RAX 0x%02x\n", reflect.ValueOf(&p.Rax).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RCX 0x%02x\n", reflect.ValueOf(&p.Rcx).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RDX 0x%02x\n", reflect.ValueOf(&p.Rdx).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RSI 0x%02x\n", reflect.ValueOf(&p.Rsi).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RDI 0x%02x\n", reflect.ValueOf(&p.Rdi).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_ORIGRAX 0x%02x\n", reflect.ValueOf(&p.Orig_rax).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RIP 0x%02x\n", reflect.ValueOf(&p.Rip).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_CS 0x%02x\n", reflect.ValueOf(&p.Cs).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_FLAGS 0x%02x\n", reflect.ValueOf(&p.Eflags).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_RSP 0x%02x\n", reflect.ValueOf(&p.Rsp).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_SS 0x%02x\n", reflect.ValueOf(&p.Ss).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_FS 0x%02x\n", reflect.ValueOf(&p.Fs_base).Pointer()-reflect.ValueOf(p).Pointer())
+ fmt.Fprintf(w, "#define PTRACE_GS 0x%02x\n", reflect.ValueOf(&p.Gs_base).Pointer()-reflect.ValueOf(p).Pointer())
+}
+
+// Useful bits.
+const (
+ _CR0_PE = 1 << 0
+ _CR0_ET = 1 << 4
+ _CR0_AM = 1 << 18
+ _CR0_PG = 1 << 31
+
+ _CR4_PSE = 1 << 4
+ _CR4_PAE = 1 << 5
+ _CR4_PGE = 1 << 7
+ _CR4_OSFXSR = 1 << 9
+ _CR4_OSXMMEXCPT = 1 << 10
+ _CR4_FSGSBASE = 1 << 16
+ _CR4_PCIDE = 1 << 17
+ _CR4_OSXSAVE = 1 << 18
+ _CR4_SMEP = 1 << 20
+
+ _RFLAGS_AC = 1 << 18
+ _RFLAGS_NT = 1 << 14
+ _RFLAGS_IOPL = 3 << 12
+ _RFLAGS_DF = 1 << 10
+ _RFLAGS_IF = 1 << 9
+ _RFLAGS_STEP = 1 << 8
+ _RFLAGS_RESERVED = 1 << 1
+
+ _EFER_SCE = 0x001
+ _EFER_LME = 0x100
+ _EFER_LMA = 0x400
+ _EFER_NX = 0x800
+
+ _MSR_STAR = 0xc0000081
+ _MSR_LSTAR = 0xc0000082
+ _MSR_CSTAR = 0xc0000083
+ _MSR_SYSCALL_MASK = 0xc0000084
+ _MSR_PLATFORM_INFO = 0xce
+ _MSR_MISC_FEATURES = 0x140
+
+ _PLATFORM_INFO_CPUID_FAULT = 1 << 31
+
+ _MISC_FEATURE_CPUID_TRAP = 0x1
+)
+
+const (
+ // KernelFlagsSet should always be set in the kernel.
+ KernelFlagsSet = _RFLAGS_RESERVED
+
+ // UserFlagsSet are always set in userspace.
+ UserFlagsSet = _RFLAGS_RESERVED | _RFLAGS_IF
+
+ // KernelFlagsClear should always be clear in the kernel.
+ KernelFlagsClear = _RFLAGS_STEP | _RFLAGS_IF | _RFLAGS_IOPL | _RFLAGS_AC | _RFLAGS_NT
+
+ // UserFlagsClear are always cleared in userspace.
+ UserFlagsClear = _RFLAGS_NT | _RFLAGS_IOPL
+)
+
+// Vector is an exception vector.
+type Vector uintptr
+
+// Exception vectors.
+const (
+ DivideByZero Vector = iota
+ Debug
+ NMI
+ Breakpoint
+ Overflow
+ BoundRangeExceeded
+ InvalidOpcode
+ DeviceNotAvailable
+ DoubleFault
+ CoprocessorSegmentOverrun
+ InvalidTSS
+ SegmentNotPresent
+ StackSegmentFault
+ GeneralProtectionFault
+ PageFault
+ _
+ X87FloatingPointException
+ AlignmentCheck
+ MachineCheck
+ SIMDFloatingPointException
+ VirtualizationException
+ SecurityException = 0x1e
+ SyscallInt80 = 0x80
+ _NR_INTERRUPTS = SyscallInt80 + 1
+)
+
+// System call vectors.
+const (
+ Syscall Vector = _NR_INTERRUPTS
+)
+
+// VirtualAddressBits returns the number bits available for virtual addresses.
+//
+// Note that sign-extension semantics apply to the highest order bit.
+//
+// FIXME(b/69382326): This should use the cpuid passed to Init.
+func VirtualAddressBits() uint32 {
+ ax, _, _, _ := cpuid.HostID(0x80000008, 0)
+ return (ax >> 8) & 0xff
+}
+
+// PhysicalAddressBits returns the number of bits available for physical addresses.
+//
+// FIXME(b/69382326): This should use the cpuid passed to Init.
+func PhysicalAddressBits() uint32 {
+ ax, _, _, _ := cpuid.HostID(0x80000008, 0)
+ return ax & 0xff
+}
+
+// Selector is a segment Selector.
+type Selector uint16
+
+// SegmentDescriptor is a segment descriptor.
+type SegmentDescriptor struct {
+ bits [2]uint32
+}
+
+// descriptorTable is a collection of descriptors.
+type descriptorTable [32]SegmentDescriptor
+
+// SegmentDescriptorFlags are typed flags within a descriptor.
+type SegmentDescriptorFlags uint32
+
+// SegmentDescriptorFlag declarations.
+const (
+ SegmentDescriptorAccess SegmentDescriptorFlags = 1 << 8 // Access bit (always set).
+ SegmentDescriptorWrite = 1 << 9 // Write permission.
+ SegmentDescriptorExpandDown = 1 << 10 // Grows down, not used.
+ SegmentDescriptorExecute = 1 << 11 // Execute permission.
+ SegmentDescriptorSystem = 1 << 12 // Zero => system, 1 => user code/data.
+ SegmentDescriptorPresent = 1 << 15 // Present.
+ SegmentDescriptorAVL = 1 << 20 // Available.
+ SegmentDescriptorLong = 1 << 21 // Long mode.
+ SegmentDescriptorDB = 1 << 22 // 16 or 32-bit.
+ SegmentDescriptorG = 1 << 23 // Granularity: page or byte.
+)
+
+// Base returns the descriptor's base linear address.
+func (d *SegmentDescriptor) Base() uint32 {
+ return d.bits[1]&0xFF000000 | (d.bits[1]&0x000000FF)<<16 | d.bits[0]>>16
+}
+
+// Limit returns the descriptor size.
+func (d *SegmentDescriptor) Limit() uint32 {
+ l := d.bits[0]&0xFFFF | d.bits[1]&0xF0000
+ if d.bits[1]&uint32(SegmentDescriptorG) != 0 {
+ l <<= 12
+ l |= 0xFFF
+ }
+ return l
+}
+
+// Flags returns descriptor flags.
+func (d *SegmentDescriptor) Flags() SegmentDescriptorFlags {
+ return SegmentDescriptorFlags(d.bits[1] & 0x00F09F00)
+}
+
+// DPL returns the descriptor privilege level.
+func (d *SegmentDescriptor) DPL() int {
+ return int((d.bits[1] >> 13) & 3)
+}
+
+func (d *SegmentDescriptor) setNull() {
+ d.bits[0] = 0
+ d.bits[1] = 0
+}
+
+func (d *SegmentDescriptor) set(base, limit uint32, dpl int, flags SegmentDescriptorFlags) {
+ flags |= SegmentDescriptorPresent
+ if limit>>12 != 0 {
+ limit >>= 12
+ flags |= SegmentDescriptorG
+ }
+ d.bits[0] = base<<16 | limit&0xFFFF
+ d.bits[1] = base&0xFF000000 | (base>>16)&0xFF | limit&0x000F0000 | uint32(flags) | uint32(dpl)<<13
+}
+
+func (d *SegmentDescriptor) setCode32(base, limit uint32, dpl int) {
+ d.set(base, limit, dpl,
+ SegmentDescriptorDB|
+ SegmentDescriptorExecute|
+ SegmentDescriptorSystem)
+}
+
+func (d *SegmentDescriptor) setCode64(base, limit uint32, dpl int) {
+ d.set(base, limit, dpl,
+ SegmentDescriptorG|
+ SegmentDescriptorLong|
+ SegmentDescriptorExecute|
+ SegmentDescriptorSystem)
+}
+
+func (d *SegmentDescriptor) setData(base, limit uint32, dpl int) {
+ d.set(base, limit, dpl,
+ SegmentDescriptorWrite|
+ SegmentDescriptorSystem)
+}
+
+// setHi is only used for the TSS segment, which is magically 64-bits.
+func (d *SegmentDescriptor) setHi(base uint32) {
+ d.bits[0] = base
+ d.bits[1] = 0
+}
+
+// Gate64 is a 64-bit task, trap, or interrupt gate.
+type Gate64 struct {
+ bits [4]uint32
+}
+
+// idt64 is a 64-bit interrupt descriptor table.
+type idt64 [_NR_INTERRUPTS]Gate64
+
+func (g *Gate64) setInterrupt(cs Selector, rip uint64, dpl int, ist int) {
+ g.bits[0] = uint32(cs)<<16 | uint32(rip)&0xFFFF
+ g.bits[1] = uint32(rip)&0xFFFF0000 | SegmentDescriptorPresent | uint32(dpl)<<13 | 14<<8 | uint32(ist)&0x7
+ g.bits[2] = uint32(rip >> 32)
+}
+
+func (g *Gate64) setTrap(cs Selector, rip uint64, dpl int, ist int) {
+ g.setInterrupt(cs, rip, dpl, ist)
+ g.bits[1] |= 1 << 8
+}
+
+// TaskState64 is a 64-bit task state structure.
+type TaskState64 struct {
+ _ uint32
+ rsp0Lo, rsp0Hi uint32
+ rsp1Lo, rsp1Hi uint32
+ rsp2Lo, rsp2Hi uint32
+ _ [2]uint32
+ ist1Lo, ist1Hi uint32
+ ist2Lo, ist2Hi uint32
+ ist3Lo, ist3Hi uint32
+ ist4Lo, ist4Hi uint32
+ ist5Lo, ist5Hi uint32
+ ist6Lo, ist6Hi uint32
+ ist7Lo, ist7Hi uint32
+ _ [2]uint32
+ _ uint16
+ ioPerm uint16
+}
diff --git a/pkg/sentry/platform/ring0/entry_amd64.s b/pkg/sentry/platform/ring0/entry_impl_amd64.s
index 02df38331..daba45f9d 100644..100755
--- a/pkg/sentry/platform/ring0/entry_amd64.s
+++ b/pkg/sentry/platform/ring0/entry_impl_amd64.s
@@ -1,3 +1,67 @@
+// build +amd64
+
+// Automatically generated, do not edit.
+
+// CPU offsets.
+#define CPU_SELF 0x00
+#define CPU_REGISTERS 0x288
+#define CPU_STACK_TOP 0x110
+#define CPU_ERROR_CODE 0x110
+#define CPU_ERROR_TYPE 0x118
+
+// Bits.
+#define _RFLAGS_IF 0x200
+#define _KERNEL_FLAGS 0x02
+
+// Vectors.
+#define DivideByZero 0x00
+#define Debug 0x01
+#define NMI 0x02
+#define Breakpoint 0x03
+#define Overflow 0x04
+#define BoundRangeExceeded 0x05
+#define InvalidOpcode 0x06
+#define DeviceNotAvailable 0x07
+#define DoubleFault 0x08
+#define CoprocessorSegmentOverrun 0x09
+#define InvalidTSS 0x0a
+#define SegmentNotPresent 0x0b
+#define StackSegmentFault 0x0c
+#define GeneralProtectionFault 0x0d
+#define PageFault 0x0e
+#define X87FloatingPointException 0x10
+#define AlignmentCheck 0x11
+#define MachineCheck 0x12
+#define SIMDFloatingPointException 0x13
+#define VirtualizationException 0x14
+#define SecurityException 0x1e
+#define SyscallInt80 0x80
+#define Syscall 0x81
+
+// Ptrace registers.
+#define PTRACE_R15 0x00
+#define PTRACE_R14 0x08
+#define PTRACE_R13 0x10
+#define PTRACE_R12 0x18
+#define PTRACE_RBP 0x20
+#define PTRACE_RBX 0x28
+#define PTRACE_R11 0x30
+#define PTRACE_R10 0x38
+#define PTRACE_R9 0x40
+#define PTRACE_R8 0x48
+#define PTRACE_RAX 0x50
+#define PTRACE_RCX 0x58
+#define PTRACE_RDX 0x60
+#define PTRACE_RSI 0x68
+#define PTRACE_RDI 0x70
+#define PTRACE_ORIGRAX 0x78
+#define PTRACE_RIP 0x80
+#define PTRACE_CS 0x88
+#define PTRACE_FLAGS 0x90
+#define PTRACE_RSP 0x98
+#define PTRACE_SS 0xa0
+#define PTRACE_FS 0xa8
+#define PTRACE_GS 0xb0
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/pkg/sentry/platform/ring0/gen_offsets/BUILD b/pkg/sentry/platform/ring0/gen_offsets/BUILD
deleted file mode 100644
index d7029d5a9..000000000
--- a/pkg/sentry/platform/ring0/gen_offsets/BUILD
+++ /dev/null
@@ -1,26 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary")
-
-package(licenses = ["notice"])
-
-load("//tools/go_generics:defs.bzl", "go_template_instance")
-
-go_template_instance(
- name = "defs_impl",
- out = "defs_impl.go",
- package = "main",
- template = "//pkg/sentry/platform/ring0:defs",
-)
-
-go_binary(
- name = "gen_offsets",
- srcs = [
- "defs_impl.go",
- "main.go",
- ],
- visibility = ["//pkg/sentry/platform/ring0:__pkg__"],
- deps = [
- "//pkg/cpuid",
- "//pkg/sentry/platform/ring0/pagetables",
- "//pkg/sentry/usermem",
- ],
-)
diff --git a/pkg/sentry/platform/ring0/gen_offsets/main.go b/pkg/sentry/platform/ring0/gen_offsets/main.go
deleted file mode 100644
index a4927da2f..000000000
--- a/pkg/sentry/platform/ring0/gen_offsets/main.go
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Binary gen_offsets is a helper for generating offset headers.
-package main
-
-import (
- "os"
-)
-
-func main() {
- Emit(os.Stdout)
-}
diff --git a/pkg/sentry/platform/ring0/offsets_amd64.go b/pkg/sentry/platform/ring0/offsets_amd64.go
deleted file mode 100644
index 85cc3fdad..000000000
--- a/pkg/sentry/platform/ring0/offsets_amd64.go
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build amd64
-
-package ring0
-
-import (
- "fmt"
- "io"
- "reflect"
- "syscall"
-)
-
-// Emit prints architecture-specific offsets.
-func Emit(w io.Writer) {
- fmt.Fprintf(w, "// Automatically generated, do not edit.\n")
-
- c := &CPU{}
- fmt.Fprintf(w, "\n// CPU offsets.\n")
- fmt.Fprintf(w, "#define CPU_SELF 0x%02x\n", reflect.ValueOf(&c.self).Pointer()-reflect.ValueOf(c).Pointer())
- fmt.Fprintf(w, "#define CPU_REGISTERS 0x%02x\n", reflect.ValueOf(&c.registers).Pointer()-reflect.ValueOf(c).Pointer())
- fmt.Fprintf(w, "#define CPU_STACK_TOP 0x%02x\n", reflect.ValueOf(&c.stack[0]).Pointer()-reflect.ValueOf(c).Pointer()+uintptr(len(c.stack)))
- fmt.Fprintf(w, "#define CPU_ERROR_CODE 0x%02x\n", reflect.ValueOf(&c.errorCode).Pointer()-reflect.ValueOf(c).Pointer())
- fmt.Fprintf(w, "#define CPU_ERROR_TYPE 0x%02x\n", reflect.ValueOf(&c.errorType).Pointer()-reflect.ValueOf(c).Pointer())
-
- fmt.Fprintf(w, "\n// Bits.\n")
- fmt.Fprintf(w, "#define _RFLAGS_IF 0x%02x\n", _RFLAGS_IF)
- fmt.Fprintf(w, "#define _KERNEL_FLAGS 0x%02x\n", KernelFlagsSet)
-
- fmt.Fprintf(w, "\n// Vectors.\n")
- fmt.Fprintf(w, "#define DivideByZero 0x%02x\n", DivideByZero)
- fmt.Fprintf(w, "#define Debug 0x%02x\n", Debug)
- fmt.Fprintf(w, "#define NMI 0x%02x\n", NMI)
- fmt.Fprintf(w, "#define Breakpoint 0x%02x\n", Breakpoint)
- fmt.Fprintf(w, "#define Overflow 0x%02x\n", Overflow)
- fmt.Fprintf(w, "#define BoundRangeExceeded 0x%02x\n", BoundRangeExceeded)
- fmt.Fprintf(w, "#define InvalidOpcode 0x%02x\n", InvalidOpcode)
- fmt.Fprintf(w, "#define DeviceNotAvailable 0x%02x\n", DeviceNotAvailable)
- fmt.Fprintf(w, "#define DoubleFault 0x%02x\n", DoubleFault)
- fmt.Fprintf(w, "#define CoprocessorSegmentOverrun 0x%02x\n", CoprocessorSegmentOverrun)
- fmt.Fprintf(w, "#define InvalidTSS 0x%02x\n", InvalidTSS)
- fmt.Fprintf(w, "#define SegmentNotPresent 0x%02x\n", SegmentNotPresent)
- fmt.Fprintf(w, "#define StackSegmentFault 0x%02x\n", StackSegmentFault)
- fmt.Fprintf(w, "#define GeneralProtectionFault 0x%02x\n", GeneralProtectionFault)
- fmt.Fprintf(w, "#define PageFault 0x%02x\n", PageFault)
- fmt.Fprintf(w, "#define X87FloatingPointException 0x%02x\n", X87FloatingPointException)
- fmt.Fprintf(w, "#define AlignmentCheck 0x%02x\n", AlignmentCheck)
- fmt.Fprintf(w, "#define MachineCheck 0x%02x\n", MachineCheck)
- fmt.Fprintf(w, "#define SIMDFloatingPointException 0x%02x\n", SIMDFloatingPointException)
- fmt.Fprintf(w, "#define VirtualizationException 0x%02x\n", VirtualizationException)
- fmt.Fprintf(w, "#define SecurityException 0x%02x\n", SecurityException)
- fmt.Fprintf(w, "#define SyscallInt80 0x%02x\n", SyscallInt80)
- fmt.Fprintf(w, "#define Syscall 0x%02x\n", Syscall)
-
- p := &syscall.PtraceRegs{}
- fmt.Fprintf(w, "\n// Ptrace registers.\n")
- fmt.Fprintf(w, "#define PTRACE_R15 0x%02x\n", reflect.ValueOf(&p.R15).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_R14 0x%02x\n", reflect.ValueOf(&p.R14).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_R13 0x%02x\n", reflect.ValueOf(&p.R13).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_R12 0x%02x\n", reflect.ValueOf(&p.R12).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RBP 0x%02x\n", reflect.ValueOf(&p.Rbp).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RBX 0x%02x\n", reflect.ValueOf(&p.Rbx).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_R11 0x%02x\n", reflect.ValueOf(&p.R11).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_R10 0x%02x\n", reflect.ValueOf(&p.R10).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_R9 0x%02x\n", reflect.ValueOf(&p.R9).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_R8 0x%02x\n", reflect.ValueOf(&p.R8).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RAX 0x%02x\n", reflect.ValueOf(&p.Rax).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RCX 0x%02x\n", reflect.ValueOf(&p.Rcx).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RDX 0x%02x\n", reflect.ValueOf(&p.Rdx).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RSI 0x%02x\n", reflect.ValueOf(&p.Rsi).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RDI 0x%02x\n", reflect.ValueOf(&p.Rdi).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_ORIGRAX 0x%02x\n", reflect.ValueOf(&p.Orig_rax).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RIP 0x%02x\n", reflect.ValueOf(&p.Rip).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_CS 0x%02x\n", reflect.ValueOf(&p.Cs).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_FLAGS 0x%02x\n", reflect.ValueOf(&p.Eflags).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_RSP 0x%02x\n", reflect.ValueOf(&p.Rsp).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_SS 0x%02x\n", reflect.ValueOf(&p.Ss).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_FS 0x%02x\n", reflect.ValueOf(&p.Fs_base).Pointer()-reflect.ValueOf(p).Pointer())
- fmt.Fprintf(w, "#define PTRACE_GS 0x%02x\n", reflect.ValueOf(&p.Gs_base).Pointer()-reflect.ValueOf(p).Pointer())
-}
diff --git a/pkg/sentry/platform/ring0/pagetables/BUILD b/pkg/sentry/platform/ring0/pagetables/BUILD
deleted file mode 100644
index 3b95af617..000000000
--- a/pkg/sentry/platform/ring0/pagetables/BUILD
+++ /dev/null
@@ -1,105 +0,0 @@
-load("//tools/go_stateify:defs.bzl", "go_library", "go_test")
-
-package(licenses = ["notice"])
-
-load("//tools/go_generics:defs.bzl", "go_template", "go_template_instance")
-
-go_template(
- name = "generic_walker",
- srcs = [
- "walker_amd64.go",
- ],
- opt_types = [
- "Visitor",
- ],
- visibility = [":__pkg__"],
-)
-
-go_template_instance(
- name = "walker_map",
- out = "walker_map.go",
- package = "pagetables",
- prefix = "map",
- template = ":generic_walker",
- types = {
- "Visitor": "mapVisitor",
- },
-)
-
-go_template_instance(
- name = "walker_unmap",
- out = "walker_unmap.go",
- package = "pagetables",
- prefix = "unmap",
- template = ":generic_walker",
- types = {
- "Visitor": "unmapVisitor",
- },
-)
-
-go_template_instance(
- name = "walker_lookup",
- out = "walker_lookup.go",
- package = "pagetables",
- prefix = "lookup",
- template = ":generic_walker",
- types = {
- "Visitor": "lookupVisitor",
- },
-)
-
-go_template_instance(
- name = "walker_empty",
- out = "walker_empty.go",
- package = "pagetables",
- prefix = "empty",
- template = ":generic_walker",
- types = {
- "Visitor": "emptyVisitor",
- },
-)
-
-go_template_instance(
- name = "walker_check",
- out = "walker_check.go",
- package = "pagetables",
- prefix = "check",
- template = ":generic_walker",
- types = {
- "Visitor": "checkVisitor",
- },
-)
-
-go_library(
- name = "pagetables",
- srcs = [
- "allocator.go",
- "allocator_unsafe.go",
- "pagetables.go",
- "pagetables_amd64.go",
- "pagetables_x86.go",
- "pcids_x86.go",
- "walker_empty.go",
- "walker_lookup.go",
- "walker_map.go",
- "walker_unmap.go",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables",
- visibility = [
- "//pkg/sentry/platform/kvm:__subpackages__",
- "//pkg/sentry/platform/ring0:__subpackages__",
- ],
- deps = ["//pkg/sentry/usermem"],
-)
-
-go_test(
- name = "pagetables_test",
- size = "small",
- srcs = [
- "pagetables_amd64_test.go",
- "pagetables_test.go",
- "walker_check.go",
- ],
- embed = [":pagetables"],
- deps = ["//pkg/sentry/usermem"],
-)
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_test.go b/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_test.go
deleted file mode 100644
index 35e917526..000000000
--- a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_test.go
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build amd64
-
-package pagetables
-
-import (
- "testing"
-
- "gvisor.dev/gvisor/pkg/sentry/usermem"
-)
-
-func Test2MAnd4K(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map a small page and a huge page.
- pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
- pt.Map(0x00007f0000000000, pmdSize, MapOpts{AccessType: usermem.Read}, pmdSize*47)
-
- checkMappings(t, pt, []mapping{
- {0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
- {0x00007f0000000000, pmdSize, pmdSize * 47, MapOpts{AccessType: usermem.Read}},
- })
-}
-
-func Test1GAnd4K(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map a small page and a super page.
- pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
- pt.Map(0x00007f0000000000, pudSize, MapOpts{AccessType: usermem.Read}, pudSize*47)
-
- checkMappings(t, pt, []mapping{
- {0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
- {0x00007f0000000000, pudSize, pudSize * 47, MapOpts{AccessType: usermem.Read}},
- })
-}
-
-func TestSplit1GPage(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map a super page and knock out the middle.
- pt.Map(0x00007f0000000000, pudSize, MapOpts{AccessType: usermem.Read}, pudSize*42)
- pt.Unmap(usermem.Addr(0x00007f0000000000+pteSize), pudSize-(2*pteSize))
-
- checkMappings(t, pt, []mapping{
- {0x00007f0000000000, pteSize, pudSize * 42, MapOpts{AccessType: usermem.Read}},
- {0x00007f0000000000 + pudSize - pteSize, pteSize, pudSize*42 + pudSize - pteSize, MapOpts{AccessType: usermem.Read}},
- })
-}
-
-func TestSplit2MPage(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map a huge page and knock out the middle.
- pt.Map(0x00007f0000000000, pmdSize, MapOpts{AccessType: usermem.Read}, pmdSize*42)
- pt.Unmap(usermem.Addr(0x00007f0000000000+pteSize), pmdSize-(2*pteSize))
-
- checkMappings(t, pt, []mapping{
- {0x00007f0000000000, pteSize, pmdSize * 42, MapOpts{AccessType: usermem.Read}},
- {0x00007f0000000000 + pmdSize - pteSize, pteSize, pmdSize*42 + pmdSize - pteSize, MapOpts{AccessType: usermem.Read}},
- })
-}
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_state_autogen.go b/pkg/sentry/platform/ring0/pagetables/pagetables_state_autogen.go
new file mode 100755
index 000000000..ac1ccf3d3
--- /dev/null
+++ b/pkg/sentry/platform/ring0/pagetables/pagetables_state_autogen.go
@@ -0,0 +1,4 @@
+// automatically generated by stateify.
+
+package pagetables
+
diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_test.go b/pkg/sentry/platform/ring0/pagetables/pagetables_test.go
deleted file mode 100644
index 6e95ad2b9..000000000
--- a/pkg/sentry/platform/ring0/pagetables/pagetables_test.go
+++ /dev/null
@@ -1,156 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pagetables
-
-import (
- "testing"
-
- "gvisor.dev/gvisor/pkg/sentry/usermem"
-)
-
-type mapping struct {
- start uintptr
- length uintptr
- addr uintptr
- opts MapOpts
-}
-
-type checkVisitor struct {
- expected []mapping // Input.
- current int // Temporary.
- found []mapping // Output.
- failed string // Output.
-}
-
-func (v *checkVisitor) visit(start uintptr, pte *PTE, align uintptr) {
- v.found = append(v.found, mapping{
- start: start,
- length: align + 1,
- addr: pte.Address(),
- opts: pte.Opts(),
- })
- if v.failed != "" {
- // Don't keep looking for errors.
- return
- }
-
- if v.current >= len(v.expected) {
- v.failed = "more mappings than expected"
- } else if v.expected[v.current].start != start {
- v.failed = "start didn't match expected"
- } else if v.expected[v.current].length != (align + 1) {
- v.failed = "end didn't match expected"
- } else if v.expected[v.current].addr != pte.Address() {
- v.failed = "address didn't match expected"
- } else if v.expected[v.current].opts != pte.Opts() {
- v.failed = "opts didn't match"
- }
- v.current++
-}
-
-func (*checkVisitor) requiresAlloc() bool { return false }
-
-func (*checkVisitor) requiresSplit() bool { return false }
-
-func checkMappings(t *testing.T, pt *PageTables, m []mapping) {
- // Iterate over all the mappings.
- w := checkWalker{
- pageTables: pt,
- visitor: checkVisitor{
- expected: m,
- },
- }
- w.iterateRange(0, ^uintptr(0))
-
- // Were we expected additional mappings?
- if w.visitor.failed == "" && w.visitor.current != len(w.visitor.expected) {
- w.visitor.failed = "insufficient mappings found"
- }
-
- // Emit a meaningful error message on failure.
- if w.visitor.failed != "" {
- t.Errorf("%s; got %#v, wanted %#v", w.visitor.failed, w.visitor.found, w.visitor.expected)
- }
-}
-
-func TestUnmap(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map and unmap one entry.
- pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
- pt.Unmap(0x400000, pteSize)
-
- checkMappings(t, pt, nil)
-}
-
-func TestReadOnly(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map one entry.
- pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.Read}, pteSize*42)
-
- checkMappings(t, pt, []mapping{
- {0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.Read}},
- })
-}
-
-func TestReadWrite(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map one entry.
- pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
-
- checkMappings(t, pt, []mapping{
- {0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
- })
-}
-
-func TestSerialEntries(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map two sequential entries.
- pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
- pt.Map(0x401000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*47)
-
- checkMappings(t, pt, []mapping{
- {0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
- {0x401000, pteSize, pteSize * 47, MapOpts{AccessType: usermem.ReadWrite}},
- })
-}
-
-func TestSpanningEntries(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Span a pgd with two pages.
- pt.Map(0x00007efffffff000, 2*pteSize, MapOpts{AccessType: usermem.Read}, pteSize*42)
-
- checkMappings(t, pt, []mapping{
- {0x00007efffffff000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.Read}},
- {0x00007f0000000000, pteSize, pteSize * 43, MapOpts{AccessType: usermem.Read}},
- })
-}
-
-func TestSparseEntries(t *testing.T) {
- pt := New(NewRuntimeAllocator())
-
- // Map two entries in different pgds.
- pt.Map(0x400000, pteSize, MapOpts{AccessType: usermem.ReadWrite}, pteSize*42)
- pt.Map(0x00007f0000000000, pteSize, MapOpts{AccessType: usermem.Read}, pteSize*47)
-
- checkMappings(t, pt, []mapping{
- {0x400000, pteSize, pteSize * 42, MapOpts{AccessType: usermem.ReadWrite}},
- {0x00007f0000000000, pteSize, pteSize * 47, MapOpts{AccessType: usermem.Read}},
- })
-}
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_amd64.go b/pkg/sentry/platform/ring0/pagetables/walker_empty.go
index 8f9dacd93..417784e17 100644..100755
--- a/pkg/sentry/platform/ring0/pagetables/walker_amd64.go
+++ b/pkg/sentry/platform/ring0/pagetables/walker_empty.go
@@ -1,42 +1,12 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build amd64
-
package pagetables
-// Visitor is a generic type.
-type Visitor interface {
- // visit is called on each PTE.
- visit(start uintptr, pte *PTE, align uintptr)
-
- // requiresAlloc indicates that new entries should be allocated within
- // the walked range.
- requiresAlloc() bool
-
- // requiresSplit indicates that entries in the given range should be
- // split if they are huge or jumbo pages.
- requiresSplit() bool
-}
-
// Walker walks page tables.
-type Walker struct {
+type emptyWalker struct {
// pageTables are the tables to walk.
pageTables *PageTables
// Visitor is the set of arguments.
- visitor Visitor
+ visitor emptyVisitor
}
// iterateRange iterates over all appropriate levels of page tables for the given range.
@@ -63,7 +33,7 @@ type Walker struct {
// non-canonical ranges. If they do, a panic will result.
//
//go:nosplit
-func (w *Walker) iterateRange(start, end uintptr) {
+func (w *emptyWalker) iterateRange(start, end uintptr) {
if start%pteSize != 0 {
panic("unaligned start")
}
@@ -104,7 +74,7 @@ func (w *Walker) iterateRange(start, end uintptr) {
// next returns the next address quantized by the given size.
//
//go:nosplit
-func next(start uintptr, size uintptr) uintptr {
+func emptynext(start uintptr, size uintptr) uintptr {
start &= ^(size - 1)
start += size
return start
@@ -113,7 +83,7 @@ func next(start uintptr, size uintptr) uintptr {
// iterateRangeCanonical walks a canonical range.
//
//go:nosplit
-func (w *Walker) iterateRangeCanonical(start, end uintptr) {
+func (w *emptyWalker) iterateRangeCanonical(start, end uintptr) {
for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
var (
pgdEntry = &w.pageTables.root[pgdIndex]
@@ -121,19 +91,17 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
)
if !pgdEntry.Valid() {
if !w.visitor.requiresAlloc() {
- // Skip over this entry.
- start = next(start, pgdSize)
+
+ start = emptynext(start, pgdSize)
continue
}
- // Allocate a new pgd.
pudEntries = w.pageTables.Allocator.NewPTEs()
pgdEntry.setPageTable(w.pageTables, pudEntries)
} else {
pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
}
- // Map the next level.
clearPUDEntries := uint16(0)
for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
@@ -143,33 +111,28 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
)
if !pudEntry.Valid() {
if !w.visitor.requiresAlloc() {
- // Skip over this entry.
+
clearPUDEntries++
- start = next(start, pudSize)
+ start = emptynext(start, pudSize)
continue
}
- // This level has 1-GB super pages. Is this
- // entire region at least as large as a single
- // PUD entry? If so, we can skip allocating a
- // new page for the pmd.
if start&(pudSize-1) == 0 && end-start >= pudSize {
pudEntry.SetSuper()
w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
if pudEntry.Valid() {
- start = next(start, pudSize)
+ start = emptynext(start, pudSize)
continue
}
}
- // Allocate a new pud.
pmdEntries = w.pageTables.Allocator.NewPTEs()
pudEntry.setPageTable(w.pageTables, pmdEntries)
} else if pudEntry.IsSuper() {
- // Does this page need to be split?
- if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < next(start, pudSize)) {
- // Install the relevant entries.
+
+ if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < emptynext(start, pudSize)) {
+
pmdEntries = w.pageTables.Allocator.NewPTEs()
for index := uint16(0); index < entriesPerPage; index++ {
pmdEntries[index].SetSuper()
@@ -179,23 +142,20 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
}
pudEntry.setPageTable(w.pageTables, pmdEntries)
} else {
- // A super page to be checked directly.
+
w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
- // Might have been cleared.
if !pudEntry.Valid() {
clearPUDEntries++
}
- // Note that the super page was changed.
- start = next(start, pudSize)
+ start = emptynext(start, pudSize)
continue
}
} else {
pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
}
- // Map the next level, since this is valid.
clearPMDEntries := uint16(0)
for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
@@ -205,32 +165,28 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
)
if !pmdEntry.Valid() {
if !w.visitor.requiresAlloc() {
- // Skip over this entry.
+
clearPMDEntries++
- start = next(start, pmdSize)
+ start = emptynext(start, pmdSize)
continue
}
- // This level has 2-MB huge pages. If this
- // region is contined in a single PMD entry?
- // As above, we can skip allocating a new page.
if start&(pmdSize-1) == 0 && end-start >= pmdSize {
pmdEntry.SetSuper()
w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
if pmdEntry.Valid() {
- start = next(start, pmdSize)
+ start = emptynext(start, pmdSize)
continue
}
}
- // Allocate a new pmd.
pteEntries = w.pageTables.Allocator.NewPTEs()
pmdEntry.setPageTable(w.pageTables, pteEntries)
} else if pmdEntry.IsSuper() {
- // Does this page need to be split?
- if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < next(start, pmdSize)) {
- // Install the relevant entries.
+
+ if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < emptynext(start, pmdSize)) {
+
pteEntries = w.pageTables.Allocator.NewPTEs()
for index := uint16(0); index < entriesPerPage; index++ {
pteEntries[index].Set(
@@ -239,23 +195,20 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
}
pmdEntry.setPageTable(w.pageTables, pteEntries)
} else {
- // A huge page to be checked directly.
+
w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
- // Might have been cleared.
if !pmdEntry.Valid() {
clearPMDEntries++
}
- // Note that the huge page was changed.
- start = next(start, pmdSize)
+ start = emptynext(start, pmdSize)
continue
}
} else {
pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
}
- // Map the next level, since this is valid.
clearPTEEntries := uint16(0)
for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
@@ -268,7 +221,6 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
continue
}
- // At this point, we are guaranteed that start%pteSize == 0.
w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
if !pteEntry.Valid() {
if w.visitor.requiresAlloc() {
@@ -277,12 +229,10 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
clearPTEEntries++
}
- // Note that the pte was changed.
start += pteSize
continue
}
- // Check if we no longer need this page.
if clearPTEEntries == entriesPerPage {
pmdEntry.Clear()
w.pageTables.Allocator.FreePTEs(pteEntries)
@@ -290,7 +240,6 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
}
}
- // Check if we no longer need this page.
if clearPMDEntries == entriesPerPage {
pudEntry.Clear()
w.pageTables.Allocator.FreePTEs(pmdEntries)
@@ -298,7 +247,6 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) {
}
}
- // Check if we no longer need this page.
if clearPUDEntries == entriesPerPage {
pgdEntry.Clear()
w.pageTables.Allocator.FreePTEs(pudEntries)
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_lookup.go b/pkg/sentry/platform/ring0/pagetables/walker_lookup.go
new file mode 100755
index 000000000..906c9c50f
--- /dev/null
+++ b/pkg/sentry/platform/ring0/pagetables/walker_lookup.go
@@ -0,0 +1,255 @@
+package pagetables
+
+// Walker walks page tables.
+type lookupWalker struct {
+ // pageTables are the tables to walk.
+ pageTables *PageTables
+
+ // Visitor is the set of arguments.
+ visitor lookupVisitor
+}
+
+// iterateRange iterates over all appropriate levels of page tables for the given range.
+//
+// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The
+// exception is super pages. If a valid super page (huge or jumbo) cannot be
+// installed, then the walk will continue to individual entries.
+//
+// This algorithm will attempt to maximize the use of super pages whenever
+// possible. Whether a super page is provided will be clear through the range
+// provided in the callback.
+//
+// Note that if requiresAlloc is true, then no gaps will be present. However,
+// if alloc is not set, then the iteration will likely be full of gaps.
+//
+// Note that this function should generally be avoided in favor of Map, Unmap,
+// etc. when not necessary.
+//
+// Precondition: start must be page-aligned.
+//
+// Precondition: start must be less than end.
+//
+// Precondition: If requiresAlloc is true, then start and end should not span
+// non-canonical ranges. If they do, a panic will result.
+//
+//go:nosplit
+func (w *lookupWalker) iterateRange(start, end uintptr) {
+ if start%pteSize != 0 {
+ panic("unaligned start")
+ }
+ if end < start {
+ panic("start > end")
+ }
+ if start < lowerTop {
+ if end <= lowerTop {
+ w.iterateRangeCanonical(start, end)
+ } else if end > lowerTop && end <= upperBottom {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(start, lowerTop)
+ } else {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(start, lowerTop)
+ w.iterateRangeCanonical(upperBottom, end)
+ }
+ } else if start < upperBottom {
+ if end <= upperBottom {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ } else {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(upperBottom, end)
+ }
+ } else {
+ w.iterateRangeCanonical(start, end)
+ }
+}
+
+// next returns the next address quantized by the given size.
+//
+//go:nosplit
+func lookupnext(start uintptr, size uintptr) uintptr {
+ start &= ^(size - 1)
+ start += size
+ return start
+}
+
+// iterateRangeCanonical walks a canonical range.
+//
+//go:nosplit
+func (w *lookupWalker) iterateRangeCanonical(start, end uintptr) {
+ for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
+ var (
+ pgdEntry = &w.pageTables.root[pgdIndex]
+ pudEntries *PTEs
+ )
+ if !pgdEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ start = lookupnext(start, pgdSize)
+ continue
+ }
+
+ pudEntries = w.pageTables.Allocator.NewPTEs()
+ pgdEntry.setPageTable(w.pageTables, pudEntries)
+ } else {
+ pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
+ }
+
+ clearPUDEntries := uint16(0)
+
+ for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
+ var (
+ pudEntry = &pudEntries[pudIndex]
+ pmdEntries *PTEs
+ )
+ if !pudEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ clearPUDEntries++
+ start = lookupnext(start, pudSize)
+ continue
+ }
+
+ if start&(pudSize-1) == 0 && end-start >= pudSize {
+ pudEntry.SetSuper()
+ w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
+ if pudEntry.Valid() {
+ start = lookupnext(start, pudSize)
+ continue
+ }
+ }
+
+ pmdEntries = w.pageTables.Allocator.NewPTEs()
+ pudEntry.setPageTable(w.pageTables, pmdEntries)
+
+ } else if pudEntry.IsSuper() {
+
+ if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < lookupnext(start, pudSize)) {
+
+ pmdEntries = w.pageTables.Allocator.NewPTEs()
+ for index := uint16(0); index < entriesPerPage; index++ {
+ pmdEntries[index].SetSuper()
+ pmdEntries[index].Set(
+ pudEntry.Address()+(pmdSize*uintptr(index)),
+ pudEntry.Opts())
+ }
+ pudEntry.setPageTable(w.pageTables, pmdEntries)
+ } else {
+
+ w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
+
+ if !pudEntry.Valid() {
+ clearPUDEntries++
+ }
+
+ start = lookupnext(start, pudSize)
+ continue
+ }
+ } else {
+ pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
+ }
+
+ clearPMDEntries := uint16(0)
+
+ for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
+ var (
+ pmdEntry = &pmdEntries[pmdIndex]
+ pteEntries *PTEs
+ )
+ if !pmdEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ clearPMDEntries++
+ start = lookupnext(start, pmdSize)
+ continue
+ }
+
+ if start&(pmdSize-1) == 0 && end-start >= pmdSize {
+ pmdEntry.SetSuper()
+ w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
+ if pmdEntry.Valid() {
+ start = lookupnext(start, pmdSize)
+ continue
+ }
+ }
+
+ pteEntries = w.pageTables.Allocator.NewPTEs()
+ pmdEntry.setPageTable(w.pageTables, pteEntries)
+
+ } else if pmdEntry.IsSuper() {
+
+ if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < lookupnext(start, pmdSize)) {
+
+ pteEntries = w.pageTables.Allocator.NewPTEs()
+ for index := uint16(0); index < entriesPerPage; index++ {
+ pteEntries[index].Set(
+ pmdEntry.Address()+(pteSize*uintptr(index)),
+ pmdEntry.Opts())
+ }
+ pmdEntry.setPageTable(w.pageTables, pteEntries)
+ } else {
+
+ w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
+
+ if !pmdEntry.Valid() {
+ clearPMDEntries++
+ }
+
+ start = lookupnext(start, pmdSize)
+ continue
+ }
+ } else {
+ pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
+ }
+
+ clearPTEEntries := uint16(0)
+
+ for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
+ var (
+ pteEntry = &pteEntries[pteIndex]
+ )
+ if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
+ clearPTEEntries++
+ start += pteSize
+ continue
+ }
+
+ w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
+ if !pteEntry.Valid() {
+ if w.visitor.requiresAlloc() {
+ panic("PTE not set after iteration with requiresAlloc!")
+ }
+ clearPTEEntries++
+ }
+
+ start += pteSize
+ continue
+ }
+
+ if clearPTEEntries == entriesPerPage {
+ pmdEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pteEntries)
+ clearPMDEntries++
+ }
+ }
+
+ if clearPMDEntries == entriesPerPage {
+ pudEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pmdEntries)
+ clearPUDEntries++
+ }
+ }
+
+ if clearPUDEntries == entriesPerPage {
+ pgdEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pudEntries)
+ }
+ }
+}
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_map.go b/pkg/sentry/platform/ring0/pagetables/walker_map.go
new file mode 100755
index 000000000..61ee3c825
--- /dev/null
+++ b/pkg/sentry/platform/ring0/pagetables/walker_map.go
@@ -0,0 +1,255 @@
+package pagetables
+
+// Walker walks page tables.
+type mapWalker struct {
+ // pageTables are the tables to walk.
+ pageTables *PageTables
+
+ // Visitor is the set of arguments.
+ visitor mapVisitor
+}
+
+// iterateRange iterates over all appropriate levels of page tables for the given range.
+//
+// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The
+// exception is super pages. If a valid super page (huge or jumbo) cannot be
+// installed, then the walk will continue to individual entries.
+//
+// This algorithm will attempt to maximize the use of super pages whenever
+// possible. Whether a super page is provided will be clear through the range
+// provided in the callback.
+//
+// Note that if requiresAlloc is true, then no gaps will be present. However,
+// if alloc is not set, then the iteration will likely be full of gaps.
+//
+// Note that this function should generally be avoided in favor of Map, Unmap,
+// etc. when not necessary.
+//
+// Precondition: start must be page-aligned.
+//
+// Precondition: start must be less than end.
+//
+// Precondition: If requiresAlloc is true, then start and end should not span
+// non-canonical ranges. If they do, a panic will result.
+//
+//go:nosplit
+func (w *mapWalker) iterateRange(start, end uintptr) {
+ if start%pteSize != 0 {
+ panic("unaligned start")
+ }
+ if end < start {
+ panic("start > end")
+ }
+ if start < lowerTop {
+ if end <= lowerTop {
+ w.iterateRangeCanonical(start, end)
+ } else if end > lowerTop && end <= upperBottom {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(start, lowerTop)
+ } else {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(start, lowerTop)
+ w.iterateRangeCanonical(upperBottom, end)
+ }
+ } else if start < upperBottom {
+ if end <= upperBottom {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ } else {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(upperBottom, end)
+ }
+ } else {
+ w.iterateRangeCanonical(start, end)
+ }
+}
+
+// next returns the next address quantized by the given size.
+//
+//go:nosplit
+func mapnext(start uintptr, size uintptr) uintptr {
+ start &= ^(size - 1)
+ start += size
+ return start
+}
+
+// iterateRangeCanonical walks a canonical range.
+//
+//go:nosplit
+func (w *mapWalker) iterateRangeCanonical(start, end uintptr) {
+ for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
+ var (
+ pgdEntry = &w.pageTables.root[pgdIndex]
+ pudEntries *PTEs
+ )
+ if !pgdEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ start = mapnext(start, pgdSize)
+ continue
+ }
+
+ pudEntries = w.pageTables.Allocator.NewPTEs()
+ pgdEntry.setPageTable(w.pageTables, pudEntries)
+ } else {
+ pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
+ }
+
+ clearPUDEntries := uint16(0)
+
+ for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
+ var (
+ pudEntry = &pudEntries[pudIndex]
+ pmdEntries *PTEs
+ )
+ if !pudEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ clearPUDEntries++
+ start = mapnext(start, pudSize)
+ continue
+ }
+
+ if start&(pudSize-1) == 0 && end-start >= pudSize {
+ pudEntry.SetSuper()
+ w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
+ if pudEntry.Valid() {
+ start = mapnext(start, pudSize)
+ continue
+ }
+ }
+
+ pmdEntries = w.pageTables.Allocator.NewPTEs()
+ pudEntry.setPageTable(w.pageTables, pmdEntries)
+
+ } else if pudEntry.IsSuper() {
+
+ if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < mapnext(start, pudSize)) {
+
+ pmdEntries = w.pageTables.Allocator.NewPTEs()
+ for index := uint16(0); index < entriesPerPage; index++ {
+ pmdEntries[index].SetSuper()
+ pmdEntries[index].Set(
+ pudEntry.Address()+(pmdSize*uintptr(index)),
+ pudEntry.Opts())
+ }
+ pudEntry.setPageTable(w.pageTables, pmdEntries)
+ } else {
+
+ w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
+
+ if !pudEntry.Valid() {
+ clearPUDEntries++
+ }
+
+ start = mapnext(start, pudSize)
+ continue
+ }
+ } else {
+ pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
+ }
+
+ clearPMDEntries := uint16(0)
+
+ for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
+ var (
+ pmdEntry = &pmdEntries[pmdIndex]
+ pteEntries *PTEs
+ )
+ if !pmdEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ clearPMDEntries++
+ start = mapnext(start, pmdSize)
+ continue
+ }
+
+ if start&(pmdSize-1) == 0 && end-start >= pmdSize {
+ pmdEntry.SetSuper()
+ w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
+ if pmdEntry.Valid() {
+ start = mapnext(start, pmdSize)
+ continue
+ }
+ }
+
+ pteEntries = w.pageTables.Allocator.NewPTEs()
+ pmdEntry.setPageTable(w.pageTables, pteEntries)
+
+ } else if pmdEntry.IsSuper() {
+
+ if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < mapnext(start, pmdSize)) {
+
+ pteEntries = w.pageTables.Allocator.NewPTEs()
+ for index := uint16(0); index < entriesPerPage; index++ {
+ pteEntries[index].Set(
+ pmdEntry.Address()+(pteSize*uintptr(index)),
+ pmdEntry.Opts())
+ }
+ pmdEntry.setPageTable(w.pageTables, pteEntries)
+ } else {
+
+ w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
+
+ if !pmdEntry.Valid() {
+ clearPMDEntries++
+ }
+
+ start = mapnext(start, pmdSize)
+ continue
+ }
+ } else {
+ pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
+ }
+
+ clearPTEEntries := uint16(0)
+
+ for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
+ var (
+ pteEntry = &pteEntries[pteIndex]
+ )
+ if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
+ clearPTEEntries++
+ start += pteSize
+ continue
+ }
+
+ w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
+ if !pteEntry.Valid() {
+ if w.visitor.requiresAlloc() {
+ panic("PTE not set after iteration with requiresAlloc!")
+ }
+ clearPTEEntries++
+ }
+
+ start += pteSize
+ continue
+ }
+
+ if clearPTEEntries == entriesPerPage {
+ pmdEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pteEntries)
+ clearPMDEntries++
+ }
+ }
+
+ if clearPMDEntries == entriesPerPage {
+ pudEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pmdEntries)
+ clearPUDEntries++
+ }
+ }
+
+ if clearPUDEntries == entriesPerPage {
+ pgdEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pudEntries)
+ }
+ }
+}
diff --git a/pkg/sentry/platform/ring0/pagetables/walker_unmap.go b/pkg/sentry/platform/ring0/pagetables/walker_unmap.go
new file mode 100755
index 000000000..be2aa0ce4
--- /dev/null
+++ b/pkg/sentry/platform/ring0/pagetables/walker_unmap.go
@@ -0,0 +1,255 @@
+package pagetables
+
+// Walker walks page tables.
+type unmapWalker struct {
+ // pageTables are the tables to walk.
+ pageTables *PageTables
+
+ // Visitor is the set of arguments.
+ visitor unmapVisitor
+}
+
+// iterateRange iterates over all appropriate levels of page tables for the given range.
+//
+// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The
+// exception is super pages. If a valid super page (huge or jumbo) cannot be
+// installed, then the walk will continue to individual entries.
+//
+// This algorithm will attempt to maximize the use of super pages whenever
+// possible. Whether a super page is provided will be clear through the range
+// provided in the callback.
+//
+// Note that if requiresAlloc is true, then no gaps will be present. However,
+// if alloc is not set, then the iteration will likely be full of gaps.
+//
+// Note that this function should generally be avoided in favor of Map, Unmap,
+// etc. when not necessary.
+//
+// Precondition: start must be page-aligned.
+//
+// Precondition: start must be less than end.
+//
+// Precondition: If requiresAlloc is true, then start and end should not span
+// non-canonical ranges. If they do, a panic will result.
+//
+//go:nosplit
+func (w *unmapWalker) iterateRange(start, end uintptr) {
+ if start%pteSize != 0 {
+ panic("unaligned start")
+ }
+ if end < start {
+ panic("start > end")
+ }
+ if start < lowerTop {
+ if end <= lowerTop {
+ w.iterateRangeCanonical(start, end)
+ } else if end > lowerTop && end <= upperBottom {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(start, lowerTop)
+ } else {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(start, lowerTop)
+ w.iterateRangeCanonical(upperBottom, end)
+ }
+ } else if start < upperBottom {
+ if end <= upperBottom {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ } else {
+ if w.visitor.requiresAlloc() {
+ panic("alloc spans non-canonical range")
+ }
+ w.iterateRangeCanonical(upperBottom, end)
+ }
+ } else {
+ w.iterateRangeCanonical(start, end)
+ }
+}
+
+// next returns the next address quantized by the given size.
+//
+//go:nosplit
+func unmapnext(start uintptr, size uintptr) uintptr {
+ start &= ^(size - 1)
+ start += size
+ return start
+}
+
+// iterateRangeCanonical walks a canonical range.
+//
+//go:nosplit
+func (w *unmapWalker) iterateRangeCanonical(start, end uintptr) {
+ for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
+ var (
+ pgdEntry = &w.pageTables.root[pgdIndex]
+ pudEntries *PTEs
+ )
+ if !pgdEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ start = unmapnext(start, pgdSize)
+ continue
+ }
+
+ pudEntries = w.pageTables.Allocator.NewPTEs()
+ pgdEntry.setPageTable(w.pageTables, pudEntries)
+ } else {
+ pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
+ }
+
+ clearPUDEntries := uint16(0)
+
+ for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ {
+ var (
+ pudEntry = &pudEntries[pudIndex]
+ pmdEntries *PTEs
+ )
+ if !pudEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ clearPUDEntries++
+ start = unmapnext(start, pudSize)
+ continue
+ }
+
+ if start&(pudSize-1) == 0 && end-start >= pudSize {
+ pudEntry.SetSuper()
+ w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
+ if pudEntry.Valid() {
+ start = unmapnext(start, pudSize)
+ continue
+ }
+ }
+
+ pmdEntries = w.pageTables.Allocator.NewPTEs()
+ pudEntry.setPageTable(w.pageTables, pmdEntries)
+
+ } else if pudEntry.IsSuper() {
+
+ if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < unmapnext(start, pudSize)) {
+
+ pmdEntries = w.pageTables.Allocator.NewPTEs()
+ for index := uint16(0); index < entriesPerPage; index++ {
+ pmdEntries[index].SetSuper()
+ pmdEntries[index].Set(
+ pudEntry.Address()+(pmdSize*uintptr(index)),
+ pudEntry.Opts())
+ }
+ pudEntry.setPageTable(w.pageTables, pmdEntries)
+ } else {
+
+ w.visitor.visit(uintptr(start), pudEntry, pudSize-1)
+
+ if !pudEntry.Valid() {
+ clearPUDEntries++
+ }
+
+ start = unmapnext(start, pudSize)
+ continue
+ }
+ } else {
+ pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address())
+ }
+
+ clearPMDEntries := uint16(0)
+
+ for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
+ var (
+ pmdEntry = &pmdEntries[pmdIndex]
+ pteEntries *PTEs
+ )
+ if !pmdEntry.Valid() {
+ if !w.visitor.requiresAlloc() {
+
+ clearPMDEntries++
+ start = unmapnext(start, pmdSize)
+ continue
+ }
+
+ if start&(pmdSize-1) == 0 && end-start >= pmdSize {
+ pmdEntry.SetSuper()
+ w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
+ if pmdEntry.Valid() {
+ start = unmapnext(start, pmdSize)
+ continue
+ }
+ }
+
+ pteEntries = w.pageTables.Allocator.NewPTEs()
+ pmdEntry.setPageTable(w.pageTables, pteEntries)
+
+ } else if pmdEntry.IsSuper() {
+
+ if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < unmapnext(start, pmdSize)) {
+
+ pteEntries = w.pageTables.Allocator.NewPTEs()
+ for index := uint16(0); index < entriesPerPage; index++ {
+ pteEntries[index].Set(
+ pmdEntry.Address()+(pteSize*uintptr(index)),
+ pmdEntry.Opts())
+ }
+ pmdEntry.setPageTable(w.pageTables, pteEntries)
+ } else {
+
+ w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1)
+
+ if !pmdEntry.Valid() {
+ clearPMDEntries++
+ }
+
+ start = unmapnext(start, pmdSize)
+ continue
+ }
+ } else {
+ pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
+ }
+
+ clearPTEEntries := uint16(0)
+
+ for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
+ var (
+ pteEntry = &pteEntries[pteIndex]
+ )
+ if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
+ clearPTEEntries++
+ start += pteSize
+ continue
+ }
+
+ w.visitor.visit(uintptr(start), pteEntry, pteSize-1)
+ if !pteEntry.Valid() {
+ if w.visitor.requiresAlloc() {
+ panic("PTE not set after iteration with requiresAlloc!")
+ }
+ clearPTEEntries++
+ }
+
+ start += pteSize
+ continue
+ }
+
+ if clearPTEEntries == entriesPerPage {
+ pmdEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pteEntries)
+ clearPMDEntries++
+ }
+ }
+
+ if clearPMDEntries == entriesPerPage {
+ pudEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pmdEntries)
+ clearPUDEntries++
+ }
+ }
+
+ if clearPUDEntries == entriesPerPage {
+ pgdEntry.Clear()
+ w.pageTables.Allocator.FreePTEs(pudEntries)
+ }
+ }
+}
diff --git a/pkg/sentry/platform/ring0/ring0_state_autogen.go b/pkg/sentry/platform/ring0/ring0_state_autogen.go
new file mode 100755
index 000000000..462f9a446
--- /dev/null
+++ b/pkg/sentry/platform/ring0/ring0_state_autogen.go
@@ -0,0 +1,4 @@
+// automatically generated by stateify.
+
+package ring0
+
diff --git a/pkg/sentry/platform/ring0/x86.go b/pkg/sentry/platform/ring0/x86.go
deleted file mode 100644
index 5f80d64e8..000000000
--- a/pkg/sentry/platform/ring0/x86.go
+++ /dev/null
@@ -1,264 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build i386 amd64
-
-package ring0
-
-import (
- "gvisor.dev/gvisor/pkg/cpuid"
-)
-
-// Useful bits.
-const (
- _CR0_PE = 1 << 0
- _CR0_ET = 1 << 4
- _CR0_AM = 1 << 18
- _CR0_PG = 1 << 31
-
- _CR4_PSE = 1 << 4
- _CR4_PAE = 1 << 5
- _CR4_PGE = 1 << 7
- _CR4_OSFXSR = 1 << 9
- _CR4_OSXMMEXCPT = 1 << 10
- _CR4_FSGSBASE = 1 << 16
- _CR4_PCIDE = 1 << 17
- _CR4_OSXSAVE = 1 << 18
- _CR4_SMEP = 1 << 20
-
- _RFLAGS_AC = 1 << 18
- _RFLAGS_NT = 1 << 14
- _RFLAGS_IOPL = 3 << 12
- _RFLAGS_DF = 1 << 10
- _RFLAGS_IF = 1 << 9
- _RFLAGS_STEP = 1 << 8
- _RFLAGS_RESERVED = 1 << 1
-
- _EFER_SCE = 0x001
- _EFER_LME = 0x100
- _EFER_LMA = 0x400
- _EFER_NX = 0x800
-
- _MSR_STAR = 0xc0000081
- _MSR_LSTAR = 0xc0000082
- _MSR_CSTAR = 0xc0000083
- _MSR_SYSCALL_MASK = 0xc0000084
- _MSR_PLATFORM_INFO = 0xce
- _MSR_MISC_FEATURES = 0x140
-
- _PLATFORM_INFO_CPUID_FAULT = 1 << 31
-
- _MISC_FEATURE_CPUID_TRAP = 0x1
-)
-
-const (
- // KernelFlagsSet should always be set in the kernel.
- KernelFlagsSet = _RFLAGS_RESERVED
-
- // UserFlagsSet are always set in userspace.
- UserFlagsSet = _RFLAGS_RESERVED | _RFLAGS_IF
-
- // KernelFlagsClear should always be clear in the kernel.
- KernelFlagsClear = _RFLAGS_STEP | _RFLAGS_IF | _RFLAGS_IOPL | _RFLAGS_AC | _RFLAGS_NT
-
- // UserFlagsClear are always cleared in userspace.
- UserFlagsClear = _RFLAGS_NT | _RFLAGS_IOPL
-)
-
-// Vector is an exception vector.
-type Vector uintptr
-
-// Exception vectors.
-const (
- DivideByZero Vector = iota
- Debug
- NMI
- Breakpoint
- Overflow
- BoundRangeExceeded
- InvalidOpcode
- DeviceNotAvailable
- DoubleFault
- CoprocessorSegmentOverrun
- InvalidTSS
- SegmentNotPresent
- StackSegmentFault
- GeneralProtectionFault
- PageFault
- _
- X87FloatingPointException
- AlignmentCheck
- MachineCheck
- SIMDFloatingPointException
- VirtualizationException
- SecurityException = 0x1e
- SyscallInt80 = 0x80
- _NR_INTERRUPTS = SyscallInt80 + 1
-)
-
-// System call vectors.
-const (
- Syscall Vector = _NR_INTERRUPTS
-)
-
-// VirtualAddressBits returns the number bits available for virtual addresses.
-//
-// Note that sign-extension semantics apply to the highest order bit.
-//
-// FIXME(b/69382326): This should use the cpuid passed to Init.
-func VirtualAddressBits() uint32 {
- ax, _, _, _ := cpuid.HostID(0x80000008, 0)
- return (ax >> 8) & 0xff
-}
-
-// PhysicalAddressBits returns the number of bits available for physical addresses.
-//
-// FIXME(b/69382326): This should use the cpuid passed to Init.
-func PhysicalAddressBits() uint32 {
- ax, _, _, _ := cpuid.HostID(0x80000008, 0)
- return ax & 0xff
-}
-
-// Selector is a segment Selector.
-type Selector uint16
-
-// SegmentDescriptor is a segment descriptor.
-type SegmentDescriptor struct {
- bits [2]uint32
-}
-
-// descriptorTable is a collection of descriptors.
-type descriptorTable [32]SegmentDescriptor
-
-// SegmentDescriptorFlags are typed flags within a descriptor.
-type SegmentDescriptorFlags uint32
-
-// SegmentDescriptorFlag declarations.
-const (
- SegmentDescriptorAccess SegmentDescriptorFlags = 1 << 8 // Access bit (always set).
- SegmentDescriptorWrite = 1 << 9 // Write permission.
- SegmentDescriptorExpandDown = 1 << 10 // Grows down, not used.
- SegmentDescriptorExecute = 1 << 11 // Execute permission.
- SegmentDescriptorSystem = 1 << 12 // Zero => system, 1 => user code/data.
- SegmentDescriptorPresent = 1 << 15 // Present.
- SegmentDescriptorAVL = 1 << 20 // Available.
- SegmentDescriptorLong = 1 << 21 // Long mode.
- SegmentDescriptorDB = 1 << 22 // 16 or 32-bit.
- SegmentDescriptorG = 1 << 23 // Granularity: page or byte.
-)
-
-// Base returns the descriptor's base linear address.
-func (d *SegmentDescriptor) Base() uint32 {
- return d.bits[1]&0xFF000000 | (d.bits[1]&0x000000FF)<<16 | d.bits[0]>>16
-}
-
-// Limit returns the descriptor size.
-func (d *SegmentDescriptor) Limit() uint32 {
- l := d.bits[0]&0xFFFF | d.bits[1]&0xF0000
- if d.bits[1]&uint32(SegmentDescriptorG) != 0 {
- l <<= 12
- l |= 0xFFF
- }
- return l
-}
-
-// Flags returns descriptor flags.
-func (d *SegmentDescriptor) Flags() SegmentDescriptorFlags {
- return SegmentDescriptorFlags(d.bits[1] & 0x00F09F00)
-}
-
-// DPL returns the descriptor privilege level.
-func (d *SegmentDescriptor) DPL() int {
- return int((d.bits[1] >> 13) & 3)
-}
-
-func (d *SegmentDescriptor) setNull() {
- d.bits[0] = 0
- d.bits[1] = 0
-}
-
-func (d *SegmentDescriptor) set(base, limit uint32, dpl int, flags SegmentDescriptorFlags) {
- flags |= SegmentDescriptorPresent
- if limit>>12 != 0 {
- limit >>= 12
- flags |= SegmentDescriptorG
- }
- d.bits[0] = base<<16 | limit&0xFFFF
- d.bits[1] = base&0xFF000000 | (base>>16)&0xFF | limit&0x000F0000 | uint32(flags) | uint32(dpl)<<13
-}
-
-func (d *SegmentDescriptor) setCode32(base, limit uint32, dpl int) {
- d.set(base, limit, dpl,
- SegmentDescriptorDB|
- SegmentDescriptorExecute|
- SegmentDescriptorSystem)
-}
-
-func (d *SegmentDescriptor) setCode64(base, limit uint32, dpl int) {
- d.set(base, limit, dpl,
- SegmentDescriptorG|
- SegmentDescriptorLong|
- SegmentDescriptorExecute|
- SegmentDescriptorSystem)
-}
-
-func (d *SegmentDescriptor) setData(base, limit uint32, dpl int) {
- d.set(base, limit, dpl,
- SegmentDescriptorWrite|
- SegmentDescriptorSystem)
-}
-
-// setHi is only used for the TSS segment, which is magically 64-bits.
-func (d *SegmentDescriptor) setHi(base uint32) {
- d.bits[0] = base
- d.bits[1] = 0
-}
-
-// Gate64 is a 64-bit task, trap, or interrupt gate.
-type Gate64 struct {
- bits [4]uint32
-}
-
-// idt64 is a 64-bit interrupt descriptor table.
-type idt64 [_NR_INTERRUPTS]Gate64
-
-func (g *Gate64) setInterrupt(cs Selector, rip uint64, dpl int, ist int) {
- g.bits[0] = uint32(cs)<<16 | uint32(rip)&0xFFFF
- g.bits[1] = uint32(rip)&0xFFFF0000 | SegmentDescriptorPresent | uint32(dpl)<<13 | 14<<8 | uint32(ist)&0x7
- g.bits[2] = uint32(rip >> 32)
-}
-
-func (g *Gate64) setTrap(cs Selector, rip uint64, dpl int, ist int) {
- g.setInterrupt(cs, rip, dpl, ist)
- g.bits[1] |= 1 << 8
-}
-
-// TaskState64 is a 64-bit task state structure.
-type TaskState64 struct {
- _ uint32
- rsp0Lo, rsp0Hi uint32
- rsp1Lo, rsp1Hi uint32
- rsp2Lo, rsp2Hi uint32
- _ [2]uint32
- ist1Lo, ist1Hi uint32
- ist2Lo, ist2Hi uint32
- ist3Lo, ist3Hi uint32
- ist4Lo, ist4Hi uint32
- ist5Lo, ist5Hi uint32
- ist6Lo, ist6Hi uint32
- ist7Lo, ist7Hi uint32
- _ [2]uint32
- _ uint16
- ioPerm uint16
-}
diff --git a/pkg/sentry/platform/safecopy/BUILD b/pkg/sentry/platform/safecopy/BUILD
deleted file mode 100644
index 924d8a6d6..000000000
--- a/pkg/sentry/platform/safecopy/BUILD
+++ /dev/null
@@ -1,30 +0,0 @@
-load("//tools/go_stateify:defs.bzl", "go_library", "go_test")
-
-package(licenses = ["notice"])
-
-go_library(
- name = "safecopy",
- srcs = [
- "atomic_amd64.s",
- "atomic_arm64.s",
- "memclr_amd64.s",
- "memclr_arm64.s",
- "memcpy_amd64.s",
- "memcpy_arm64.s",
- "safecopy.go",
- "safecopy_unsafe.go",
- "sighandler_amd64.s",
- "sighandler_arm64.s",
- ],
- importpath = "gvisor.dev/gvisor/pkg/sentry/platform/safecopy",
- visibility = ["//pkg/sentry:internal"],
- deps = ["//pkg/syserror"],
-)
-
-go_test(
- name = "safecopy_test",
- srcs = [
- "safecopy_test.go",
- ],
- embed = [":safecopy"],
-)
diff --git a/pkg/sentry/platform/safecopy/LICENSE b/pkg/sentry/platform/safecopy/LICENSE
deleted file mode 100644
index 6a66aea5e..000000000
--- a/pkg/sentry/platform/safecopy/LICENSE
+++ /dev/null
@@ -1,27 +0,0 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/pkg/sentry/platform/safecopy/safecopy_state_autogen.go b/pkg/sentry/platform/safecopy/safecopy_state_autogen.go
new file mode 100755
index 000000000..58fd8fbd0
--- /dev/null
+++ b/pkg/sentry/platform/safecopy/safecopy_state_autogen.go
@@ -0,0 +1,4 @@
+// automatically generated by stateify.
+
+package safecopy
+
diff --git a/pkg/sentry/platform/safecopy/safecopy_test.go b/pkg/sentry/platform/safecopy/safecopy_test.go
deleted file mode 100644
index 5818f7f9b..000000000
--- a/pkg/sentry/platform/safecopy/safecopy_test.go
+++ /dev/null
@@ -1,617 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package safecopy
-
-import (
- "bytes"
- "fmt"
- "io/ioutil"
- "math/rand"
- "os"
- "runtime/debug"
- "syscall"
- "testing"
- "unsafe"
-)
-
-// Size of a page in bytes. Cloned from usermem.PageSize to avoid a circular
-// dependency.
-const pageSize = 4096
-
-func initRandom(b []byte) {
- for i := range b {
- b[i] = byte(rand.Intn(256))
- }
-}
-
-func randBuf(size int) []byte {
- b := make([]byte, size)
- initRandom(b)
- return b
-}
-
-func TestCopyInSuccess(t *testing.T) {
- // Test that CopyIn does not return an error when all pages are accessible.
- const bufLen = 8192
- a := randBuf(bufLen)
- b := make([]byte, bufLen)
-
- n, err := CopyIn(b, unsafe.Pointer(&a[0]))
- if n != bufLen {
- t.Errorf("Unexpected copy length, got %v, want %v", n, bufLen)
- }
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- if !bytes.Equal(a, b) {
- t.Errorf("Buffers are not equal when they should be: %v %v", a, b)
- }
-}
-
-func TestCopyOutSuccess(t *testing.T) {
- // Test that CopyOut does not return an error when all pages are
- // accessible.
- const bufLen = 8192
- a := randBuf(bufLen)
- b := make([]byte, bufLen)
-
- n, err := CopyOut(unsafe.Pointer(&b[0]), a)
- if n != bufLen {
- t.Errorf("Unexpected copy length, got %v, want %v", n, bufLen)
- }
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- if !bytes.Equal(a, b) {
- t.Errorf("Buffers are not equal when they should be: %v %v", a, b)
- }
-}
-
-func TestCopySuccess(t *testing.T) {
- // Test that Copy does not return an error when all pages are accessible.
- const bufLen = 8192
- a := randBuf(bufLen)
- b := make([]byte, bufLen)
-
- n, err := Copy(unsafe.Pointer(&b[0]), unsafe.Pointer(&a[0]), bufLen)
- if n != bufLen {
- t.Errorf("Unexpected copy length, got %v, want %v", n, bufLen)
- }
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- if !bytes.Equal(a, b) {
- t.Errorf("Buffers are not equal when they should be: %v %v", a, b)
- }
-}
-
-func TestZeroOutSuccess(t *testing.T) {
- // Test that ZeroOut does not return an error when all pages are
- // accessible.
- const bufLen = 8192
- a := make([]byte, bufLen)
- b := randBuf(bufLen)
-
- n, err := ZeroOut(unsafe.Pointer(&b[0]), bufLen)
- if n != bufLen {
- t.Errorf("Unexpected copy length, got %v, want %v", n, bufLen)
- }
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- if !bytes.Equal(a, b) {
- t.Errorf("Buffers are not equal when they should be: %v %v", a, b)
- }
-}
-
-func TestSwapUint32Success(t *testing.T) {
- // Test that SwapUint32 does not return an error when the page is
- // accessible.
- before := uint32(rand.Int31())
- after := uint32(rand.Int31())
- val := before
-
- old, err := SwapUint32(unsafe.Pointer(&val), after)
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- if old != before {
- t.Errorf("Unexpected old value: got %v, want %v", old, before)
- }
- if val != after {
- t.Errorf("Unexpected new value: got %v, want %v", val, after)
- }
-}
-
-func TestSwapUint32AlignmentError(t *testing.T) {
- // Test that SwapUint32 returns an AlignmentError when passed an unaligned
- // address.
- data := new(struct{ val uint64 })
- addr := uintptr(unsafe.Pointer(&data.val)) + 1
- want := AlignmentError{Addr: addr, Alignment: 4}
- if _, err := SwapUint32(unsafe.Pointer(addr), 1); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
-}
-
-func TestSwapUint64Success(t *testing.T) {
- // Test that SwapUint64 does not return an error when the page is
- // accessible.
- before := uint64(rand.Int63())
- after := uint64(rand.Int63())
- // "The first word in ... an allocated struct or slice can be relied upon
- // to be 64-bit aligned." - sync/atomic docs
- data := new(struct{ val uint64 })
- data.val = before
-
- old, err := SwapUint64(unsafe.Pointer(&data.val), after)
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- if old != before {
- t.Errorf("Unexpected old value: got %v, want %v", old, before)
- }
- if data.val != after {
- t.Errorf("Unexpected new value: got %v, want %v", data.val, after)
- }
-}
-
-func TestSwapUint64AlignmentError(t *testing.T) {
- // Test that SwapUint64 returns an AlignmentError when passed an unaligned
- // address.
- data := new(struct{ val1, val2 uint64 })
- addr := uintptr(unsafe.Pointer(&data.val1)) + 1
- want := AlignmentError{Addr: addr, Alignment: 8}
- if _, err := SwapUint64(unsafe.Pointer(addr), 1); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
-}
-
-func TestCompareAndSwapUint32Success(t *testing.T) {
- // Test that CompareAndSwapUint32 does not return an error when the page is
- // accessible.
- before := uint32(rand.Int31())
- after := uint32(rand.Int31())
- val := before
-
- old, err := CompareAndSwapUint32(unsafe.Pointer(&val), before, after)
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- }
- if old != before {
- t.Errorf("Unexpected old value: got %v, want %v", old, before)
- }
- if val != after {
- t.Errorf("Unexpected new value: got %v, want %v", val, after)
- }
-}
-
-func TestCompareAndSwapUint32AlignmentError(t *testing.T) {
- // Test that CompareAndSwapUint32 returns an AlignmentError when passed an
- // unaligned address.
- data := new(struct{ val uint64 })
- addr := uintptr(unsafe.Pointer(&data.val)) + 1
- want := AlignmentError{Addr: addr, Alignment: 4}
- if _, err := CompareAndSwapUint32(unsafe.Pointer(addr), 0, 1); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
-}
-
-// withSegvErrorTestMapping calls fn with a two-page mapping. The first page
-// contains random data, and the second page generates SIGSEGV when accessed.
-func withSegvErrorTestMapping(t *testing.T, fn func(m []byte)) {
- mapping, err := syscall.Mmap(-1, 0, 2*pageSize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANONYMOUS|syscall.MAP_PRIVATE)
- if err != nil {
- t.Fatalf("Mmap failed: %v", err)
- }
- defer syscall.Munmap(mapping)
- if err := syscall.Mprotect(mapping[pageSize:], syscall.PROT_NONE); err != nil {
- t.Fatalf("Mprotect failed: %v", err)
- }
- initRandom(mapping[:pageSize])
-
- fn(mapping)
-}
-
-// withBusErrorTestMapping calls fn with a two-page mapping. The first page
-// contains random data, and the second page generates SIGBUS when accessed.
-func withBusErrorTestMapping(t *testing.T, fn func(m []byte)) {
- f, err := ioutil.TempFile("", "sigbus_test")
- if err != nil {
- t.Fatalf("TempFile failed: %v", err)
- }
- defer f.Close()
- if err := f.Truncate(pageSize); err != nil {
- t.Fatalf("Truncate failed: %v", err)
- }
- mapping, err := syscall.Mmap(int(f.Fd()), 0, 2*pageSize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
- if err != nil {
- t.Fatalf("Mmap failed: %v", err)
- }
- defer syscall.Munmap(mapping)
- initRandom(mapping[:pageSize])
-
- fn(mapping)
-}
-
-func TestCopyInSegvError(t *testing.T) {
- // Test that CopyIn returns a SegvError when reaching a page that signals
- // SIGSEGV.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGSEGV", bytesBeforeFault), func(t *testing.T) {
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- src := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- dst := randBuf(pageSize)
- n, err := CopyIn(dst, src)
- if n != bytesBeforeFault {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := dst[:bytesBeforeFault], mapping[pageSize-bytesBeforeFault:pageSize]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestCopyInBusError(t *testing.T) {
- // Test that CopyIn returns a BusError when reaching a page that signals
- // SIGBUS.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGBUS", bytesBeforeFault), func(t *testing.T) {
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- src := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- dst := randBuf(pageSize)
- n, err := CopyIn(dst, src)
- if n != bytesBeforeFault {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := dst[:bytesBeforeFault], mapping[pageSize-bytesBeforeFault:pageSize]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestCopyOutSegvError(t *testing.T) {
- // Test that CopyOut returns a SegvError when reaching a page that signals
- // SIGSEGV.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGSEGV", bytesBeforeFault), func(t *testing.T) {
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- dst := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- src := randBuf(pageSize)
- n, err := CopyOut(dst, src)
- if n != bytesBeforeFault {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := mapping[pageSize-bytesBeforeFault:pageSize], src[:bytesBeforeFault]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestCopyOutBusError(t *testing.T) {
- // Test that CopyOut returns a BusError when reaching a page that signals
- // SIGBUS.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGSEGV", bytesBeforeFault), func(t *testing.T) {
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- dst := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- src := randBuf(pageSize)
- n, err := CopyOut(dst, src)
- if n != bytesBeforeFault {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := mapping[pageSize-bytesBeforeFault:pageSize], src[:bytesBeforeFault]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestCopySourceSegvError(t *testing.T) {
- // Test that Copy returns a SegvError when copying from a page that signals
- // SIGSEGV.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGSEGV", bytesBeforeFault), func(t *testing.T) {
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- src := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- dst := randBuf(pageSize)
- n, err := Copy(unsafe.Pointer(&dst[0]), src, pageSize)
- if n != uintptr(bytesBeforeFault) {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := dst[:bytesBeforeFault], mapping[pageSize-bytesBeforeFault:pageSize]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestCopySourceBusError(t *testing.T) {
- // Test that Copy returns a BusError when copying from a page that signals
- // SIGBUS.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGBUS", bytesBeforeFault), func(t *testing.T) {
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- src := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- dst := randBuf(pageSize)
- n, err := Copy(unsafe.Pointer(&dst[0]), src, pageSize)
- if n != uintptr(bytesBeforeFault) {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := dst[:bytesBeforeFault], mapping[pageSize-bytesBeforeFault:pageSize]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestCopyDestinationSegvError(t *testing.T) {
- // Test that Copy returns a SegvError when copying to a page that signals
- // SIGSEGV.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGSEGV", bytesBeforeFault), func(t *testing.T) {
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- dst := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- src := randBuf(pageSize)
- n, err := Copy(dst, unsafe.Pointer(&src[0]), pageSize)
- if n != uintptr(bytesBeforeFault) {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := mapping[pageSize-bytesBeforeFault:pageSize], src[:bytesBeforeFault]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestCopyDestinationBusError(t *testing.T) {
- // Test that Copy returns a BusError when copying to a page that signals
- // SIGBUS.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting copy %d bytes before SIGBUS", bytesBeforeFault), func(t *testing.T) {
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- dst := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- src := randBuf(pageSize)
- n, err := Copy(dst, unsafe.Pointer(&src[0]), pageSize)
- if n != uintptr(bytesBeforeFault) {
- t.Errorf("Unexpected copy length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := mapping[pageSize-bytesBeforeFault:pageSize], src[:bytesBeforeFault]; !bytes.Equal(got, want) {
- t.Errorf("Buffers are not equal when they should be: %v %v", got, want)
- }
- })
- })
- }
-}
-
-func TestZeroOutSegvError(t *testing.T) {
- // Test that ZeroOut returns a SegvError when reaching a page that signals
- // SIGSEGV.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting write %d bytes before SIGSEGV", bytesBeforeFault), func(t *testing.T) {
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- dst := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- n, err := ZeroOut(dst, pageSize)
- if n != uintptr(bytesBeforeFault) {
- t.Errorf("Unexpected write length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := mapping[pageSize-bytesBeforeFault:pageSize], make([]byte, bytesBeforeFault); !bytes.Equal(got, want) {
- t.Errorf("Non-zero bytes in written part of mapping: %v", got)
- }
- })
- })
- }
-}
-
-func TestZeroOutBusError(t *testing.T) {
- // Test that ZeroOut returns a BusError when reaching a page that signals
- // SIGBUS.
- for bytesBeforeFault := 0; bytesBeforeFault <= 2*maxRegisterSize; bytesBeforeFault++ {
- t.Run(fmt.Sprintf("starting write %d bytes before SIGBUS", bytesBeforeFault), func(t *testing.T) {
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- dst := unsafe.Pointer(secondPage - uintptr(bytesBeforeFault))
- n, err := ZeroOut(dst, pageSize)
- if n != uintptr(bytesBeforeFault) {
- t.Errorf("Unexpected write length: got %v, want %v", n, bytesBeforeFault)
- }
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- if got, want := mapping[pageSize-bytesBeforeFault:pageSize], make([]byte, bytesBeforeFault); !bytes.Equal(got, want) {
- t.Errorf("Non-zero bytes in written part of mapping: %v", got)
- }
- })
- })
- }
-}
-
-func TestSwapUint32SegvError(t *testing.T) {
- // Test that SwapUint32 returns a SegvError when reaching a page that
- // signals SIGSEGV.
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- _, err := SwapUint32(unsafe.Pointer(secondPage), 1)
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- })
-}
-
-func TestSwapUint32BusError(t *testing.T) {
- // Test that SwapUint32 returns a BusError when reaching a page that
- // signals SIGBUS.
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- _, err := SwapUint32(unsafe.Pointer(secondPage), 1)
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- })
-}
-
-func TestSwapUint64SegvError(t *testing.T) {
- // Test that SwapUint64 returns a SegvError when reaching a page that
- // signals SIGSEGV.
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- _, err := SwapUint64(unsafe.Pointer(secondPage), 1)
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- })
-}
-
-func TestSwapUint64BusError(t *testing.T) {
- // Test that SwapUint64 returns a BusError when reaching a page that
- // signals SIGBUS.
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- _, err := SwapUint64(unsafe.Pointer(secondPage), 1)
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- })
-}
-
-func TestCompareAndSwapUint32SegvError(t *testing.T) {
- // Test that CompareAndSwapUint32 returns a SegvError when reaching a page
- // that signals SIGSEGV.
- withSegvErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- _, err := CompareAndSwapUint32(unsafe.Pointer(secondPage), 0, 1)
- if want := (SegvError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- })
-}
-
-func TestCompareAndSwapUint32BusError(t *testing.T) {
- // Test that CompareAndSwapUint32 returns a BusError when reaching a page
- // that signals SIGBUS.
- withBusErrorTestMapping(t, func(mapping []byte) {
- secondPage := uintptr(unsafe.Pointer(&mapping[0])) + pageSize
- _, err := CompareAndSwapUint32(unsafe.Pointer(secondPage), 0, 1)
- if want := (BusError{secondPage}); err != want {
- t.Errorf("Unexpected error: got %v, want %v", err, want)
- }
- })
-}
-
-func testCopy(dst, src []byte) (panicked bool) {
- defer func() {
- if r := recover(); r != nil {
- panicked = true
- }
- }()
- debug.SetPanicOnFault(true)
- copy(dst, src)
- return
-}
-
-func TestSegVOnMemmove(t *testing.T) {
- // Test that SIGSEGVs received by runtime.memmove when *not* doing
- // CopyIn or CopyOut work gets propagated to the runtime.
- const bufLen = pageSize
- a, err := syscall.Mmap(-1, 0, bufLen, syscall.PROT_NONE, syscall.MAP_ANON|syscall.MAP_PRIVATE)
- if err != nil {
- t.Fatalf("Mmap failed: %v", err)
-
- }
- defer syscall.Munmap(a)
- b := randBuf(bufLen)
-
- if !testCopy(b, a) {
- t.Fatalf("testCopy didn't panic when it should have")
- }
-
- if !testCopy(a, b) {
- t.Fatalf("testCopy didn't panic when it should have")
- }
-}
-
-func TestSigbusOnMemmove(t *testing.T) {
- // Test that SIGBUS received by runtime.memmove when *not* doing
- // CopyIn or CopyOut work gets propagated to the runtime.
- const bufLen = pageSize
- f, err := ioutil.TempFile("", "sigbus_test")
- if err != nil {
- t.Fatalf("TempFile failed: %v", err)
- }
- os.Remove(f.Name())
- defer f.Close()
-
- a, err := syscall.Mmap(int(f.Fd()), 0, bufLen, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
- if err != nil {
- t.Fatalf("Mmap failed: %v", err)
-
- }
- defer syscall.Munmap(a)
- b := randBuf(bufLen)
-
- if !testCopy(b, a) {
- t.Fatalf("testCopy didn't panic when it should have")
- }
-
- if !testCopy(a, b) {
- t.Fatalf("testCopy didn't panic when it should have")
- }
-}