diff options
| author | gVisor bot <gvisor-bot@google.com> | 2021-02-02 20:16:40 +0000 |
|---|---|---|
| committer | gVisor bot <gvisor-bot@google.com> | 2021-02-02 20:16:40 +0000 |
| commit | 5011c8943e1048c3b34b9dc11a888ee3c8d42302 (patch) | |
| tree | 88cb34e1b1515c30ea029beb8b4fd575a66f779c /pkg/sentry/platform | |
| parent | c4bddc8deecbb4d6e3e9269e321610cdf085aab7 (diff) | |
| parent | f884ea13b713143ff9978092ddb352c159346167 (diff) | |
Merge release-20210125.0-61-gf884ea13b (automated)
Diffstat (limited to 'pkg/sentry/platform')
65 files changed, 21 insertions, 7696 deletions
diff --git a/pkg/sentry/platform/kvm/address_space.go b/pkg/sentry/platform/kvm/address_space.go index af5c5e191..25c21e843 100644 --- a/pkg/sentry/platform/kvm/address_space.go +++ b/pkg/sentry/platform/kvm/address_space.go @@ -18,9 +18,9 @@ import ( "sync/atomic" "gvisor.dev/gvisor/pkg/atomicbitops" + "gvisor.dev/gvisor/pkg/ring0/pagetables" "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/platform" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/usermem" ) diff --git a/pkg/sentry/platform/kvm/bluepill.go b/pkg/sentry/platform/kvm/bluepill.go index 4b23f7803..2c970162e 100644 --- a/pkg/sentry/platform/kvm/bluepill.go +++ b/pkg/sentry/platform/kvm/bluepill.go @@ -19,9 +19,9 @@ import ( "reflect" "syscall" + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/safecopy" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" ) // bluepill enters guest mode. diff --git a/pkg/sentry/platform/kvm/bluepill_allocator.go b/pkg/sentry/platform/kvm/bluepill_allocator.go index 9485e1301..1825edc3a 100644 --- a/pkg/sentry/platform/kvm/bluepill_allocator.go +++ b/pkg/sentry/platform/kvm/bluepill_allocator.go @@ -17,7 +17,7 @@ package kvm import ( "fmt" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" + "gvisor.dev/gvisor/pkg/ring0/pagetables" ) type allocator struct { diff --git a/pkg/sentry/platform/kvm/bluepill_amd64.go b/pkg/sentry/platform/kvm/bluepill_amd64.go index ddc1554d5..83a4766fb 100644 --- a/pkg/sentry/platform/kvm/bluepill_amd64.go +++ b/pkg/sentry/platform/kvm/bluepill_amd64.go @@ -19,8 +19,8 @@ package kvm import ( "syscall" + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" ) var ( diff --git a/pkg/sentry/platform/kvm/bluepill_amd64_unsafe.go b/pkg/sentry/platform/kvm/bluepill_amd64_unsafe.go index f8ccb7430..0063e947b 100644 --- a/pkg/sentry/platform/kvm/bluepill_amd64_unsafe.go +++ b/pkg/sentry/platform/kvm/bluepill_amd64_unsafe.go @@ -20,8 +20,8 @@ import ( "syscall" "unsafe" + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" ) // dieArchSetup initializes the state for dieTrampoline. diff --git a/pkg/sentry/platform/kvm/bluepill_arm64.go b/pkg/sentry/platform/kvm/bluepill_arm64.go index 1f09813ba..35298135a 100644 --- a/pkg/sentry/platform/kvm/bluepill_arm64.go +++ b/pkg/sentry/platform/kvm/bluepill_arm64.go @@ -19,8 +19,8 @@ package kvm import ( "syscall" + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" ) var ( diff --git a/pkg/sentry/platform/kvm/bluepill_arm64_unsafe.go b/pkg/sentry/platform/kvm/bluepill_arm64_unsafe.go index 4d912769a..dbbf2a897 100644 --- a/pkg/sentry/platform/kvm/bluepill_arm64_unsafe.go +++ b/pkg/sentry/platform/kvm/bluepill_arm64_unsafe.go @@ -20,8 +20,8 @@ import ( "syscall" "unsafe" + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" ) // fpsimdPtr returns a fpsimd64 for the given address. diff --git a/pkg/sentry/platform/kvm/context.go b/pkg/sentry/platform/kvm/context.go index 17268d127..aeae01dbd 100644 --- a/pkg/sentry/platform/kvm/context.go +++ b/pkg/sentry/platform/kvm/context.go @@ -18,10 +18,10 @@ import ( "sync/atomic" pkgcontext "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sentry/platform/interrupt" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" "gvisor.dev/gvisor/pkg/usermem" ) diff --git a/pkg/sentry/platform/kvm/kvm.go b/pkg/sentry/platform/kvm/kvm.go index 5979aef97..7bdf57436 100644 --- a/pkg/sentry/platform/kvm/kvm.go +++ b/pkg/sentry/platform/kvm/kvm.go @@ -20,9 +20,9 @@ import ( "os" "syscall" + "gvisor.dev/gvisor/pkg/ring0" + "gvisor.dev/gvisor/pkg/ring0/pagetables" "gvisor.dev/gvisor/pkg/sentry/platform" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/usermem" ) diff --git a/pkg/sentry/platform/kvm/kvm_amd64.go b/pkg/sentry/platform/kvm/kvm_amd64.go index 093497bc4..b9ed4a706 100644 --- a/pkg/sentry/platform/kvm/kvm_amd64.go +++ b/pkg/sentry/platform/kvm/kvm_amd64.go @@ -18,7 +18,7 @@ package kvm import ( "gvisor.dev/gvisor/pkg/cpuid" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" + "gvisor.dev/gvisor/pkg/ring0" ) // userRegs represents KVM user registers. diff --git a/pkg/sentry/platform/kvm/kvm_arm64.go b/pkg/sentry/platform/kvm/kvm_arm64.go index 9db1db4e9..b73340f0e 100644 --- a/pkg/sentry/platform/kvm/kvm_arm64.go +++ b/pkg/sentry/platform/kvm/kvm_arm64.go @@ -17,8 +17,8 @@ package kvm import ( + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" ) type kvmOneReg struct { diff --git a/pkg/sentry/platform/kvm/machine.go b/pkg/sentry/platform/kvm/machine.go index e2fffc99b..1ece1b8d8 100644 --- a/pkg/sentry/platform/kvm/machine.go +++ b/pkg/sentry/platform/kvm/machine.go @@ -23,8 +23,8 @@ import ( "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/procid" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" + "gvisor.dev/gvisor/pkg/ring0" + "gvisor.dev/gvisor/pkg/ring0/pagetables" ktime "gvisor.dev/gvisor/pkg/sentry/time" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/usermem" diff --git a/pkg/sentry/platform/kvm/machine_amd64.go b/pkg/sentry/platform/kvm/machine_amd64.go index 8e03c310d..59c752d73 100644 --- a/pkg/sentry/platform/kvm/machine_amd64.go +++ b/pkg/sentry/platform/kvm/machine_amd64.go @@ -24,10 +24,10 @@ import ( "syscall" "gvisor.dev/gvisor/pkg/cpuid" + "gvisor.dev/gvisor/pkg/ring0" + "gvisor.dev/gvisor/pkg/ring0/pagetables" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/platform" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" ktime "gvisor.dev/gvisor/pkg/sentry/time" "gvisor.dev/gvisor/pkg/usermem" ) diff --git a/pkg/sentry/platform/kvm/machine_arm64.go b/pkg/sentry/platform/kvm/machine_arm64.go index aa2d21748..7d7857067 100644 --- a/pkg/sentry/platform/kvm/machine_arm64.go +++ b/pkg/sentry/platform/kvm/machine_arm64.go @@ -17,10 +17,10 @@ package kvm import ( + "gvisor.dev/gvisor/pkg/ring0" + "gvisor.dev/gvisor/pkg/ring0/pagetables" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/platform" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" "gvisor.dev/gvisor/pkg/usermem" ) diff --git a/pkg/sentry/platform/kvm/machine_arm64_unsafe.go b/pkg/sentry/platform/kvm/machine_arm64_unsafe.go index a466acf4d..dca0cdb60 100644 --- a/pkg/sentry/platform/kvm/machine_arm64_unsafe.go +++ b/pkg/sentry/platform/kvm/machine_arm64_unsafe.go @@ -23,10 +23,10 @@ import ( "syscall" "unsafe" + "gvisor.dev/gvisor/pkg/ring0" + "gvisor.dev/gvisor/pkg/ring0/pagetables" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/platform" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" "gvisor.dev/gvisor/pkg/usermem" ) diff --git a/pkg/sentry/platform/kvm/physical_map.go b/pkg/sentry/platform/kvm/physical_map.go index f7fa2f98d..8bdec93ae 100644 --- a/pkg/sentry/platform/kvm/physical_map.go +++ b/pkg/sentry/platform/kvm/physical_map.go @@ -20,7 +20,7 @@ import ( "syscall" "gvisor.dev/gvisor/pkg/log" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0" + "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/usermem" ) diff --git a/pkg/sentry/platform/ring0/defs_impl_amd64.go b/pkg/sentry/platform/ring0/defs_impl_amd64.go deleted file mode 100644 index 3282254a7..000000000 --- a/pkg/sentry/platform/ring0/defs_impl_amd64.go +++ /dev/null @@ -1,598 +0,0 @@ -// +build amd64 -// +build amd64 -// +build 386 amd64 - -package ring0 - -import ( - "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" - - "fmt" - "gvisor.dev/gvisor/pkg/cpuid" - "gvisor.dev/gvisor/pkg/usermem" - "io" - "reflect" -) - -// Kernel is a global kernel object. -// -// This contains global state, shared by multiple CPUs. -type Kernel struct { - // PageTables are the kernel pagetables; this must be provided. - PageTables *pagetables.PageTables - - KernelArchState -} - -// Hooks are hooks for kernel functions. -type Hooks interface { - // KernelSyscall is called for kernel system calls. - // - // Return from this call will restore registers and return to the kernel: the - // registers must be modified directly. - // - // If this function is not provided, a kernel exception results in halt. - // - // This must be go:nosplit, as this will be on the interrupt stack. - // Closures are permitted, as the pointer to the closure frame is not - // passed on the stack. - KernelSyscall() - - // KernelException handles an exception during kernel execution. - // - // Return from this call will restore registers and return to the kernel: the - // registers must be modified directly. - // - // If this function is not provided, a kernel exception results in halt. - // - // This must be go:nosplit, as this will be on the interrupt stack. - // Closures are permitted, as the pointer to the closure frame is not - // passed on the stack. - KernelException(Vector) -} - -// CPU is the per-CPU struct. -type CPU struct { - // self is a self reference. - // - // This is always guaranteed to be at offset zero. - self *CPU - - // kernel is reference to the kernel that this CPU was initialized - // with. This reference is kept for garbage collection purposes: CPU - // registers may refer to objects within the Kernel object that cannot - // be safely freed. - kernel *Kernel - - // CPUArchState is architecture-specific state. - CPUArchState - - // registers is a set of registers; these may be used on kernel system - // calls and exceptions via the Registers function. - registers arch.Registers - - // hooks are kernel hooks. - hooks Hooks -} - -// Registers returns a modifiable-copy of the kernel registers. -// -// This is explicitly safe to call during KernelException and KernelSyscall. -// -//go:nosplit -func (c *CPU) Registers() *arch.Registers { - return &c.registers -} - -// SwitchOpts are passed to the Switch function. -type SwitchOpts struct { - // Registers are the user register state. - Registers *arch.Registers - - // FloatingPointState is a byte pointer where floating point state is - // saved and restored. - FloatingPointState *byte - - // PageTables are the application page tables. - PageTables *pagetables.PageTables - - // Flush indicates that a TLB flush should be forced on switch. - Flush bool - - // FullRestore indicates that an iret-based restore should be used. - FullRestore bool - - // SwitchArchOpts are architecture-specific options. - SwitchArchOpts -} - -var ( - // UserspaceSize is the total size of userspace. - UserspaceSize = uintptr(1) << (VirtualAddressBits() - 1) - - // MaximumUserAddress is the largest possible user address. - MaximumUserAddress = (UserspaceSize - 1) & ^uintptr(usermem.PageSize-1) - - // KernelStartAddress is the starting kernel address. - KernelStartAddress = ^uintptr(0) - (UserspaceSize - 1) -) - -// Segment indices and Selectors. -const ( - // Index into GDT array. - _ = iota // Null descriptor first. - _ // Reserved (Linux is kernel 32). - segKcode // Kernel code (64-bit). - segKdata // Kernel data. - segUcode32 // User code (32-bit). - segUdata // User data. - segUcode64 // User code (64-bit). - segTss // Task segment descriptor. - segTssHi // Upper bits for TSS. - segLast // Last segment (terminal, not included). -) - -// Selectors. -const ( - Kcode Selector = segKcode << 3 - Kdata Selector = segKdata << 3 - Ucode32 Selector = (segUcode32 << 3) | 3 - Udata Selector = (segUdata << 3) | 3 - Ucode64 Selector = (segUcode64 << 3) | 3 - Tss Selector = segTss << 3 -) - -// Standard segments. -var ( - UserCodeSegment32 SegmentDescriptor - UserDataSegment SegmentDescriptor - UserCodeSegment64 SegmentDescriptor - KernelCodeSegment SegmentDescriptor - KernelDataSegment SegmentDescriptor -) - -// KernelArchState contains architecture-specific state. -type KernelArchState struct { - // cpuEntries is array of kernelEntry for all cpus. - cpuEntries []kernelEntry - - // globalIDT is our set of interrupt gates. - globalIDT *idt64 -} - -// kernelEntry contains minimal CPU-specific arch state -// that can be mapped at the upper of the address space. -// Malicious APP might steal info from it via CPU bugs. -type kernelEntry struct { - // stack is the stack used for interrupts on this CPU. - stack [256]byte - - // scratch space for temporary usage. - scratch0 uint64 - - // stackTop is the top of the stack. - stackTop uint64 - - // cpuSelf is back reference to CPU. - cpuSelf *CPU - - // kernelCR3 is the cr3 used for sentry kernel. - kernelCR3 uintptr - - // gdt is the CPU's descriptor table. - gdt descriptorTable - - // tss is the CPU's task state. - tss TaskState64 -} - -// CPUArchState contains CPU-specific arch state. -type CPUArchState struct { - // errorCode is the error code from the last exception. - errorCode uintptr - - // errorType indicates the type of error code here, it is always set - // along with the errorCode value above. - // - // It will either by 1, which indicates a user error, or 0 indicating a - // kernel error. If the error code below returns false (kernel error), - // then it cannot provide relevant information about the last - // exception. - errorType uintptr - - *kernelEntry -} - -// ErrorCode returns the last error code. -// -// The returned boolean indicates whether the error code corresponds to the -// last user error or not. If it does not, then fault information must be -// ignored. This is generally the result of a kernel fault while servicing a -// user fault. -// -//go:nosplit -func (c *CPU) ErrorCode() (value uintptr, user bool) { - return c.errorCode, c.errorType != 0 -} - -// ClearErrorCode resets the error code. -// -//go:nosplit -func (c *CPU) ClearErrorCode() { - c.errorCode = 0 - c.errorType = 1 -} - -// SwitchArchOpts are embedded in SwitchOpts. -type SwitchArchOpts struct { - // UserPCID indicates that the application PCID to be used on switch, - // assuming that PCIDs are supported. - // - // Per pagetables_x86.go, a zero PCID implies a flush. - UserPCID uint16 - - // KernelPCID indicates that the kernel PCID to be used on return, - // assuming that PCIDs are supported. - // - // Per pagetables_x86.go, a zero PCID implies a flush. - KernelPCID uint16 -} - -func init() { - KernelCodeSegment.setCode64(0, 0, 0) - KernelDataSegment.setData(0, 0xffffffff, 0) - UserCodeSegment32.setCode64(0, 0, 3) - UserDataSegment.setData(0, 0xffffffff, 3) - UserCodeSegment64.setCode64(0, 0, 3) -} - -// Emit prints architecture-specific offsets. -func Emit(w io.Writer) { - fmt.Fprintf(w, "// Automatically generated, do not edit.\n") - - c := &CPU{} - fmt.Fprintf(w, "\n// CPU offsets.\n") - fmt.Fprintf(w, "#define CPU_REGISTERS 0x%02x\n", reflect.ValueOf(&c.registers).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ERROR_CODE 0x%02x\n", reflect.ValueOf(&c.errorCode).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ERROR_TYPE 0x%02x\n", reflect.ValueOf(&c.errorType).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ENTRY 0x%02x\n", reflect.ValueOf(&c.kernelEntry).Pointer()-reflect.ValueOf(c).Pointer()) - - e := &kernelEntry{} - fmt.Fprintf(w, "\n// CPU entry offsets.\n") - fmt.Fprintf(w, "#define ENTRY_SCRATCH0 0x%02x\n", reflect.ValueOf(&e.scratch0).Pointer()-reflect.ValueOf(e).Pointer()) - fmt.Fprintf(w, "#define ENTRY_STACK_TOP 0x%02x\n", reflect.ValueOf(&e.stackTop).Pointer()-reflect.ValueOf(e).Pointer()) - fmt.Fprintf(w, "#define ENTRY_CPU_SELF 0x%02x\n", reflect.ValueOf(&e.cpuSelf).Pointer()-reflect.ValueOf(e).Pointer()) - fmt.Fprintf(w, "#define ENTRY_KERNEL_CR3 0x%02x\n", reflect.ValueOf(&e.kernelCR3).Pointer()-reflect.ValueOf(e).Pointer()) - - fmt.Fprintf(w, "\n// Bits.\n") - fmt.Fprintf(w, "#define _RFLAGS_IF 0x%02x\n", _RFLAGS_IF) - fmt.Fprintf(w, "#define _RFLAGS_IOPL0 0x%02x\n", _RFLAGS_IOPL0) - fmt.Fprintf(w, "#define _KERNEL_FLAGS 0x%02x\n", KernelFlagsSet) - - fmt.Fprintf(w, "\n// Vectors.\n") - fmt.Fprintf(w, "#define DivideByZero 0x%02x\n", DivideByZero) - fmt.Fprintf(w, "#define Debug 0x%02x\n", Debug) - fmt.Fprintf(w, "#define NMI 0x%02x\n", NMI) - fmt.Fprintf(w, "#define Breakpoint 0x%02x\n", Breakpoint) - fmt.Fprintf(w, "#define Overflow 0x%02x\n", Overflow) - fmt.Fprintf(w, "#define BoundRangeExceeded 0x%02x\n", BoundRangeExceeded) - fmt.Fprintf(w, "#define InvalidOpcode 0x%02x\n", InvalidOpcode) - fmt.Fprintf(w, "#define DeviceNotAvailable 0x%02x\n", DeviceNotAvailable) - fmt.Fprintf(w, "#define DoubleFault 0x%02x\n", DoubleFault) - fmt.Fprintf(w, "#define CoprocessorSegmentOverrun 0x%02x\n", CoprocessorSegmentOverrun) - fmt.Fprintf(w, "#define InvalidTSS 0x%02x\n", InvalidTSS) - fmt.Fprintf(w, "#define SegmentNotPresent 0x%02x\n", SegmentNotPresent) - fmt.Fprintf(w, "#define StackSegmentFault 0x%02x\n", StackSegmentFault) - fmt.Fprintf(w, "#define GeneralProtectionFault 0x%02x\n", GeneralProtectionFault) - fmt.Fprintf(w, "#define PageFault 0x%02x\n", PageFault) - fmt.Fprintf(w, "#define X87FloatingPointException 0x%02x\n", X87FloatingPointException) - fmt.Fprintf(w, "#define AlignmentCheck 0x%02x\n", AlignmentCheck) - fmt.Fprintf(w, "#define MachineCheck 0x%02x\n", MachineCheck) - fmt.Fprintf(w, "#define SIMDFloatingPointException 0x%02x\n", SIMDFloatingPointException) - fmt.Fprintf(w, "#define VirtualizationException 0x%02x\n", VirtualizationException) - fmt.Fprintf(w, "#define SecurityException 0x%02x\n", SecurityException) - fmt.Fprintf(w, "#define SyscallInt80 0x%02x\n", SyscallInt80) - fmt.Fprintf(w, "#define Syscall 0x%02x\n", Syscall) - - p := &arch.Registers{} - fmt.Fprintf(w, "\n// Ptrace registers.\n") - fmt.Fprintf(w, "#define PTRACE_R15 0x%02x\n", reflect.ValueOf(&p.R15).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R14 0x%02x\n", reflect.ValueOf(&p.R14).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R13 0x%02x\n", reflect.ValueOf(&p.R13).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R12 0x%02x\n", reflect.ValueOf(&p.R12).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RBP 0x%02x\n", reflect.ValueOf(&p.Rbp).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RBX 0x%02x\n", reflect.ValueOf(&p.Rbx).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R11 0x%02x\n", reflect.ValueOf(&p.R11).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R10 0x%02x\n", reflect.ValueOf(&p.R10).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R9 0x%02x\n", reflect.ValueOf(&p.R9).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R8 0x%02x\n", reflect.ValueOf(&p.R8).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RAX 0x%02x\n", reflect.ValueOf(&p.Rax).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RCX 0x%02x\n", reflect.ValueOf(&p.Rcx).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RDX 0x%02x\n", reflect.ValueOf(&p.Rdx).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RSI 0x%02x\n", reflect.ValueOf(&p.Rsi).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RDI 0x%02x\n", reflect.ValueOf(&p.Rdi).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_ORIGRAX 0x%02x\n", reflect.ValueOf(&p.Orig_rax).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RIP 0x%02x\n", reflect.ValueOf(&p.Rip).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_CS 0x%02x\n", reflect.ValueOf(&p.Cs).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_FLAGS 0x%02x\n", reflect.ValueOf(&p.Eflags).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RSP 0x%02x\n", reflect.ValueOf(&p.Rsp).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_SS 0x%02x\n", reflect.ValueOf(&p.Ss).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_FS 0x%02x\n", reflect.ValueOf(&p.Fs_base).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_GS 0x%02x\n", reflect.ValueOf(&p.Gs_base).Pointer()-reflect.ValueOf(p).Pointer()) -} - -// Useful bits. -const ( - _CR0_PE = 1 << 0 - _CR0_ET = 1 << 4 - _CR0_AM = 1 << 18 - _CR0_PG = 1 << 31 - - _CR4_PSE = 1 << 4 - _CR4_PAE = 1 << 5 - _CR4_PGE = 1 << 7 - _CR4_OSFXSR = 1 << 9 - _CR4_OSXMMEXCPT = 1 << 10 - _CR4_FSGSBASE = 1 << 16 - _CR4_PCIDE = 1 << 17 - _CR4_OSXSAVE = 1 << 18 - _CR4_SMEP = 1 << 20 - - _RFLAGS_AC = 1 << 18 - _RFLAGS_NT = 1 << 14 - _RFLAGS_IOPL0 = 1 << 12 - _RFLAGS_IOPL1 = 1 << 13 - _RFLAGS_IOPL = _RFLAGS_IOPL0 | _RFLAGS_IOPL1 - _RFLAGS_DF = 1 << 10 - _RFLAGS_IF = 1 << 9 - _RFLAGS_STEP = 1 << 8 - _RFLAGS_RESERVED = 1 << 1 - - _EFER_SCE = 0x001 - _EFER_LME = 0x100 - _EFER_LMA = 0x400 - _EFER_NX = 0x800 - - _MSR_STAR = 0xc0000081 - _MSR_LSTAR = 0xc0000082 - _MSR_CSTAR = 0xc0000083 - _MSR_SYSCALL_MASK = 0xc0000084 - _MSR_PLATFORM_INFO = 0xce - _MSR_MISC_FEATURES = 0x140 - - _PLATFORM_INFO_CPUID_FAULT = 1 << 31 - - _MISC_FEATURE_CPUID_TRAP = 0x1 -) - -const ( - // KernelFlagsSet should always be set in the kernel. - KernelFlagsSet = _RFLAGS_RESERVED - - // UserFlagsSet are always set in userspace. - // - // _RFLAGS_IOPL is a set of two bits and it shows the I/O privilege - // level. The Current Privilege Level (CPL) of the task must be less - // than or equal to the IOPL in order for the task or program to access - // I/O ports. - // - // Here, _RFLAGS_IOPL0 is used only to determine whether the task is - // running in the kernel or userspace mode. In the user mode, the CPL is - // always 3 and it doesn't matter what IOPL is set if it is bellow CPL. - // - // We need to have one bit which will be always different in user and - // kernel modes. And we have to remember that even though we have - // KernelFlagsClear, we still can see some of these flags in the kernel - // mode. This can happen when the goruntime switches on a goroutine - // which has been saved in the host mode. On restore, the popf - // instruction is used to restore flags and this means that all flags - // what the goroutine has in the host mode will be restored in the - // kernel mode. - // - // _RFLAGS_IOPL0 is never set in host and kernel modes and we always set - // it in the user mode. So if this flag is set, the task is running in - // the user mode and if it isn't set, the task is running in the kernel - // mode. - UserFlagsSet = _RFLAGS_RESERVED | _RFLAGS_IF | _RFLAGS_IOPL0 - - // KernelFlagsClear should always be clear in the kernel. - KernelFlagsClear = _RFLAGS_STEP | _RFLAGS_IF | _RFLAGS_IOPL | _RFLAGS_AC | _RFLAGS_NT - - // UserFlagsClear are always cleared in userspace. - UserFlagsClear = _RFLAGS_NT | _RFLAGS_IOPL1 -) - -// IsKernelFlags returns true if rflags coresponds to the kernel mode. -// -// go:nosplit -func IsKernelFlags(rflags uint64) bool { - return rflags&_RFLAGS_IOPL0 == 0 -} - -// Vector is an exception vector. -type Vector uintptr - -// Exception vectors. -const ( - DivideByZero Vector = iota - Debug - NMI - Breakpoint - Overflow - BoundRangeExceeded - InvalidOpcode - DeviceNotAvailable - DoubleFault - CoprocessorSegmentOverrun - InvalidTSS - SegmentNotPresent - StackSegmentFault - GeneralProtectionFault - PageFault - _ - X87FloatingPointException - AlignmentCheck - MachineCheck - SIMDFloatingPointException - VirtualizationException - SecurityException = 0x1e - SyscallInt80 = 0x80 - _NR_INTERRUPTS = 0x100 -) - -// System call vectors. -const ( - Syscall Vector = _NR_INTERRUPTS -) - -// VirtualAddressBits returns the number bits available for virtual addresses. -// -// Note that sign-extension semantics apply to the highest order bit. -// -// FIXME(b/69382326): This should use the cpuid passed to Init. -func VirtualAddressBits() uint32 { - ax, _, _, _ := cpuid.HostID(0x80000008, 0) - return (ax >> 8) & 0xff -} - -// PhysicalAddressBits returns the number of bits available for physical addresses. -// -// FIXME(b/69382326): This should use the cpuid passed to Init. -func PhysicalAddressBits() uint32 { - ax, _, _, _ := cpuid.HostID(0x80000008, 0) - return ax & 0xff -} - -// Selector is a segment Selector. -type Selector uint16 - -// SegmentDescriptor is a segment descriptor. -type SegmentDescriptor struct { - bits [2]uint32 -} - -// descriptorTable is a collection of descriptors. -type descriptorTable [32]SegmentDescriptor - -// SegmentDescriptorFlags are typed flags within a descriptor. -type SegmentDescriptorFlags uint32 - -// SegmentDescriptorFlag declarations. -const ( - SegmentDescriptorAccess SegmentDescriptorFlags = 1 << 8 // Access bit (always set). - SegmentDescriptorWrite = 1 << 9 // Write permission. - SegmentDescriptorExpandDown = 1 << 10 // Grows down, not used. - SegmentDescriptorExecute = 1 << 11 // Execute permission. - SegmentDescriptorSystem = 1 << 12 // Zero => system, 1 => user code/data. - SegmentDescriptorPresent = 1 << 15 // Present. - SegmentDescriptorAVL = 1 << 20 // Available. - SegmentDescriptorLong = 1 << 21 // Long mode. - SegmentDescriptorDB = 1 << 22 // 16 or 32-bit. - SegmentDescriptorG = 1 << 23 // Granularity: page or byte. -) - -// Base returns the descriptor's base linear address. -func (d *SegmentDescriptor) Base() uint32 { - return d.bits[1]&0xFF000000 | (d.bits[1]&0x000000FF)<<16 | d.bits[0]>>16 -} - -// Limit returns the descriptor size. -func (d *SegmentDescriptor) Limit() uint32 { - l := d.bits[0]&0xFFFF | d.bits[1]&0xF0000 - if d.bits[1]&uint32(SegmentDescriptorG) != 0 { - l <<= 12 - l |= 0xFFF - } - return l -} - -// Flags returns descriptor flags. -func (d *SegmentDescriptor) Flags() SegmentDescriptorFlags { - return SegmentDescriptorFlags(d.bits[1] & 0x00F09F00) -} - -// DPL returns the descriptor privilege level. -func (d *SegmentDescriptor) DPL() int { - return int((d.bits[1] >> 13) & 3) -} - -func (d *SegmentDescriptor) setNull() { - d.bits[0] = 0 - d.bits[1] = 0 -} - -func (d *SegmentDescriptor) set(base, limit uint32, dpl int, flags SegmentDescriptorFlags) { - flags |= SegmentDescriptorPresent - if limit>>12 != 0 { - limit >>= 12 - flags |= SegmentDescriptorG - } - d.bits[0] = base<<16 | limit&0xFFFF - d.bits[1] = base&0xFF000000 | (base>>16)&0xFF | limit&0x000F0000 | uint32(flags) | uint32(dpl)<<13 -} - -func (d *SegmentDescriptor) setCode32(base, limit uint32, dpl int) { - d.set(base, limit, dpl, - SegmentDescriptorDB| - SegmentDescriptorExecute| - SegmentDescriptorSystem) -} - -func (d *SegmentDescriptor) setCode64(base, limit uint32, dpl int) { - d.set(base, limit, dpl, - SegmentDescriptorG| - SegmentDescriptorLong| - SegmentDescriptorExecute| - SegmentDescriptorSystem) -} - -func (d *SegmentDescriptor) setData(base, limit uint32, dpl int) { - d.set(base, limit, dpl, - SegmentDescriptorWrite| - SegmentDescriptorSystem) -} - -// setHi is only used for the TSS segment, which is magically 64-bits. -func (d *SegmentDescriptor) setHi(base uint32) { - d.bits[0] = base - d.bits[1] = 0 -} - -// Gate64 is a 64-bit task, trap, or interrupt gate. -type Gate64 struct { - bits [4]uint32 -} - -// idt64 is a 64-bit interrupt descriptor table. -type idt64 [_NR_INTERRUPTS]Gate64 - -func (g *Gate64) setInterrupt(cs Selector, rip uint64, dpl int, ist int) { - g.bits[0] = uint32(cs)<<16 | uint32(rip)&0xFFFF - g.bits[1] = uint32(rip)&0xFFFF0000 | SegmentDescriptorPresent | uint32(dpl)<<13 | 14<<8 | uint32(ist)&0x7 - g.bits[2] = uint32(rip >> 32) -} - -func (g *Gate64) setTrap(cs Selector, rip uint64, dpl int, ist int) { - g.setInterrupt(cs, rip, dpl, ist) - g.bits[1] |= 1 << 8 -} - -// TaskState64 is a 64-bit task state structure. -type TaskState64 struct { - _ uint32 - rsp0Lo, rsp0Hi uint32 - rsp1Lo, rsp1Hi uint32 - rsp2Lo, rsp2Hi uint32 - _ [2]uint32 - ist1Lo, ist1Hi uint32 - ist2Lo, ist2Hi uint32 - ist3Lo, ist3Hi uint32 - ist4Lo, ist4Hi uint32 - ist5Lo, ist5Hi uint32 - ist6Lo, ist6Hi uint32 - ist7Lo, ist7Hi uint32 - _ [2]uint32 - _ uint16 - ioPerm uint16 -} diff --git a/pkg/sentry/platform/ring0/defs_impl_arm64.go b/pkg/sentry/platform/ring0/defs_impl_arm64.go deleted file mode 100644 index 92c64fe1f..000000000 --- a/pkg/sentry/platform/ring0/defs_impl_arm64.go +++ /dev/null @@ -1,428 +0,0 @@ -// +build arm64 -// +build arm64 -// +build arm64 - -package ring0 - -import ( - "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables" - - "fmt" - "gvisor.dev/gvisor/pkg/usermem" - "io" - "reflect" -) - -// Useful bits. -const ( - _PGD_PGT_BASE = 0x1000 - _PGD_PGT_SIZE = 0x1000 - _PUD_PGT_BASE = 0x2000 - _PUD_PGT_SIZE = 0x1000 - _PMD_PGT_BASE = 0x3000 - _PMD_PGT_SIZE = 0x4000 - _PTE_PGT_BASE = 0x7000 - _PTE_PGT_SIZE = 0x1000 -) - -const ( - // DAIF bits:debug, sError, IRQ, FIQ. - _PSR_D_BIT = 0x00000200 - _PSR_A_BIT = 0x00000100 - _PSR_I_BIT = 0x00000080 - _PSR_F_BIT = 0x00000040 - _PSR_DAIF_SHIFT = 6 - _PSR_DAIF_MASK = 0xf << _PSR_DAIF_SHIFT - - // PSR bits. - _PSR_MODE_EL0t = 0x00000000 - _PSR_MODE_EL1t = 0x00000004 - _PSR_MODE_EL1h = 0x00000005 - _PSR_MODE_MASK = 0x0000000f - - PsrFlagsClear = _PSR_MODE_MASK | _PSR_DAIF_MASK - PsrModeMask = _PSR_MODE_MASK - - // KernelFlagsSet should always be set in the kernel. - KernelFlagsSet = _PSR_MODE_EL1h | _PSR_D_BIT | _PSR_A_BIT | _PSR_I_BIT | _PSR_F_BIT - - // UserFlagsSet are always set in userspace. - UserFlagsSet = _PSR_MODE_EL0t -) - -// Vector is an exception vector. -type Vector uintptr - -// Exception vectors. -const ( - El1InvSync = iota - El1InvIrq - El1InvFiq - El1InvError - - El1Sync - El1Irq - El1Fiq - El1Err - - El0Sync - El0Irq - El0Fiq - El0Err - - El0InvSync - El0InvIrq - El0InvFiq - El0InvErr - - El1SyncDa - El1SyncIa - El1SyncSpPc - El1SyncUndef - El1SyncDbg - El1SyncInv - - El0SyncSVC - El0SyncDa - El0SyncIa - El0SyncFpsimdAcc - El0SyncSveAcc - El0SyncFpsimdExc - El0SyncSys - El0SyncSpPc - El0SyncUndef - El0SyncDbg - El0SyncWfx - El0SyncInv - - El0ErrNMI - El0ErrBounce - - _NR_INTERRUPTS -) - -// System call vectors. -const ( - Syscall Vector = El0SyncSVC - PageFault Vector = El0SyncDa - VirtualizationException Vector = El0ErrBounce -) - -// VirtualAddressBits returns the number bits available for virtual addresses. -func VirtualAddressBits() uint32 { - return 48 -} - -// PhysicalAddressBits returns the number of bits available for physical addresses. -func PhysicalAddressBits() uint32 { - return 40 -} - -// Kernel is a global kernel object. -// -// This contains global state, shared by multiple CPUs. -type Kernel struct { - // PageTables are the kernel pagetables; this must be provided. - PageTables *pagetables.PageTables - - KernelArchState -} - -// Hooks are hooks for kernel functions. -type Hooks interface { - // KernelSyscall is called for kernel system calls. - // - // Return from this call will restore registers and return to the kernel: the - // registers must be modified directly. - // - // If this function is not provided, a kernel exception results in halt. - // - // This must be go:nosplit, as this will be on the interrupt stack. - // Closures are permitted, as the pointer to the closure frame is not - // passed on the stack. - KernelSyscall() - - // KernelException handles an exception during kernel execution. - // - // Return from this call will restore registers and return to the kernel: the - // registers must be modified directly. - // - // If this function is not provided, a kernel exception results in halt. - // - // This must be go:nosplit, as this will be on the interrupt stack. - // Closures are permitted, as the pointer to the closure frame is not - // passed on the stack. - KernelException(Vector) -} - -// CPU is the per-CPU struct. -type CPU struct { - // self is a self reference. - // - // This is always guaranteed to be at offset zero. - self *CPU - - // kernel is reference to the kernel that this CPU was initialized - // with. This reference is kept for garbage collection purposes: CPU - // registers may refer to objects within the Kernel object that cannot - // be safely freed. - kernel *Kernel - - // CPUArchState is architecture-specific state. - CPUArchState - - // registers is a set of registers; these may be used on kernel system - // calls and exceptions via the Registers function. - registers arch.Registers - - // hooks are kernel hooks. - hooks Hooks -} - -// Registers returns a modifiable-copy of the kernel registers. -// -// This is explicitly safe to call during KernelException and KernelSyscall. -// -//go:nosplit -func (c *CPU) Registers() *arch.Registers { - return &c.registers -} - -// SwitchOpts are passed to the Switch function. -type SwitchOpts struct { - // Registers are the user register state. - Registers *arch.Registers - - // FloatingPointState is a byte pointer where floating point state is - // saved and restored. - FloatingPointState *byte - - // PageTables are the application page tables. - PageTables *pagetables.PageTables - - // Flush indicates that a TLB flush should be forced on switch. - Flush bool - - // FullRestore indicates that an iret-based restore should be used. - FullRestore bool - - // SwitchArchOpts are architecture-specific options. - SwitchArchOpts -} - -var ( - // UserspaceSize is the total size of userspace. - UserspaceSize = uintptr(1) << (VirtualAddressBits()) - - // MaximumUserAddress is the largest possible user address. - MaximumUserAddress = (UserspaceSize - 1) & ^uintptr(usermem.PageSize-1) - - // KernelStartAddress is the starting kernel address. - KernelStartAddress = ^uintptr(0) - (UserspaceSize - 1) -) - -// KernelArchState contains architecture-specific state. -type KernelArchState struct { -} - -// CPUArchState contains CPU-specific arch state. -type CPUArchState struct { - // stack is the stack used for interrupts on this CPU. - stack [512]byte - - // errorCode is the error code from the last exception. - errorCode uintptr - - // errorType indicates the type of error code here, it is always set - // along with the errorCode value above. - // - // It will either by 1, which indicates a user error, or 0 indicating a - // kernel error. If the error code below returns false (kernel error), - // then it cannot provide relevant information about the last - // exception. - errorType uintptr - - // faultAddr is the value of far_el1. - faultAddr uintptr - - // ttbr0Kvm is the value of ttbr0_el1 for sentry. - ttbr0Kvm uintptr - - // ttbr0App is the value of ttbr0_el1 for applicaton. - ttbr0App uintptr - - // exception vector. - vecCode Vector - - // application context pointer. - appAddr uintptr - - // lazyVFP is the value of cpacr_el1. - lazyVFP uintptr - - // appASID is the asid value of guest application. - appASID uintptr -} - -// ErrorCode returns the last error code. -// -// The returned boolean indicates whether the error code corresponds to the -// last user error or not. If it does not, then fault information must be -// ignored. This is generally the result of a kernel fault while servicing a -// user fault. -// -//go:nosplit -func (c *CPU) ErrorCode() (value uintptr, user bool) { - return c.errorCode, c.errorType != 0 -} - -// ClearErrorCode resets the error code. -// -//go:nosplit -func (c *CPU) ClearErrorCode() { - c.errorCode = 0 - c.errorType = 1 -} - -//go:nosplit -func (c *CPU) GetFaultAddr() (value uintptr) { - return c.faultAddr -} - -//go:nosplit -func (c *CPU) SetTtbr0Kvm(value uintptr) { - c.ttbr0Kvm = value -} - -//go:nosplit -func (c *CPU) SetTtbr0App(value uintptr) { - c.ttbr0App = value -} - -//go:nosplit -func (c *CPU) GetVector() (value Vector) { - return c.vecCode -} - -//go:nosplit -func (c *CPU) SetAppAddr(value uintptr) { - c.appAddr = value -} - -// GetLazyVFP returns the value of cpacr_el1. -//go:nosplit -func (c *CPU) GetLazyVFP() (value uintptr) { - return c.lazyVFP -} - -// SwitchArchOpts are embedded in SwitchOpts. -type SwitchArchOpts struct { - // UserASID indicates that the application ASID to be used on switch, - UserASID uint16 - - // KernelASID indicates that the kernel ASID to be used on return, - KernelASID uint16 -} - -func init() { -} - -// Emit prints architecture-specific offsets. -func Emit(w io.Writer) { - fmt.Fprintf(w, "// Automatically generated, do not edit.\n") - - c := &CPU{} - fmt.Fprintf(w, "\n// CPU offsets.\n") - fmt.Fprintf(w, "#define CPU_SELF 0x%02x\n", reflect.ValueOf(&c.self).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_REGISTERS 0x%02x\n", reflect.ValueOf(&c.registers).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_STACK_TOP 0x%02x\n", reflect.ValueOf(&c.stack[0]).Pointer()-reflect.ValueOf(c).Pointer()+uintptr(len(c.stack))) - fmt.Fprintf(w, "#define CPU_ERROR_CODE 0x%02x\n", reflect.ValueOf(&c.errorCode).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ERROR_TYPE 0x%02x\n", reflect.ValueOf(&c.errorType).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_FAULT_ADDR 0x%02x\n", reflect.ValueOf(&c.faultAddr).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_TTBR0_KVM 0x%02x\n", reflect.ValueOf(&c.ttbr0Kvm).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_TTBR0_APP 0x%02x\n", reflect.ValueOf(&c.ttbr0App).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_VECTOR_CODE 0x%02x\n", reflect.ValueOf(&c.vecCode).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_APP_ADDR 0x%02x\n", reflect.ValueOf(&c.appAddr).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_LAZY_VFP 0x%02x\n", reflect.ValueOf(&c.lazyVFP).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_APP_ASID 0x%02x\n", reflect.ValueOf(&c.appASID).Pointer()-reflect.ValueOf(c).Pointer()) - - fmt.Fprintf(w, "\n// Bits.\n") - fmt.Fprintf(w, "#define _KERNEL_FLAGS 0x%02x\n", KernelFlagsSet) - - fmt.Fprintf(w, "\n// Vectors.\n") - - fmt.Fprintf(w, "#define El1Sync 0x%02x\n", El1Sync) - fmt.Fprintf(w, "#define El1Irq 0x%02x\n", El1Irq) - fmt.Fprintf(w, "#define El1Fiq 0x%02x\n", El1Fiq) - fmt.Fprintf(w, "#define El1Err 0x%02x\n", El1Err) - - fmt.Fprintf(w, "#define El0Sync 0x%02x\n", El0Sync) - fmt.Fprintf(w, "#define El0Irq 0x%02x\n", El0Irq) - fmt.Fprintf(w, "#define El0Fiq 0x%02x\n", El0Fiq) - fmt.Fprintf(w, "#define El0Err 0x%02x\n", El0Err) - - fmt.Fprintf(w, "#define El1SyncDa 0x%02x\n", El1SyncDa) - fmt.Fprintf(w, "#define El1SyncIa 0x%02x\n", El1SyncIa) - fmt.Fprintf(w, "#define El1SyncSpPc 0x%02x\n", El1SyncSpPc) - fmt.Fprintf(w, "#define El1SyncUndef 0x%02x\n", El1SyncUndef) - fmt.Fprintf(w, "#define El1SyncDbg 0x%02x\n", El1SyncDbg) - fmt.Fprintf(w, "#define El1SyncInv 0x%02x\n", El1SyncInv) - - fmt.Fprintf(w, "#define El0SyncSVC 0x%02x\n", El0SyncSVC) - fmt.Fprintf(w, "#define El0SyncDa 0x%02x\n", El0SyncDa) - fmt.Fprintf(w, "#define El0SyncIa 0x%02x\n", El0SyncIa) - fmt.Fprintf(w, "#define El0SyncFpsimdAcc 0x%02x\n", El0SyncFpsimdAcc) - fmt.Fprintf(w, "#define El0SyncSveAcc 0x%02x\n", El0SyncSveAcc) - fmt.Fprintf(w, "#define El0SyncFpsimdExc 0x%02x\n", El0SyncFpsimdExc) - fmt.Fprintf(w, "#define El0SyncSys 0x%02x\n", El0SyncSys) - fmt.Fprintf(w, "#define El0SyncSpPc 0x%02x\n", El0SyncSpPc) - fmt.Fprintf(w, "#define El0SyncUndef 0x%02x\n", El0SyncUndef) - fmt.Fprintf(w, "#define El0SyncDbg 0x%02x\n", El0SyncDbg) - fmt.Fprintf(w, "#define El0SyncWfx 0x%02x\n", El0SyncWfx) - fmt.Fprintf(w, "#define El0SyncInv 0x%02x\n", El0SyncInv) - - fmt.Fprintf(w, "#define El0ErrNMI 0x%02x\n", El0ErrNMI) - - fmt.Fprintf(w, "#define PageFault 0x%02x\n", PageFault) - fmt.Fprintf(w, "#define Syscall 0x%02x\n", Syscall) - fmt.Fprintf(w, "#define VirtualizationException 0x%02x\n", VirtualizationException) - - p := &arch.Registers{} - fmt.Fprintf(w, "\n// Ptrace registers.\n") - fmt.Fprintf(w, "#define PTRACE_R0 0x%02x\n", reflect.ValueOf(&p.Regs[0]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R1 0x%02x\n", reflect.ValueOf(&p.Regs[1]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R2 0x%02x\n", reflect.ValueOf(&p.Regs[2]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R3 0x%02x\n", reflect.ValueOf(&p.Regs[3]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R4 0x%02x\n", reflect.ValueOf(&p.Regs[4]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R5 0x%02x\n", reflect.ValueOf(&p.Regs[5]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R6 0x%02x\n", reflect.ValueOf(&p.Regs[6]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R7 0x%02x\n", reflect.ValueOf(&p.Regs[7]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R8 0x%02x\n", reflect.ValueOf(&p.Regs[8]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R9 0x%02x\n", reflect.ValueOf(&p.Regs[9]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R10 0x%02x\n", reflect.ValueOf(&p.Regs[10]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R11 0x%02x\n", reflect.ValueOf(&p.Regs[11]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R12 0x%02x\n", reflect.ValueOf(&p.Regs[12]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R13 0x%02x\n", reflect.ValueOf(&p.Regs[13]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R14 0x%02x\n", reflect.ValueOf(&p.Regs[14]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R15 0x%02x\n", reflect.ValueOf(&p.Regs[15]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R16 0x%02x\n", reflect.ValueOf(&p.Regs[16]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R17 0x%02x\n", reflect.ValueOf(&p.Regs[17]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R18 0x%02x\n", reflect.ValueOf(&p.Regs[18]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R19 0x%02x\n", reflect.ValueOf(&p.Regs[19]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R20 0x%02x\n", reflect.ValueOf(&p.Regs[20]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R21 0x%02x\n", reflect.ValueOf(&p.Regs[21]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R22 0x%02x\n", reflect.ValueOf(&p.Regs[22]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R23 0x%02x\n", reflect.ValueOf(&p.Regs[23]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R24 0x%02x\n", reflect.ValueOf(&p.Regs[24]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R25 0x%02x\n", reflect.ValueOf(&p.Regs[25]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R26 0x%02x\n", reflect.ValueOf(&p.Regs[26]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R27 0x%02x\n", reflect.ValueOf(&p.Regs[27]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R28 0x%02x\n", reflect.ValueOf(&p.Regs[28]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R29 0x%02x\n", reflect.ValueOf(&p.Regs[29]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R30 0x%02x\n", reflect.ValueOf(&p.Regs[30]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_SP 0x%02x\n", reflect.ValueOf(&p.Sp).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_PC 0x%02x\n", reflect.ValueOf(&p.Pc).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_PSTATE 0x%02x\n", reflect.ValueOf(&p.Pstate).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_TLS 0x%02x\n", reflect.ValueOf(&p.TPIDR_EL0).Pointer()-reflect.ValueOf(p).Pointer()) -} diff --git a/pkg/sentry/platform/ring0/entry_amd64.go b/pkg/sentry/platform/ring0/entry_amd64.go deleted file mode 100644 index d87b1fd00..000000000 --- a/pkg/sentry/platform/ring0/entry_amd64.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package ring0 - -import ( - "gvisor.dev/gvisor/pkg/sentry/arch" -) - -// This is an assembly function. -// -// The sysenter function is invoked in two situations: -// -// (1) The guest kernel has executed a system call. -// (2) The guest application has executed a system call. -// -// The interrupt flag is examined to determine whether the system call was -// executed from kernel mode or not and the appropriate stub is called. -func sysenter() - -// swapgs swaps the current GS value. -// -// This must be called prior to sysret/iret. -func swapgs() - -// jumpToKernel jumps to the kernel version of the current RIP. -func jumpToKernel() - -// sysret returns to userspace from a system call. -// -// The return code is the vector that interrupted execution. -// -// See stubs.go for a note regarding the frame size of this function. -func sysret(cpu *CPU, regs *arch.Registers, userCR3 uintptr) Vector - -// "iret is the cadillac of CPL switching." -// -// -- Neel Natu -// -// iret is nearly identical to sysret, except an iret is used to fully restore -// all user state. This must be called in cases where all registers need to be -// restored. -func iret(cpu *CPU, regs *arch.Registers, userCR3 uintptr) Vector - -// exception is the generic exception entry. -// -// This is called by the individual stub definitions. -func exception() - -// resume is a stub that restores the CPU kernel registers. -// -// This is used when processing kernel exceptions and syscalls. -func resume() - -// Start is the CPU entrypoint. -// -// The following start conditions must be satisfied: -// -// * AX should contain the CPU pointer. -// * c.GDT() should be loaded as the GDT. -// * c.IDT() should be loaded as the IDT. -// * c.CR0() should be the current CR0 value. -// * c.CR3() should be set to the kernel PageTables. -// * c.CR4() should be the current CR4 value. -// * c.EFER() should be the current EFER value. -// -// The CPU state will be set to c.Registers(). -func Start() - -// Exception stubs. -func divideByZero() -func debug() -func nmi() -func breakpoint() -func overflow() -func boundRangeExceeded() -func invalidOpcode() -func deviceNotAvailable() -func doubleFault() -func coprocessorSegmentOverrun() -func invalidTSS() -func segmentNotPresent() -func stackSegmentFault() -func generalProtectionFault() -func pageFault() -func x87FloatingPointException() -func alignmentCheck() -func machineCheck() -func simdFloatingPointException() -func virtualizationException() -func securityException() -func syscallInt80() - -// Exception handler index. -var handlers = map[Vector]func(){ - DivideByZero: divideByZero, - Debug: debug, - NMI: nmi, - Breakpoint: breakpoint, - Overflow: overflow, - BoundRangeExceeded: boundRangeExceeded, - InvalidOpcode: invalidOpcode, - DeviceNotAvailable: deviceNotAvailable, - DoubleFault: doubleFault, - CoprocessorSegmentOverrun: coprocessorSegmentOverrun, - InvalidTSS: invalidTSS, - SegmentNotPresent: segmentNotPresent, - StackSegmentFault: stackSegmentFault, - GeneralProtectionFault: generalProtectionFault, - PageFault: pageFault, - X87FloatingPointException: x87FloatingPointException, - AlignmentCheck: alignmentCheck, - MachineCheck: machineCheck, - SIMDFloatingPointException: simdFloatingPointException, - VirtualizationException: virtualizationException, - SecurityException: securityException, - SyscallInt80: syscallInt80, -} diff --git a/pkg/sentry/platform/ring0/entry_arm64.go b/pkg/sentry/platform/ring0/entry_arm64.go deleted file mode 100644 index 62a93f3d6..000000000 --- a/pkg/sentry/platform/ring0/entry_arm64.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -package ring0 - -// This is an assembly function. -// -// The sysenter function is invoked in two situations: -// -// (1) The guest kernel has executed a system call. -// (2) The guest application has executed a system call. -// -// The interrupt flag is examined to determine whether the system call was -// executed from kernel mode or not and the appropriate stub is called. - -func El1_sync_invalid() -func El1_irq_invalid() -func El1_fiq_invalid() -func El1_error_invalid() - -func El1_sync() -func El1_irq() -func El1_fiq() -func El1_error() - -func El0_sync() -func El0_irq() -func El0_fiq() -func El0_error() - -func El0_sync_invalid() -func El0_irq_invalid() -func El0_fiq_invalid() -func El0_error_invalid() - -func Vectors() - -// Start is the CPU entrypoint. -// -// The CPU state will be set to c.Registers(). -func Start() -func kernelExitToEl1() - -func kernelExitToEl0() - -// Shutdown execution -func Shutdown() diff --git a/pkg/sentry/platform/ring0/entry_impl_amd64.s b/pkg/sentry/platform/ring0/entry_impl_amd64.s deleted file mode 100644 index 9a5d6c064..000000000 --- a/pkg/sentry/platform/ring0/entry_impl_amd64.s +++ /dev/null @@ -1,441 +0,0 @@ -// build +amd64 - -// Automatically generated, do not edit. - -// CPU offsets. -#define CPU_REGISTERS 0x28 -#define CPU_ERROR_CODE 0x10 -#define CPU_ERROR_TYPE 0x18 -#define CPU_ENTRY 0x20 - -// CPU entry offsets. -#define ENTRY_SCRATCH0 0x100 -#define ENTRY_STACK_TOP 0x108 -#define ENTRY_CPU_SELF 0x110 -#define ENTRY_KERNEL_CR3 0x118 - -// Bits. -#define _RFLAGS_IF 0x200 -#define _RFLAGS_IOPL0 0x1000 -#define _KERNEL_FLAGS 0x02 - -// Vectors. -#define DivideByZero 0x00 -#define Debug 0x01 -#define NMI 0x02 -#define Breakpoint 0x03 -#define Overflow 0x04 -#define BoundRangeExceeded 0x05 -#define InvalidOpcode 0x06 -#define DeviceNotAvailable 0x07 -#define DoubleFault 0x08 -#define CoprocessorSegmentOverrun 0x09 -#define InvalidTSS 0x0a -#define SegmentNotPresent 0x0b -#define StackSegmentFault 0x0c -#define GeneralProtectionFault 0x0d -#define PageFault 0x0e -#define X87FloatingPointException 0x10 -#define AlignmentCheck 0x11 -#define MachineCheck 0x12 -#define SIMDFloatingPointException 0x13 -#define VirtualizationException 0x14 -#define SecurityException 0x1e -#define SyscallInt80 0x80 -#define Syscall 0x100 - -// Ptrace registers. -#define PTRACE_R15 0x00 -#define PTRACE_R14 0x08 -#define PTRACE_R13 0x10 -#define PTRACE_R12 0x18 -#define PTRACE_RBP 0x20 -#define PTRACE_RBX 0x28 -#define PTRACE_R11 0x30 -#define PTRACE_R10 0x38 -#define PTRACE_R9 0x40 -#define PTRACE_R8 0x48 -#define PTRACE_RAX 0x50 -#define PTRACE_RCX 0x58 -#define PTRACE_RDX 0x60 -#define PTRACE_RSI 0x68 -#define PTRACE_RDI 0x70 -#define PTRACE_ORIGRAX 0x78 -#define PTRACE_RIP 0x80 -#define PTRACE_CS 0x88 -#define PTRACE_FLAGS 0x90 -#define PTRACE_RSP 0x98 -#define PTRACE_SS 0xa0 -#define PTRACE_FS 0xa8 -#define PTRACE_GS 0xb0 -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "funcdata.h" -#include "textflag.h" - -// NB: Offsets are programmatically generated (see BUILD). -// -// This file is concatenated with the definitions. - -// Saves a register set. -// -// This is a macro because it may need to executed in contents where a stack is -// not available for calls. -// -// The following registers are not saved: AX, SP, IP, FLAGS, all segments. -#define REGISTERS_SAVE(reg, offset) \ - MOVQ R15, offset+PTRACE_R15(reg); \ - MOVQ R14, offset+PTRACE_R14(reg); \ - MOVQ R13, offset+PTRACE_R13(reg); \ - MOVQ R12, offset+PTRACE_R12(reg); \ - MOVQ BP, offset+PTRACE_RBP(reg); \ - MOVQ BX, offset+PTRACE_RBX(reg); \ - MOVQ CX, offset+PTRACE_RCX(reg); \ - MOVQ DX, offset+PTRACE_RDX(reg); \ - MOVQ R11, offset+PTRACE_R11(reg); \ - MOVQ R10, offset+PTRACE_R10(reg); \ - MOVQ R9, offset+PTRACE_R9(reg); \ - MOVQ R8, offset+PTRACE_R8(reg); \ - MOVQ SI, offset+PTRACE_RSI(reg); \ - MOVQ DI, offset+PTRACE_RDI(reg); - -// Loads a register set. -// -// This is a macro because it may need to executed in contents where a stack is -// not available for calls. -// -// The following registers are not loaded: AX, SP, IP, FLAGS, all segments. -#define REGISTERS_LOAD(reg, offset) \ - MOVQ offset+PTRACE_R15(reg), R15; \ - MOVQ offset+PTRACE_R14(reg), R14; \ - MOVQ offset+PTRACE_R13(reg), R13; \ - MOVQ offset+PTRACE_R12(reg), R12; \ - MOVQ offset+PTRACE_RBP(reg), BP; \ - MOVQ offset+PTRACE_RBX(reg), BX; \ - MOVQ offset+PTRACE_RCX(reg), CX; \ - MOVQ offset+PTRACE_RDX(reg), DX; \ - MOVQ offset+PTRACE_R11(reg), R11; \ - MOVQ offset+PTRACE_R10(reg), R10; \ - MOVQ offset+PTRACE_R9(reg), R9; \ - MOVQ offset+PTRACE_R8(reg), R8; \ - MOVQ offset+PTRACE_RSI(reg), SI; \ - MOVQ offset+PTRACE_RDI(reg), DI; - -// WRITE_CR3() writes the given CR3 value. -// -// The code corresponds to: -// -// mov %rax, %cr3 -// -#define WRITE_CR3() \ - BYTE $0x0f; BYTE $0x22; BYTE $0xd8; - -// SWAP_GS swaps the kernel GS (CPU). -#define SWAP_GS() \ - BYTE $0x0F; BYTE $0x01; BYTE $0xf8; - -// IRET returns from an interrupt frame. -#define IRET() \ - BYTE $0x48; BYTE $0xcf; - -// SYSRET64 executes the sysret instruction. -#define SYSRET64() \ - BYTE $0x48; BYTE $0x0f; BYTE $0x07; - -// LOAD_KERNEL_STACK loads the kernel stack. -#define LOAD_KERNEL_STACK(entry) \ - MOVQ ENTRY_STACK_TOP(entry), SP; - -// See kernel.go. -TEXT ·Halt(SB),NOSPLIT,$0 - HLT - RET - -// See entry_amd64.go. -TEXT ·swapgs(SB),NOSPLIT,$0 - SWAP_GS() - RET - -// jumpToKernel changes execution to the kernel address space. -// -// This works by changing the return value to the kernel version. -TEXT ·jumpToKernel(SB),NOSPLIT,$0 - MOVQ 0(SP), AX - ORQ ·KernelStartAddress(SB), AX // Future return value. - MOVQ AX, 0(SP) - RET - -// See entry_amd64.go. -TEXT ·sysret(SB),NOSPLIT,$0-24 - CALL ·jumpToKernel(SB) - // Save original state and stack. sysenter() or exception() - // from APP(gr3) will switch to this stack, set the return - // value (vector: 32(SP)) and then do RET, which will also - // automatically return to the lower half. - MOVQ cpu+0(FP), BX - MOVQ regs+8(FP), AX - MOVQ userCR3+16(FP), CX - MOVQ SP, CPU_REGISTERS+PTRACE_RSP(BX) - MOVQ BP, CPU_REGISTERS+PTRACE_RBP(BX) - MOVQ AX, CPU_REGISTERS+PTRACE_RAX(BX) - - // save SP AX userCR3 on the kernel stack. - MOVQ CPU_ENTRY(BX), BX - LOAD_KERNEL_STACK(BX) - PUSHQ PTRACE_RSP(AX) - PUSHQ PTRACE_RAX(AX) - PUSHQ CX - - // Restore user register state. - REGISTERS_LOAD(AX, 0) - MOVQ PTRACE_RIP(AX), CX // Needed for SYSRET. - MOVQ PTRACE_FLAGS(AX), R11 // Needed for SYSRET. - - // restore userCR3, AX, SP. - POPQ AX // Get userCR3. - WRITE_CR3() // Switch to userCR3. - POPQ AX // Restore AX. - POPQ SP // Restore SP. - SYSRET64() - -// See entry_amd64.go. -TEXT ·iret(SB),NOSPLIT,$0-24 - CALL ·jumpToKernel(SB) - // Save original state and stack. sysenter() or exception() - // from APP(gr3) will switch to this stack, set the return - // value (vector: 32(SP)) and then do RET, which will also - // automatically return to the lower half. - MOVQ cpu+0(FP), BX - MOVQ regs+8(FP), AX - MOVQ userCR3+16(FP), CX - MOVQ SP, CPU_REGISTERS+PTRACE_RSP(BX) - MOVQ BP, CPU_REGISTERS+PTRACE_RBP(BX) - MOVQ AX, CPU_REGISTERS+PTRACE_RAX(BX) - - // Build an IRET frame & restore state. - MOVQ CPU_ENTRY(BX), BX - LOAD_KERNEL_STACK(BX) - PUSHQ PTRACE_SS(AX) - PUSHQ PTRACE_RSP(AX) - PUSHQ PTRACE_FLAGS(AX) - PUSHQ PTRACE_CS(AX) - PUSHQ PTRACE_RIP(AX) - PUSHQ PTRACE_RAX(AX) // Save AX on kernel stack. - PUSHQ CX // Save userCR3 on kernel stack. - REGISTERS_LOAD(AX, 0) // Restore most registers. - POPQ AX // Get userCR3. - WRITE_CR3() // Switch to userCR3. - POPQ AX // Restore AX. - IRET() - -// See entry_amd64.go. -TEXT ·resume(SB),NOSPLIT,$0 - // See iret, above. - MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU. - PUSHQ CPU_REGISTERS+PTRACE_SS(AX) - PUSHQ CPU_REGISTERS+PTRACE_RSP(AX) - PUSHQ CPU_REGISTERS+PTRACE_FLAGS(AX) - PUSHQ CPU_REGISTERS+PTRACE_CS(AX) - PUSHQ CPU_REGISTERS+PTRACE_RIP(AX) - REGISTERS_LOAD(AX, CPU_REGISTERS) - MOVQ CPU_REGISTERS+PTRACE_RAX(AX), AX - IRET() - -// See entry_amd64.go. -TEXT ·Start(SB),NOSPLIT,$0 - PUSHQ $0x0 // Previous frame pointer. - MOVQ SP, BP // Set frame pointer. - PUSHQ AX // First argument (CPU). - CALL ·start(SB) // Call Go hook. - JMP ·resume(SB) // Restore to registers. - -// See entry_amd64.go. -TEXT ·sysenter(SB),NOSPLIT,$0 - // _RFLAGS_IOPL0 is always set in the user mode and it is never set in - // the kernel mode. See the comment of UserFlagsSet for more details. - TESTL $_RFLAGS_IOPL0, R11 - JZ kernel -user: - SWAP_GS() - MOVQ AX, ENTRY_SCRATCH0(GS) // Save user AX on scratch. - MOVQ ENTRY_KERNEL_CR3(GS), AX // Get kernel cr3 on AX. - WRITE_CR3() // Switch to kernel cr3. - - MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU. - MOVQ CPU_REGISTERS+PTRACE_RAX(AX), AX // Get user regs. - REGISTERS_SAVE(AX, 0) // Save all except IP, FLAGS, SP, AX. - MOVQ CX, PTRACE_RIP(AX) - MOVQ R11, PTRACE_FLAGS(AX) - MOVQ SP, PTRACE_RSP(AX) - MOVQ ENTRY_SCRATCH0(GS), CX // Load saved user AX value. - MOVQ CX, PTRACE_RAX(AX) // Save everything else. - MOVQ CX, PTRACE_ORIGRAX(AX) - - MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU. - MOVQ CPU_REGISTERS+PTRACE_RSP(AX), SP // Get stacks. - MOVQ $0, CPU_ERROR_CODE(AX) // Clear error code. - MOVQ $1, CPU_ERROR_TYPE(AX) // Set error type to user. - - // Return to the kernel, where the frame is: - // - // vector (sp+32) - // userCR3 (sp+24) - // regs (sp+16) - // cpu (sp+8) - // vcpu.Switch (sp+0) - // - MOVQ CPU_REGISTERS+PTRACE_RBP(AX), BP // Original base pointer. - MOVQ $Syscall, 32(SP) // Output vector. - RET - -kernel: - // We can't restore the original stack, but we can access the registers - // in the CPU state directly. No need for temporary juggling. - MOVQ AX, ENTRY_SCRATCH0(GS) - MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU. - REGISTERS_SAVE(AX, CPU_REGISTERS) - MOVQ CX, CPU_REGISTERS+PTRACE_RIP(AX) - MOVQ R11, CPU_REGISTERS+PTRACE_FLAGS(AX) - MOVQ SP, CPU_REGISTERS+PTRACE_RSP(AX) - MOVQ ENTRY_SCRATCH0(GS), BX - MOVQ BX, CPU_REGISTERS+PTRACE_ORIGRAX(AX) - MOVQ BX, CPU_REGISTERS+PTRACE_RAX(AX) - MOVQ $0, CPU_ERROR_CODE(AX) // Clear error code. - MOVQ $0, CPU_ERROR_TYPE(AX) // Set error type to kernel. - - // Call the syscall trampoline. - LOAD_KERNEL_STACK(GS) - PUSHQ AX // First argument (vCPU). - CALL ·kernelSyscall(SB) // Call the trampoline. - POPQ AX // Pop vCPU. - JMP ·resume(SB) - -// exception is a generic exception handler. -// -// There are two cases handled: -// -// 1) An exception in kernel mode: this results in saving the state at the time -// of the exception and calling the defined hook. -// -// 2) An exception in guest mode: the original kernel frame is restored, and -// the vector & error codes are pushed as return values. -// -// See below for the stubs that call exception. -TEXT ·exception(SB),NOSPLIT,$0 - // Determine whether the exception occurred in kernel mode or user - // mode, based on the flags. We expect the following stack: - // - // SS (sp+48) - // SP (sp+40) - // FLAGS (sp+32) - // CS (sp+24) - // IP (sp+16) - // ERROR_CODE (sp+8) - // VECTOR (sp+0) - // - TESTL $_RFLAGS_IOPL0, 32(SP) - JZ kernel - -user: - SWAP_GS() - ADDQ $-8, SP // Adjust for flags. - MOVQ $_KERNEL_FLAGS, 0(SP); BYTE $0x9d; // Reset flags (POPFQ). - PUSHQ AX // Save user AX on stack. - MOVQ ENTRY_KERNEL_CR3(GS), AX // Get kernel cr3 on AX. - WRITE_CR3() // Switch to kernel cr3. - - MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU. - MOVQ CPU_REGISTERS+PTRACE_RAX(AX), AX // Get user regs. - REGISTERS_SAVE(AX, 0) // Save all except IP, FLAGS, SP, AX. - POPQ BX // Restore original AX. - MOVQ BX, PTRACE_RAX(AX) // Save it. - MOVQ BX, PTRACE_ORIGRAX(AX) - MOVQ 16(SP), BX; MOVQ BX, PTRACE_RIP(AX) - MOVQ 24(SP), CX; MOVQ CX, PTRACE_CS(AX) - MOVQ 32(SP), DX; MOVQ DX, PTRACE_FLAGS(AX) - MOVQ 40(SP), DI; MOVQ DI, PTRACE_RSP(AX) - MOVQ 48(SP), SI; MOVQ SI, PTRACE_SS(AX) - - // Copy out and return. - MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU. - MOVQ 0(SP), BX // Load vector. - MOVQ 8(SP), CX // Load error code. - MOVQ CPU_REGISTERS+PTRACE_RSP(AX), SP // Original stack (kernel version). - MOVQ CPU_REGISTERS+PTRACE_RBP(AX), BP // Original base pointer. - MOVQ CX, CPU_ERROR_CODE(AX) // Set error code. - MOVQ $1, CPU_ERROR_TYPE(AX) // Set error type to user. - MOVQ BX, 32(SP) // Output vector. - RET - -kernel: - // As per above, we can save directly. - PUSHQ AX - MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU. - REGISTERS_SAVE(AX, CPU_REGISTERS) - POPQ BX - MOVQ BX, CPU_REGISTERS+PTRACE_RAX(AX) - MOVQ BX, CPU_REGISTERS+PTRACE_ORIGRAX(AX) - MOVQ 16(SP), BX; MOVQ BX, CPU_REGISTERS+PTRACE_RIP(AX) - MOVQ 32(SP), BX; MOVQ BX, CPU_REGISTERS+PTRACE_FLAGS(AX) - MOVQ 40(SP), BX; MOVQ BX, CPU_REGISTERS+PTRACE_RSP(AX) - - // Set the error code and adjust the stack. - MOVQ 8(SP), BX // Load the error code. - MOVQ BX, CPU_ERROR_CODE(AX) // Copy out to the CPU. - MOVQ $0, CPU_ERROR_TYPE(AX) // Set error type to kernel. - MOVQ 0(SP), BX // BX contains the vector. - - // Call the exception trampoline. - LOAD_KERNEL_STACK(GS) - PUSHQ BX // Second argument (vector). - PUSHQ AX // First argument (vCPU). - CALL ·kernelException(SB) // Call the trampoline. - POPQ BX // Pop vector. - POPQ AX // Pop vCPU. - JMP ·resume(SB) - -#define EXCEPTION_WITH_ERROR(value, symbol) \ -TEXT symbol,NOSPLIT,$0; \ - PUSHQ $value; \ - JMP ·exception(SB); - -#define EXCEPTION_WITHOUT_ERROR(value, symbol) \ -TEXT symbol,NOSPLIT,$0; \ - PUSHQ $0x0; \ - PUSHQ $value; \ - JMP ·exception(SB); - -EXCEPTION_WITHOUT_ERROR(DivideByZero, ·divideByZero(SB)) -EXCEPTION_WITHOUT_ERROR(Debug, ·debug(SB)) -EXCEPTION_WITHOUT_ERROR(NMI, ·nmi(SB)) -EXCEPTION_WITHOUT_ERROR(Breakpoint, ·breakpoint(SB)) -EXCEPTION_WITHOUT_ERROR(Overflow, ·overflow(SB)) -EXCEPTION_WITHOUT_ERROR(BoundRangeExceeded, ·boundRangeExceeded(SB)) -EXCEPTION_WITHOUT_ERROR(InvalidOpcode, ·invalidOpcode(SB)) -EXCEPTION_WITHOUT_ERROR(DeviceNotAvailable, ·deviceNotAvailable(SB)) -EXCEPTION_WITH_ERROR(DoubleFault, ·doubleFault(SB)) -EXCEPTION_WITHOUT_ERROR(CoprocessorSegmentOverrun, ·coprocessorSegmentOverrun(SB)) -EXCEPTION_WITH_ERROR(InvalidTSS, ·invalidTSS(SB)) -EXCEPTION_WITH_ERROR(SegmentNotPresent, ·segmentNotPresent(SB)) -EXCEPTION_WITH_ERROR(StackSegmentFault, ·stackSegmentFault(SB)) -EXCEPTION_WITH_ERROR(GeneralProtectionFault, ·generalProtectionFault(SB)) -EXCEPTION_WITH_ERROR(PageFault, ·pageFault(SB)) -EXCEPTION_WITHOUT_ERROR(X87FloatingPointException, ·x87FloatingPointException(SB)) -EXCEPTION_WITH_ERROR(AlignmentCheck, ·alignmentCheck(SB)) -EXCEPTION_WITHOUT_ERROR(MachineCheck, ·machineCheck(SB)) -EXCEPTION_WITHOUT_ERROR(SIMDFloatingPointException, ·simdFloatingPointException(SB)) -EXCEPTION_WITHOUT_ERROR(VirtualizationException, ·virtualizationException(SB)) -EXCEPTION_WITH_ERROR(SecurityException, ·securityException(SB)) -EXCEPTION_WITHOUT_ERROR(SyscallInt80, ·syscallInt80(SB)) diff --git a/pkg/sentry/platform/ring0/entry_impl_arm64.s b/pkg/sentry/platform/ring0/entry_impl_arm64.s deleted file mode 100644 index b3d300fb7..000000000 --- a/pkg/sentry/platform/ring0/entry_impl_arm64.s +++ /dev/null @@ -1,858 +0,0 @@ -// build +arm64 - -// Automatically generated, do not edit. - -// CPU offsets. -#define CPU_SELF 0x00 -#define CPU_REGISTERS 0x258 -#define CPU_STACK_TOP 0x210 -#define CPU_ERROR_CODE 0x210 -#define CPU_ERROR_TYPE 0x218 -#define CPU_FAULT_ADDR 0x220 -#define CPU_TTBR0_KVM 0x228 -#define CPU_TTBR0_APP 0x230 -#define CPU_VECTOR_CODE 0x238 -#define CPU_APP_ADDR 0x240 -#define CPU_LAZY_VFP 0x248 -#define CPU_APP_ASID 0x250 - -// Bits. -#define _KERNEL_FLAGS 0x3c5 - -// Vectors. -#define El1Sync 0x04 -#define El1Irq 0x05 -#define El1Fiq 0x06 -#define El1Err 0x07 -#define El0Sync 0x08 -#define El0Irq 0x09 -#define El0Fiq 0x0a -#define El0Err 0x0b -#define El1SyncDa 0x10 -#define El1SyncIa 0x11 -#define El1SyncSpPc 0x12 -#define El1SyncUndef 0x13 -#define El1SyncDbg 0x14 -#define El1SyncInv 0x15 -#define El0SyncSVC 0x16 -#define El0SyncDa 0x17 -#define El0SyncIa 0x18 -#define El0SyncFpsimdAcc 0x19 -#define El0SyncSveAcc 0x1a -#define El0SyncFpsimdExc 0x1b -#define El0SyncSys 0x1c -#define El0SyncSpPc 0x1d -#define El0SyncUndef 0x1e -#define El0SyncDbg 0x1f -#define El0SyncWfx 0x20 -#define El0SyncInv 0x21 -#define El0ErrNMI 0x22 -#define PageFault 0x17 -#define Syscall 0x16 -#define VirtualizationException 0x23 - -// Ptrace registers. -#define PTRACE_R0 0x00 -#define PTRACE_R1 0x08 -#define PTRACE_R2 0x10 -#define PTRACE_R3 0x18 -#define PTRACE_R4 0x20 -#define PTRACE_R5 0x28 -#define PTRACE_R6 0x30 -#define PTRACE_R7 0x38 -#define PTRACE_R8 0x40 -#define PTRACE_R9 0x48 -#define PTRACE_R10 0x50 -#define PTRACE_R11 0x58 -#define PTRACE_R12 0x60 -#define PTRACE_R13 0x68 -#define PTRACE_R14 0x70 -#define PTRACE_R15 0x78 -#define PTRACE_R16 0x80 -#define PTRACE_R17 0x88 -#define PTRACE_R18 0x90 -#define PTRACE_R19 0x98 -#define PTRACE_R20 0xa0 -#define PTRACE_R21 0xa8 -#define PTRACE_R22 0xb0 -#define PTRACE_R23 0xb8 -#define PTRACE_R24 0xc0 -#define PTRACE_R25 0xc8 -#define PTRACE_R26 0xd0 -#define PTRACE_R27 0xd8 -#define PTRACE_R28 0xe0 -#define PTRACE_R29 0xe8 -#define PTRACE_R30 0xf0 -#define PTRACE_SP 0xf8 -#define PTRACE_PC 0x100 -#define PTRACE_PSTATE 0x108 -#define PTRACE_TLS 0x110 -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "funcdata.h" -#include "textflag.h" - -// NB: Offsets are programatically generated (see BUILD). -// -// This file is concatenated with the definitions. - -// Saves a register set. -// -// This is a macro because it may need to executed in contents where a stack is -// not available for calls. -// - -// ERET returns using the ELR and SPSR for the current exception level. -#define ERET() \ - WORD $0xd69f03e0; \ - DSB $7; \ - ISB $15; - -// RSV_REG is a register that holds el1 information temporarily. -#define RSV_REG R18_PLATFORM - -// RSV_REG_APP is a register that holds el0 information temporarily. -#define RSV_REG_APP R9 - -#define FPEN_NOTRAP 0x3 -#define FPEN_SHIFT 20 - -#define FPEN_ENABLE (FPEN_NOTRAP << FPEN_SHIFT) - -// sctlr_el1: system control register el1. -#define SCTLR_M 1 << 0 -#define SCTLR_C 1 << 2 -#define SCTLR_I 1 << 12 -#define SCTLR_DZE 1 << 14 -#define SCTLR_UCT 1 << 15 -#define SCTLR_UCI 1 << 26 - -#define SCTLR_EL1_DEFAULT (SCTLR_M | SCTLR_C | SCTLR_I | SCTLR_UCT | SCTLR_UCI | SCTLR_DZE) - -// cntkctl_el1: counter-timer kernel control register el1. -#define CNTKCTL_EL0PCTEN 1 << 0 -#define CNTKCTL_EL0VCTEN 1 << 1 - -#define CNTKCTL_EL1_DEFAULT (CNTKCTL_EL0PCTEN | CNTKCTL_EL0VCTEN) - -// Saves a register set. -// -// This is a macro because it may need to executed in contents where a stack is -// not available for calls. -// -// The following registers are not saved: R9, R18. -#define REGISTERS_SAVE(reg, offset) \ - MOVD R0, offset+PTRACE_R0(reg); \ - MOVD R1, offset+PTRACE_R1(reg); \ - MOVD R2, offset+PTRACE_R2(reg); \ - MOVD R3, offset+PTRACE_R3(reg); \ - MOVD R4, offset+PTRACE_R4(reg); \ - MOVD R5, offset+PTRACE_R5(reg); \ - MOVD R6, offset+PTRACE_R6(reg); \ - MOVD R7, offset+PTRACE_R7(reg); \ - MOVD R8, offset+PTRACE_R8(reg); \ - MOVD R10, offset+PTRACE_R10(reg); \ - MOVD R11, offset+PTRACE_R11(reg); \ - MOVD R12, offset+PTRACE_R12(reg); \ - MOVD R13, offset+PTRACE_R13(reg); \ - MOVD R14, offset+PTRACE_R14(reg); \ - MOVD R15, offset+PTRACE_R15(reg); \ - MOVD R16, offset+PTRACE_R16(reg); \ - MOVD R17, offset+PTRACE_R17(reg); \ - MOVD R19, offset+PTRACE_R19(reg); \ - MOVD R20, offset+PTRACE_R20(reg); \ - MOVD R21, offset+PTRACE_R21(reg); \ - MOVD R22, offset+PTRACE_R22(reg); \ - MOVD R23, offset+PTRACE_R23(reg); \ - MOVD R24, offset+PTRACE_R24(reg); \ - MOVD R25, offset+PTRACE_R25(reg); \ - MOVD R26, offset+PTRACE_R26(reg); \ - MOVD R27, offset+PTRACE_R27(reg); \ - MOVD g, offset+PTRACE_R28(reg); \ - MOVD R29, offset+PTRACE_R29(reg); \ - MOVD R30, offset+PTRACE_R30(reg); - -// Loads a register set. -// -// This is a macro because it may need to executed in contents where a stack is -// not available for calls. -// -// The following registers are not loaded: R9, R18. -#define REGISTERS_LOAD(reg, offset) \ - MOVD offset+PTRACE_R0(reg), R0; \ - MOVD offset+PTRACE_R1(reg), R1; \ - MOVD offset+PTRACE_R2(reg), R2; \ - MOVD offset+PTRACE_R3(reg), R3; \ - MOVD offset+PTRACE_R4(reg), R4; \ - MOVD offset+PTRACE_R5(reg), R5; \ - MOVD offset+PTRACE_R6(reg), R6; \ - MOVD offset+PTRACE_R7(reg), R7; \ - MOVD offset+PTRACE_R8(reg), R8; \ - MOVD offset+PTRACE_R10(reg), R10; \ - MOVD offset+PTRACE_R11(reg), R11; \ - MOVD offset+PTRACE_R12(reg), R12; \ - MOVD offset+PTRACE_R13(reg), R13; \ - MOVD offset+PTRACE_R14(reg), R14; \ - MOVD offset+PTRACE_R15(reg), R15; \ - MOVD offset+PTRACE_R16(reg), R16; \ - MOVD offset+PTRACE_R17(reg), R17; \ - MOVD offset+PTRACE_R19(reg), R19; \ - MOVD offset+PTRACE_R20(reg), R20; \ - MOVD offset+PTRACE_R21(reg), R21; \ - MOVD offset+PTRACE_R22(reg), R22; \ - MOVD offset+PTRACE_R23(reg), R23; \ - MOVD offset+PTRACE_R24(reg), R24; \ - MOVD offset+PTRACE_R25(reg), R25; \ - MOVD offset+PTRACE_R26(reg), R26; \ - MOVD offset+PTRACE_R27(reg), R27; \ - MOVD offset+PTRACE_R28(reg), g; \ - MOVD offset+PTRACE_R29(reg), R29; \ - MOVD offset+PTRACE_R30(reg), R30; - -#define ESR_ELx_EC_UNKNOWN (0x00) -#define ESR_ELx_EC_WFx (0x01) -/* Unallocated EC: 0x02 */ -#define ESR_ELx_EC_CP15_32 (0x03) -#define ESR_ELx_EC_CP15_64 (0x04) -#define ESR_ELx_EC_CP14_MR (0x05) -#define ESR_ELx_EC_CP14_LS (0x06) -#define ESR_ELx_EC_FP_ASIMD (0x07) -#define ESR_ELx_EC_CP10_ID (0x08) /* EL2 only */ -#define ESR_ELx_EC_PAC (0x09) /* EL2 and above */ -/* Unallocated EC: 0x0A - 0x0B */ -#define ESR_ELx_EC_CP14_64 (0x0C) -/* Unallocated EC: 0x0d */ -#define ESR_ELx_EC_ILL (0x0E) -/* Unallocated EC: 0x0F - 0x10 */ -#define ESR_ELx_EC_SVC32 (0x11) -#define ESR_ELx_EC_HVC32 (0x12) /* EL2 only */ -#define ESR_ELx_EC_SMC32 (0x13) /* EL2 and above */ -/* Unallocated EC: 0x14 */ -#define ESR_ELx_EC_SVC64 (0x15) -#define ESR_ELx_EC_HVC64 (0x16) /* EL2 and above */ -#define ESR_ELx_EC_SMC64 (0x17) /* EL2 and above */ -#define ESR_ELx_EC_SYS64 (0x18) -#define ESR_ELx_EC_SVE (0x19) -/* Unallocated EC: 0x1A - 0x1E */ -#define ESR_ELx_EC_IMP_DEF (0x1f) /* EL3 only */ -#define ESR_ELx_EC_IABT_LOW (0x20) -#define ESR_ELx_EC_IABT_CUR (0x21) -#define ESR_ELx_EC_PC_ALIGN (0x22) -/* Unallocated EC: 0x23 */ -#define ESR_ELx_EC_DABT_LOW (0x24) -#define ESR_ELx_EC_DABT_CUR (0x25) -#define ESR_ELx_EC_SP_ALIGN (0x26) -/* Unallocated EC: 0x27 */ -#define ESR_ELx_EC_FP_EXC32 (0x28) -/* Unallocated EC: 0x29 - 0x2B */ -#define ESR_ELx_EC_FP_EXC64 (0x2C) -/* Unallocated EC: 0x2D - 0x2E */ -#define ESR_ELx_EC_SERROR (0x2F) -#define ESR_ELx_EC_BREAKPT_LOW (0x30) -#define ESR_ELx_EC_BREAKPT_CUR (0x31) -#define ESR_ELx_EC_SOFTSTP_LOW (0x32) -#define ESR_ELx_EC_SOFTSTP_CUR (0x33) -#define ESR_ELx_EC_WATCHPT_LOW (0x34) -#define ESR_ELx_EC_WATCHPT_CUR (0x35) -/* Unallocated EC: 0x36 - 0x37 */ -#define ESR_ELx_EC_BKPT32 (0x38) -/* Unallocated EC: 0x39 */ -#define ESR_ELx_EC_VECTOR32 (0x3A) /* EL2 only */ -/* Unallocted EC: 0x3B */ -#define ESR_ELx_EC_BRK64 (0x3C) -/* Unallocated EC: 0x3D - 0x3F */ -#define ESR_ELx_EC_MAX (0x3F) - -#define ESR_ELx_EC_SHIFT (26) -#define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT) -#define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) - -#define ESR_ELx_IL_SHIFT (25) -#define ESR_ELx_IL (UL(1) << ESR_ELx_IL_SHIFT) -#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) - -/* ISS field definitions shared by different classes */ -#define ESR_ELx_WNR_SHIFT (6) -#define ESR_ELx_WNR (UL(1) << ESR_ELx_WNR_SHIFT) - -/* Asynchronous Error Type */ -#define ESR_ELx_IDS_SHIFT (24) -#define ESR_ELx_IDS (UL(1) << ESR_ELx_IDS_SHIFT) -#define ESR_ELx_AET_SHIFT (10) -#define ESR_ELx_AET (UL(0x7) << ESR_ELx_AET_SHIFT) - -#define ESR_ELx_AET_UC (UL(0) << ESR_ELx_AET_SHIFT) -#define ESR_ELx_AET_UEU (UL(1) << ESR_ELx_AET_SHIFT) -#define ESR_ELx_AET_UEO (UL(2) << ESR_ELx_AET_SHIFT) -#define ESR_ELx_AET_UER (UL(3) << ESR_ELx_AET_SHIFT) -#define ESR_ELx_AET_CE (UL(6) << ESR_ELx_AET_SHIFT) - -/* Shared ISS field definitions for Data/Instruction aborts */ -#define ESR_ELx_SET_SHIFT (11) -#define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT) -#define ESR_ELx_FnV_SHIFT (10) -#define ESR_ELx_FnV (UL(1) << ESR_ELx_FnV_SHIFT) -#define ESR_ELx_EA_SHIFT (9) -#define ESR_ELx_EA (UL(1) << ESR_ELx_EA_SHIFT) -#define ESR_ELx_S1PTW_SHIFT (7) -#define ESR_ELx_S1PTW (UL(1) << ESR_ELx_S1PTW_SHIFT) - -/* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */ -#define ESR_ELx_FSC (0x3F) -#define ESR_ELx_FSC_TYPE (0x3C) -#define ESR_ELx_FSC_EXTABT (0x10) -#define ESR_ELx_FSC_SERROR (0x11) -#define ESR_ELx_FSC_ACCESS (0x08) -#define ESR_ELx_FSC_FAULT (0x04) -#define ESR_ELx_FSC_PERM (0x0C) - -/* ISS field definitions for Data Aborts */ -#define ESR_ELx_ISV_SHIFT (24) -#define ESR_ELx_ISV (UL(1) << ESR_ELx_ISV_SHIFT) -#define ESR_ELx_SAS_SHIFT (22) -#define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT) -#define ESR_ELx_SSE_SHIFT (21) -#define ESR_ELx_SSE (UL(1) << ESR_ELx_SSE_SHIFT) -#define ESR_ELx_SRT_SHIFT (16) -#define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT) -#define ESR_ELx_SF_SHIFT (15) -#define ESR_ELx_SF (UL(1) << ESR_ELx_SF_SHIFT) -#define ESR_ELx_AR_SHIFT (14) -#define ESR_ELx_AR (UL(1) << ESR_ELx_AR_SHIFT) -#define ESR_ELx_CM_SHIFT (8) -#define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT) - -/* ISS field definitions for exceptions taken in to Hyp */ -#define ESR_ELx_CV (UL(1) << 24) -#define ESR_ELx_COND_SHIFT (20) -#define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT) -#define ESR_ELx_WFx_ISS_TI (UL(1) << 0) -#define ESR_ELx_WFx_ISS_WFI (UL(0) << 0) -#define ESR_ELx_WFx_ISS_WFE (UL(1) << 0) -#define ESR_ELx_xVC_IMM_MASK ((1UL << 16) - 1) - -/* ISS field definitions for system error */ -#define ESR_ELx_SERR_MASK (0x1) -#define ESR_ELx_SERR_NMI (0x1) - -// LOAD_KERNEL_ADDRESS loads a kernel address. -#define LOAD_KERNEL_ADDRESS(from, to) \ - MOVD from, to; \ - ORR $0xffff000000000000, to, to; - -// LOAD_KERNEL_STACK loads the kernel temporary stack. -#define LOAD_KERNEL_STACK(from) \ - LOAD_KERNEL_ADDRESS(CPU_SELF(from), RSV_REG); \ - MOVD $CPU_STACK_TOP(RSV_REG), RSV_REG; \ - MOVD RSV_REG, RSP; \ - WORD $0xd538d092; //MRS TPIDR_EL1, R18 - -// SWITCH_TO_APP_PAGETABLE sets a new pagetable for a container application. -#define SWITCH_TO_APP_PAGETABLE() \ - MOVD CPU_APP_ASID(RSV_REG), RSV_REG_APP; \ - MOVD CPU_TTBR0_APP(RSV_REG), RSV_REG; \ - BFI $48, RSV_REG_APP, $16, RSV_REG; \ - MSR RSV_REG, TTBR0_EL1; \ - ISB $15; - -// SWITCH_TO_KVM_PAGETABLE sets the kvm pagetable. -#define SWITCH_TO_KVM_PAGETABLE() \ - MOVD CPU_TTBR0_KVM(RSV_REG), RSV_REG; \ - MOVD $1, RSV_REG_APP; \ - BFI $48, RSV_REG_APP, $16, RSV_REG; \ - MSR RSV_REG, TTBR0_EL1; \ - ISB $15; - -TEXT ·EnableVFP(SB),NOSPLIT,$0 - MOVD $FPEN_ENABLE, R0 - WORD $0xd5181040 //MSR R0, CPACR_EL1 - ISB $15 - RET - -TEXT ·DisableVFP(SB),NOSPLIT,$0 - MOVD $0, R0 - WORD $0xd5181040 //MSR R0, CPACR_EL1 - ISB $15 - RET - -#define VFP_ENABLE \ - MOVD $FPEN_ENABLE, R0; \ - WORD $0xd5181040; \ //MSR R0, CPACR_EL1 - ISB $15; - -#define VFP_DISABLE \ - MOVD $0x0, R0; \ - WORD $0xd5181040; \ //MSR R0, CPACR_EL1 - ISB $15; - -// KERNEL_ENTRY_FROM_EL0 is the entry code of the vcpu from el0 to el1. -#define KERNEL_ENTRY_FROM_EL0 \ - SUB $16, RSP, RSP; \ // step1, save r18, r9 into kernel temporary stack. - STP (RSV_REG, RSV_REG_APP), 16*0(RSP); \ - WORD $0xd538d092; \ // MRS TPIDR_EL1, R18 - MOVD CPU_APP_ADDR(RSV_REG), RSV_REG_APP; \ // step2, load app context pointer. - REGISTERS_SAVE(RSV_REG_APP, 0); \ // step3, save app context. - MOVD RSV_REG_APP, R20; \ - LDP 16*0(RSP), (RSV_REG, RSV_REG_APP); \ - ADD $16, RSP, RSP; \ - MOVD RSV_REG, PTRACE_R18(R20); \ - MOVD RSV_REG_APP, PTRACE_R9(R20); \ - MRS TPIDR_EL0, R3; \ - MOVD R3, PTRACE_TLS(R20); \ - WORD $0xd5384003; \ // MRS SPSR_EL1, R3 - MOVD R3, PTRACE_PSTATE(R20); \ - MRS ELR_EL1, R3; \ - MOVD R3, PTRACE_PC(R20); \ - WORD $0xd5384103; \ // MRS SP_EL0, R3 - MOVD R3, PTRACE_SP(R20); - -// KERNEL_ENTRY_FROM_EL1 is the entry code of the vcpu from el1 to el1. -#define KERNEL_ENTRY_FROM_EL1 \ - WORD $0xd538d092; \ //MRS TPIDR_EL1, R18 - REGISTERS_SAVE(RSV_REG, CPU_REGISTERS); \ // Save sentry context. - MOVD RSV_REG_APP, CPU_REGISTERS+PTRACE_R9(RSV_REG); \ - MRS TPIDR_EL0, R4; \ - MOVD R4, CPU_REGISTERS+PTRACE_TLS(RSV_REG); \ - WORD $0xd5384004; \ // MRS SPSR_EL1, R4 - MOVD R4, CPU_REGISTERS+PTRACE_PSTATE(RSV_REG); \ - MRS ELR_EL1, R4; \ - MOVD R4, CPU_REGISTERS+PTRACE_PC(RSV_REG); \ - MOVD RSP, R4; \ - MOVD R4, CPU_REGISTERS+PTRACE_SP(RSV_REG); \ - LOAD_KERNEL_STACK(RSV_REG); // Load the temporary stack. - -// EXCEPTION_EL0 is a common el0 exception handler function. -#define EXCEPTION_EL0(vector) \ - WORD $0xd538d092; \ //MRS TPIDR_EL1, R18 - WORD $0xd538601a; \ //MRS FAR_EL1, R26 - MOVD R26, CPU_FAULT_ADDR(RSV_REG); \ - MOVD $1, R3; \ - MOVD R3, CPU_ERROR_TYPE(RSV_REG); \ // Set error type to user. - MOVD $vector, R3; \ - MOVD R3, CPU_VECTOR_CODE(RSV_REG); \ - MRS ESR_EL1, R3; \ - MOVD R3, CPU_ERROR_CODE(RSV_REG); \ - B ·kernelExitToEl1(SB); - -// EXCEPTION_EL1 is a common el1 exception handler function. -#define EXCEPTION_EL1(vector) \ - MOVD $vector, R3; \ - MOVD R3, 8(RSP); \ - B ·HaltEl1ExceptionAndResume(SB); - -// storeAppASID writes the application's asid value. -TEXT ·storeAppASID(SB),NOSPLIT,$0-8 - MOVD asid+0(FP), R1 - MRS TPIDR_EL1, RSV_REG - MOVD R1, CPU_APP_ASID(RSV_REG) - RET - -// Halt halts execution. -TEXT ·Halt(SB),NOSPLIT,$0 - // Clear bluepill. - WORD $0xd538d092 //MRS TPIDR_EL1, R18 - CMP RSV_REG, R9 - BNE mmio_exit - MOVD $0, CPU_REGISTERS+PTRACE_R9(RSV_REG) - -mmio_exit: - // Disable fpsimd. - WORD $0xd5381041 // MRS CPACR_EL1, R1 - MOVD R1, CPU_LAZY_VFP(RSV_REG) - VFP_DISABLE - - // Trigger MMIO_EXIT/_KVM_HYPERCALL_VMEXIT. - // - // To keep it simple, I used the address of exception table as the - // MMIO base address, so that I can trigger a MMIO-EXIT by forcibly writing - // a read-only space. - // Also, the length is engough to match a sufficient number of hypercall ID. - // Then, in host user space, I can calculate this address to find out - // which hypercall. - MRS VBAR_EL1, R9 - MOVD R0, 0x0(R9) - - RET - -// HaltAndResume halts execution and point the pointer to the resume function. -TEXT ·HaltAndResume(SB),NOSPLIT,$0 - BL ·Halt(SB) - B ·kernelExitToEl1(SB) // Resume. - -// HaltEl1SvcAndResume calls Hooks.KernelSyscall and resume. -TEXT ·HaltEl1SvcAndResume(SB),NOSPLIT,$0 - WORD $0xd538d092 // MRS TPIDR_EL1, R18 - MOVD CPU_SELF(RSV_REG), R3 // Load vCPU. - MOVD R3, 8(RSP) // First argument (vCPU). - CALL ·kernelSyscall(SB) // Call the trampoline. - B ·kernelExitToEl1(SB) // Resume. - -// HaltEl1ExceptionAndResume calls Hooks.KernelException and resume. -TEXT ·HaltEl1ExceptionAndResume(SB),NOSPLIT,$0-8 - WORD $0xd538d092 // MRS TPIDR_EL1, R18 - MOVD CPU_SELF(RSV_REG), R3 // Load vCPU. - MOVD R3, 8(RSP) // First argument (vCPU). - MOVD vector+0(FP), R3 - MOVD R3, 16(RSP) // Second argument (vector). - CALL ·kernelException(SB) // Call the trampoline. - B ·kernelExitToEl1(SB) // Resume. - -// Shutdown stops the guest. -TEXT ·Shutdown(SB),NOSPLIT,$0 - // PSCI EVENT. - MOVD $0x84000009, R0 - HVC $0 - -// See kernel.go. -TEXT ·Current(SB),NOSPLIT,$0-8 - MOVD CPU_SELF(RSV_REG), R8 - MOVD R8, ret+0(FP) - RET - -#define STACK_FRAME_SIZE 32 - -// kernelExitToEl0 is the entrypoint for application in guest_el0. -// Prepare the vcpu environment for container application. -TEXT ·kernelExitToEl0(SB),NOSPLIT,$0 - // Step1, save sentry context into memory. - MRS TPIDR_EL1, RSV_REG - REGISTERS_SAVE(RSV_REG, CPU_REGISTERS) - MOVD RSV_REG_APP, CPU_REGISTERS+PTRACE_R9(RSV_REG) - MRS TPIDR_EL0, R3 - MOVD R3, CPU_REGISTERS+PTRACE_TLS(RSV_REG) - - WORD $0xd5384003 // MRS SPSR_EL1, R3 - MOVD R3, CPU_REGISTERS+PTRACE_PSTATE(RSV_REG) - MOVD R30, CPU_REGISTERS+PTRACE_PC(RSV_REG) - MOVD RSP, R3 - MOVD R3, CPU_REGISTERS+PTRACE_SP(RSV_REG) - - MOVD CPU_REGISTERS+PTRACE_R3(RSV_REG), R3 - - // Step2, switch to temporary stack. - LOAD_KERNEL_STACK(RSV_REG) - - // Step3, load app context pointer. - MOVD CPU_APP_ADDR(RSV_REG), RSV_REG_APP - - // Step4, prepare the environment for container application. - // set sp_el0. - MOVD PTRACE_SP(RSV_REG_APP), R1 - WORD $0xd5184101 //MSR R1, SP_EL0 - // set pc. - MOVD PTRACE_PC(RSV_REG_APP), R1 - MSR R1, ELR_EL1 - // set pstate. - MOVD PTRACE_PSTATE(RSV_REG_APP), R1 - WORD $0xd5184001 //MSR R1, SPSR_EL1 - - // need use kernel space address to excute below code, since - // after SWITCH_TO_APP_PAGETABLE the ASID is changed to app's - // ASID. - WORD $0x10000061 // ADR R1, do_exit_to_el0 - ORR $0xffff000000000000, R1, R1 - JMP (R1) - -do_exit_to_el0: - // RSV_REG & RSV_REG_APP will be loaded at the end. - REGISTERS_LOAD(RSV_REG_APP, 0) - MOVD PTRACE_TLS(RSV_REG_APP), RSV_REG - MSR RSV_REG, TPIDR_EL0 - - // switch to user pagetable. - MOVD PTRACE_R18(RSV_REG_APP), RSV_REG - MOVD PTRACE_R9(RSV_REG_APP), RSV_REG_APP - - SUB $STACK_FRAME_SIZE, RSP, RSP - STP (RSV_REG, RSV_REG_APP), 16*0(RSP) - STP (R0, R1), 16*1(RSP) - - WORD $0xd538d092 //MRS TPIDR_EL1, R18 - - SWITCH_TO_APP_PAGETABLE() - - LDP 16*1(RSP), (R0, R1) - LDP 16*0(RSP), (RSV_REG, RSV_REG_APP) - ADD $STACK_FRAME_SIZE, RSP, RSP - - ERET() - -// kernelExitToEl1 is the entrypoint for sentry in guest_el1. -// Prepare the vcpu environment for sentry. -TEXT ·kernelExitToEl1(SB),NOSPLIT,$0 - WORD $0xd538d092 //MRS TPIDR_EL1, R18 - MOVD CPU_REGISTERS+PTRACE_PSTATE(RSV_REG), R1 - WORD $0xd5184001 //MSR R1, SPSR_EL1 - - MOVD CPU_REGISTERS+PTRACE_PC(RSV_REG), R1 - MSR R1, ELR_EL1 - - // restore sentry's tls. - MOVD CPU_REGISTERS+PTRACE_TLS(RSV_REG), R1 - MSR R1, TPIDR_EL0 - - MOVD CPU_REGISTERS+PTRACE_SP(RSV_REG), R1 - MOVD R1, RSP - - REGISTERS_LOAD(RSV_REG, CPU_REGISTERS) - SWITCH_TO_KVM_PAGETABLE() - MRS TPIDR_EL1, RSV_REG - - MOVD CPU_REGISTERS+PTRACE_R9(RSV_REG), RSV_REG_APP - - ERET() - -// Start is the CPU entrypoint. -TEXT ·Start(SB),NOSPLIT,$0 - // Init. - WORD $0xd508871f // __tlbi(vmalle1) - DSB $7 // dsb(nsh) - - MOVD $1<<12, R1 // Reset mdscr_el1 and disable - MSR R1, MDSCR_EL1 // access to the DCC from EL0 - ISB $15 - - MRS TTBR1_EL1, R1 - MSR R1, TTBR0_EL1 - ISB $15 - - MOVD $CNTKCTL_EL1_DEFAULT, R1 - MSR R1, CNTKCTL_EL1 - - MOVD R8, RSV_REG - ORR $0xffff000000000000, RSV_REG, RSV_REG - WORD $0xd518d092 //MSR R18, TPIDR_EL1 - - // Init. - MOVD $SCTLR_EL1_DEFAULT, R1 // re-enable the mmu. - MSR R1, SCTLR_EL1 - ISB $15 - WORD $0xd508751f // ic iallu - - DSB $7 // dsb(nsh) - ISB $15 - - B ·kernelExitToEl1(SB) - -// El1_sync_invalid is the handler for an invalid EL1_sync. -TEXT ·El1_sync_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// El1_irq_invalid is the handler for an invalid El1_irq. -TEXT ·El1_irq_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// El1_fiq_invalid is the handler for an invalid El1_fiq. -TEXT ·El1_fiq_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// El1_error_invalid is the handler for an invalid El1_error. -TEXT ·El1_error_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// El1_sync is the handler for El1_sync. -TEXT ·El1_sync(SB),NOSPLIT,$0 - KERNEL_ENTRY_FROM_EL1 - MRS ESR_EL1, R25 // read the syndrome register - LSR $ESR_ELx_EC_SHIFT, R25, R24 // exception class - CMP $ESR_ELx_EC_DABT_CUR, R24 - BEQ el1_da // data abort in EL1 - CMP $ESR_ELx_EC_IABT_CUR, R24 - BEQ el1_ia // instruction abort in EL1 - CMP $ESR_ELx_EC_SP_ALIGN, R24 - BEQ el1_sp_pc // stack alignment exception - CMP $ESR_ELx_EC_PC_ALIGN, R24 - BEQ el1_sp_pc // pc alignment exception - CMP $ESR_ELx_EC_UNKNOWN, R24 - BEQ el1_undef // unknown exception in EL1 - CMP $ESR_ELx_EC_SVC64, R24 - BEQ el1_svc // SVC in 64-bit state - CMP $ESR_ELx_EC_BREAKPT_CUR, R24 - BEQ el1_dbg // debug exception in EL1 - CMP $ESR_ELx_EC_FP_ASIMD, R24 - BEQ el1_fpsimd_acc // FP/ASIMD access - CMP $ESR_ELx_EC_SVE, R24 - BEQ el1_sve_acc // SVE access - B el1_invalid - -el1_da: - EXCEPTION_EL1(El1SyncDa) -el1_ia: - EXCEPTION_EL1(El1SyncIa) -el1_sp_pc: - EXCEPTION_EL1(El1SyncSpPc) -el1_undef: - EXCEPTION_EL1(El1SyncUndef) -el1_svc: - B ·HaltEl1SvcAndResume(SB) -el1_dbg: - EXCEPTION_EL1(El1SyncDbg) -el1_fpsimd_acc: -el1_sve_acc: - VFP_ENABLE - B ·kernelExitToEl1(SB) // Resume. -el1_invalid: - EXCEPTION_EL1(El1SyncInv) - -// El1_irq is the handler for El1_irq. -TEXT ·El1_irq(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// El1_fiq is the handler for El1_fiq. -TEXT ·El1_fiq(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// El1_error is the handler for El1_error. -TEXT ·El1_error(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// El0_sync is the handler for El0_sync. -TEXT ·El0_sync(SB),NOSPLIT,$0 - KERNEL_ENTRY_FROM_EL0 - MRS ESR_EL1, R25 // read the syndrome register - LSR $ESR_ELx_EC_SHIFT, R25, R24 // exception class - CMP $ESR_ELx_EC_SVC64, R24 - BEQ el0_svc // SVC in 64-bit state - CMP $ESR_ELx_EC_DABT_LOW, R24 - BEQ el0_da // data abort in EL0 - CMP $ESR_ELx_EC_IABT_LOW, R24 - BEQ el0_ia // instruction abort in EL0 - CMP $ESR_ELx_EC_FP_ASIMD, R24 - BEQ el0_fpsimd_acc // FP/ASIMD access - CMP $ESR_ELx_EC_SVE, R24 - BEQ el0_sve_acc // SVE access - CMP $ESR_ELx_EC_FP_EXC64, R24 - BEQ el0_fpsimd_exc // FP/ASIMD exception - CMP $ESR_ELx_EC_SP_ALIGN, R24 - BEQ el0_sp_pc // stack alignment exception - CMP $ESR_ELx_EC_PC_ALIGN, R24 - BEQ el0_sp_pc // pc alignment exception - CMP $ESR_ELx_EC_UNKNOWN, R24 - BEQ el0_undef // unknown exception in EL0 - CMP $ESR_ELx_EC_BREAKPT_LOW, R24 - BEQ el0_dbg // debug exception in EL0 - CMP $ESR_ELx_EC_SYS64, R24 - BEQ el0_sys // configurable trap - CMP $ESR_ELx_EC_WFx, R24 - BEQ el0_wfx // WFX trap - B el0_invalid - -el0_svc: - WORD $0xd538d092 //MRS TPIDR_EL1, R18 - - MOVD $0, CPU_ERROR_CODE(RSV_REG) // Clear error code. - - MOVD $1, R3 - MOVD R3, CPU_ERROR_TYPE(RSV_REG) // Set error type to user. - - MOVD $Syscall, R3 - MOVD R3, CPU_VECTOR_CODE(RSV_REG) - - B ·kernelExitToEl1(SB) - -el0_da: -el0_ia: - EXCEPTION_EL0(PageFault) -el0_fpsimd_acc: - EXCEPTION_EL0(El0SyncFpsimdAcc) -el0_sve_acc: - EXCEPTION_EL0(El0SyncSveAcc) -el0_fpsimd_exc: - EXCEPTION_EL0(El0SyncFpsimdExc) -el0_sp_pc: - EXCEPTION_EL0(El0SyncSpPc) -el0_undef: - EXCEPTION_EL0(El0SyncUndef) -el0_dbg: - EXCEPTION_EL0(El0SyncDbg) -el0_sys: - EXCEPTION_EL0(El0SyncSys) -el0_wfx: - EXCEPTION_EL0(El0SyncWfx) -el0_invalid: - EXCEPTION_EL0(El0SyncInv) - -TEXT ·El0_irq(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -TEXT ·El0_fiq(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -TEXT ·El0_error(SB),NOSPLIT,$0 - KERNEL_ENTRY_FROM_EL0 - WORD $0xd5385219 // MRS ESR_EL1, R25 - AND $ESR_ELx_SERR_MASK, R25, R24 - CMP $ESR_ELx_SERR_NMI, R24 - BEQ el0_nmi - B el0_bounce - -el0_nmi: - EXCEPTION_EL0(El0ErrNMI) -el0_bounce: - EXCEPTION_EL0(VirtualizationException) - -TEXT ·El0_sync_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -TEXT ·El0_irq_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -TEXT ·El0_fiq_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -TEXT ·El0_error_invalid(SB),NOSPLIT,$0 - B ·Shutdown(SB) - -// Vectors implements exception vector table. -// The start address of exception vector table should be 11-bits aligned. -// For detail, please refer to arm developer document: -// https://developer.arm.com/documentation/100933/0100/AArch64-exception-vector-table -// Also can refer to the code in linux kernel: arch/arm64/kernel/entry.S -TEXT ·Vectors(SB),NOSPLIT,$0 - PCALIGN $2048 - B ·El1_sync_invalid(SB) - PCALIGN $128 - B ·El1_irq_invalid(SB) - PCALIGN $128 - B ·El1_fiq_invalid(SB) - PCALIGN $128 - B ·El1_error_invalid(SB) - - PCALIGN $128 - B ·El1_sync(SB) - PCALIGN $128 - B ·El1_irq(SB) - PCALIGN $128 - B ·El1_fiq(SB) - PCALIGN $128 - B ·El1_error(SB) - - PCALIGN $128 - B ·El0_sync(SB) - PCALIGN $128 - B ·El0_irq(SB) - PCALIGN $128 - B ·El0_fiq(SB) - PCALIGN $128 - B ·El0_error(SB) - - PCALIGN $128 - B ·El0_sync_invalid(SB) - PCALIGN $128 - B ·El0_irq_invalid(SB) - PCALIGN $128 - B ·El0_fiq_invalid(SB) - PCALIGN $128 - B ·El0_error_invalid(SB) diff --git a/pkg/sentry/platform/ring0/kernel.go b/pkg/sentry/platform/ring0/kernel.go deleted file mode 100644 index 292f9d0cc..000000000 --- a/pkg/sentry/platform/ring0/kernel.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ring0 - -// Init initializes a new kernel. -// -//go:nosplit -func (k *Kernel) Init(maxCPUs int) { - k.init(maxCPUs) -} - -// Halt halts execution. -func Halt() - -// defaultHooks implements hooks. -type defaultHooks struct{} - -// KernelSyscall implements Hooks.KernelSyscall. -// -// +checkescape:all -// -//go:nosplit -func (defaultHooks) KernelSyscall() { - Halt() -} - -// KernelException implements Hooks.KernelException. -// -// +checkescape:all -// -//go:nosplit -func (defaultHooks) KernelException(Vector) { - Halt() -} - -// kernelSyscall is a trampoline. -// -// When in amd64, it is called with %rip on the upper half, so it can -// NOT access to any global data which is not mapped on upper and must -// call to function pointers or interfaces to switch to the lower half -// so that callee can access to global data. -// -// +checkescape:hard,stack -// -//go:nosplit -func kernelSyscall(c *CPU) { - c.hooks.KernelSyscall() -} - -// kernelException is a trampoline. -// -// When in amd64, it is called with %rip on the upper half, so it can -// NOT access to any global data which is not mapped on upper and must -// call to function pointers or interfaces to switch to the lower half -// so that callee can access to global data. -// -// +checkescape:hard,stack -// -//go:nosplit -func kernelException(c *CPU, vector Vector) { - c.hooks.KernelException(vector) -} - -// Init initializes a new CPU. -// -// Init allows embedding in other objects. -func (c *CPU) Init(k *Kernel, cpuID int, hooks Hooks) { - c.self = c // Set self reference. - c.kernel = k // Set kernel reference. - c.init(cpuID) // Perform architectural init. - - // Require hooks. - if hooks != nil { - c.hooks = hooks - } else { - c.hooks = defaultHooks{} - } -} diff --git a/pkg/sentry/platform/ring0/kernel_amd64.go b/pkg/sentry/platform/ring0/kernel_amd64.go deleted file mode 100644 index 36a60700e..000000000 --- a/pkg/sentry/platform/ring0/kernel_amd64.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package ring0 - -import ( - "encoding/binary" - "reflect" - - "gvisor.dev/gvisor/pkg/usermem" -) - -// init initializes architecture-specific state. -func (k *Kernel) init(maxCPUs int) { - entrySize := reflect.TypeOf(kernelEntry{}).Size() - var ( - entries []kernelEntry - padding = 1 - ) - for { - entries = make([]kernelEntry, maxCPUs+padding-1) - totalSize := entrySize * uintptr(maxCPUs+padding-1) - addr := reflect.ValueOf(&entries[0]).Pointer() - if addr&(usermem.PageSize-1) == 0 && totalSize >= usermem.PageSize { - // The runtime forces power-of-2 alignment for allocations, and we are therefore - // safe once the first address is aligned and the chunk is at least a full page. - break - } - padding = padding << 1 - } - k.cpuEntries = entries - - k.globalIDT = &idt64{} - if reflect.TypeOf(idt64{}).Size() != usermem.PageSize { - panic("Size of globalIDT should be PageSize") - } - if reflect.ValueOf(k.globalIDT).Pointer()&(usermem.PageSize-1) != 0 { - panic("Allocated globalIDT should be page aligned") - } - - // Setup the IDT, which is uniform. - for v, handler := range handlers { - // Allow Breakpoint and Overflow to be called from all - // privilege levels. - dpl := 0 - if v == Breakpoint || v == Overflow { - dpl = 3 - } - // Note that we set all traps to use the interrupt stack, this - // is defined below when setting up the TSS. - k.globalIDT[v].setInterrupt(Kcode, uint64(kernelFunc(handler)), dpl, 1 /* ist */) - } -} - -// EntryRegions returns the set of kernel entry regions (must be mapped). -func (k *Kernel) EntryRegions() map[uintptr]uintptr { - regions := make(map[uintptr]uintptr) - - addr := reflect.ValueOf(&k.cpuEntries[0]).Pointer() - size := reflect.TypeOf(kernelEntry{}).Size() * uintptr(len(k.cpuEntries)) - end, _ := usermem.Addr(addr + size).RoundUp() - regions[uintptr(usermem.Addr(addr).RoundDown())] = uintptr(end) - - addr = reflect.ValueOf(k.globalIDT).Pointer() - size = reflect.TypeOf(idt64{}).Size() - end, _ = usermem.Addr(addr + size).RoundUp() - regions[uintptr(usermem.Addr(addr).RoundDown())] = uintptr(end) - - return regions -} - -// init initializes architecture-specific state. -func (c *CPU) init(cpuID int) { - c.kernelEntry = &c.kernel.cpuEntries[cpuID] - c.cpuSelf = c - // Null segment. - c.gdt[0].setNull() - - // Kernel & user segments. - c.gdt[segKcode] = KernelCodeSegment - c.gdt[segKdata] = KernelDataSegment - c.gdt[segUcode32] = UserCodeSegment32 - c.gdt[segUdata] = UserDataSegment - c.gdt[segUcode64] = UserCodeSegment64 - - // The task segment, this spans two entries. - tssBase, tssLimit, _ := c.TSS() - c.gdt[segTss].set( - uint32(tssBase), - uint32(tssLimit), - 0, // Privilege level zero. - SegmentDescriptorPresent| - SegmentDescriptorAccess| - SegmentDescriptorWrite| - SegmentDescriptorExecute) - c.gdt[segTssHi].setHi(uint32((tssBase) >> 32)) - - // Set the kernel stack pointer in the TSS (virtual address). - stackAddr := c.StackTop() - c.stackTop = stackAddr - c.tss.rsp0Lo = uint32(stackAddr) - c.tss.rsp0Hi = uint32(stackAddr >> 32) - c.tss.ist1Lo = uint32(stackAddr) - c.tss.ist1Hi = uint32(stackAddr >> 32) - - // Set the I/O bitmap base address beyond the last byte in the TSS - // to block access to the entire I/O address range. - // - // From section 18.5.2 "I/O Permission Bit Map" from Intel SDM vol1: - // I/O addresses not spanned by the map are treated as if they had set - // bits in the map. - c.tss.ioPerm = tssLimit + 1 - - // Permanently set the kernel segments. - c.registers.Cs = uint64(Kcode) - c.registers.Ds = uint64(Kdata) - c.registers.Es = uint64(Kdata) - c.registers.Ss = uint64(Kdata) - c.registers.Fs = uint64(Kdata) - c.registers.Gs = uint64(Kdata) - - // Set mandatory flags. - c.registers.Eflags = KernelFlagsSet -} - -// StackTop returns the kernel's stack address. -// -//go:nosplit -func (c *CPU) StackTop() uint64 { - return uint64(kernelAddr(&c.stack[0])) + uint64(len(c.stack)) -} - -// IDT returns the CPU's IDT base and limit. -// -//go:nosplit -func (c *CPU) IDT() (uint64, uint16) { - return uint64(kernelAddr(&c.kernel.globalIDT[0])), uint16(binary.Size(&c.kernel.globalIDT) - 1) -} - -// GDT returns the CPU's GDT base and limit. -// -//go:nosplit -func (c *CPU) GDT() (uint64, uint16) { - return uint64(kernelAddr(&c.gdt[0])), uint16(8*segLast - 1) -} - -// TSS returns the CPU's TSS base, limit and value. -// -//go:nosplit -func (c *CPU) TSS() (uint64, uint16, *SegmentDescriptor) { - return uint64(kernelAddr(&c.tss)), uint16(binary.Size(&c.tss) - 1), &c.gdt[segTss] -} - -// CR0 returns the CPU's CR0 value. -// -//go:nosplit -func (c *CPU) CR0() uint64 { - return _CR0_PE | _CR0_PG | _CR0_AM | _CR0_ET -} - -// CR4 returns the CPU's CR4 value. -// -//go:nosplit -func (c *CPU) CR4() uint64 { - cr4 := uint64(_CR4_PAE | _CR4_PSE | _CR4_OSFXSR | _CR4_OSXMMEXCPT) - if hasPCID { - cr4 |= _CR4_PCIDE - } - if hasXSAVE { - cr4 |= _CR4_OSXSAVE - } - if hasSMEP { - cr4 |= _CR4_SMEP - } - if hasFSGSBASE { - cr4 |= _CR4_FSGSBASE - } - return cr4 -} - -// EFER returns the CPU's EFER value. -// -//go:nosplit -func (c *CPU) EFER() uint64 { - return _EFER_LME | _EFER_LMA | _EFER_SCE | _EFER_NX -} - -// IsCanonical indicates whether addr is canonical per the amd64 spec. -// -//go:nosplit -func IsCanonical(addr uint64) bool { - return addr <= 0x00007fffffffffff || addr > 0xffff800000000000 -} - -// SwitchToUser performs either a sysret or an iret. -// -// The return value is the vector that interrupted execution. -// -// This function will not split the stack. Callers will probably want to call -// runtime.entersyscall (and pair with a call to runtime.exitsyscall) prior to -// calling this function. -// -// When this is done, this region is quite sensitive to things like system -// calls. After calling entersyscall, any memory used must have been allocated -// and no function calls without go:nosplit are permitted. Any calls made here -// are protected appropriately (e.g. IsCanonical and CR3). -// -// Also note that this function transitively depends on the compiler generating -// code that uses IP-relative addressing inside of absolute addresses. That's -// the case for amd64, but may not be the case for other architectures. -// -// Precondition: the Rip, Rsp, Fs and Gs registers must be canonical. -// -// +checkescape:all -// -//go:nosplit -func (c *CPU) SwitchToUser(switchOpts SwitchOpts) (vector Vector) { - userCR3 := switchOpts.PageTables.CR3(!switchOpts.Flush, switchOpts.UserPCID) - c.kernelCR3 = uintptr(c.kernel.PageTables.CR3(true, switchOpts.KernelPCID)) - - // Sanitize registers. - regs := switchOpts.Registers - regs.Eflags &= ^uint64(UserFlagsClear) - regs.Eflags |= UserFlagsSet - regs.Cs = uint64(Ucode64) // Required for iret. - regs.Ss = uint64(Udata) // Ditto. - - // Perform the switch. - swapgs() // GS will be swapped on return. - WriteFS(uintptr(regs.Fs_base)) // escapes: no. Set application FS. - WriteGS(uintptr(regs.Gs_base)) // escapes: no. Set application GS. - LoadFloatingPoint(switchOpts.FloatingPointState) // escapes: no. Copy in floating point. - if switchOpts.FullRestore { - vector = iret(c, regs, uintptr(userCR3)) - } else { - vector = sysret(c, regs, uintptr(userCR3)) - } - SaveFloatingPoint(switchOpts.FloatingPointState) // escapes: no. Copy out floating point. - WriteFS(uintptr(c.registers.Fs_base)) // escapes: no. Restore kernel FS. - return -} - -// start is the CPU entrypoint. -// -// This is called from the Start asm stub (see entry_amd64.go); on return the -// registers in c.registers will be restored (not segments). -// -//go:nosplit -func start(c *CPU) { - // Save per-cpu & FS segment. - WriteGS(kernelAddr(c.kernelEntry)) - WriteFS(uintptr(c.registers.Fs_base)) - - // Initialize floating point. - // - // Note that on skylake, the valid XCR0 mask reported seems to be 0xff. - // This breaks down as: - // - // bit0 - x87 - // bit1 - SSE - // bit2 - AVX - // bit3-4 - MPX - // bit5-7 - AVX512 - // - // For some reason, enabled MPX & AVX512 on platforms that report them - // seems to be cause a general protection fault. (Maybe there are some - // virtualization issues and these aren't exported to the guest cpuid.) - // This needs further investigation, but we can limit the floating - // point operations to x87, SSE & AVX for now. - fninit() - xsetbv(0, validXCR0Mask&0x7) - - // Set the syscall target. - wrmsr(_MSR_LSTAR, kernelFunc(sysenter)) - wrmsr(_MSR_SYSCALL_MASK, KernelFlagsClear|_RFLAGS_DF) - - // NOTE: This depends on having the 64-bit segments immediately - // following the 32-bit user segments. This is simply the way the - // sysret instruction is designed to work (it assumes they follow). - wrmsr(_MSR_STAR, uintptr(uint64(Kcode)<<32|uint64(Ucode32)<<48)) - wrmsr(_MSR_CSTAR, kernelFunc(sysenter)) -} - -// SetCPUIDFaulting sets CPUID faulting per the boolean value. -// -// True is returned if faulting could be set. -// -//go:nosplit -func SetCPUIDFaulting(on bool) bool { - // Per the SDM (Vol 3, Table 2-43), PLATFORM_INFO bit 31 denotes support - // for CPUID faulting, and we enable and disable via the MISC_FEATURES MSR. - if rdmsr(_MSR_PLATFORM_INFO)&_PLATFORM_INFO_CPUID_FAULT != 0 { - features := rdmsr(_MSR_MISC_FEATURES) - if on { - features |= _MISC_FEATURE_CPUID_TRAP - } else { - features &^= _MISC_FEATURE_CPUID_TRAP - } - wrmsr(_MSR_MISC_FEATURES, features) - return true // Setting successful. - } - return false -} - -// ReadCR2 reads the current CR2 value. -// -//go:nosplit -func ReadCR2() uintptr { - return readCR2() -} diff --git a/pkg/sentry/platform/ring0/kernel_arm64.go b/pkg/sentry/platform/ring0/kernel_arm64.go deleted file mode 100644 index c05284641..000000000 --- a/pkg/sentry/platform/ring0/kernel_arm64.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -package ring0 - -// HaltAndResume halts execution and point the pointer to the resume function. -//go:nosplit -func HaltAndResume() - -// HaltEl1SvcAndResume calls Hooks.KernelSyscall and resume. -//go:nosplit -func HaltEl1SvcAndResume() - -// HaltEl1ExceptionAndResume calls Hooks.KernelException and resume. -//go:nosplit -func HaltEl1ExceptionAndResume() - -// init initializes architecture-specific state. -func (k *Kernel) init(maxCPUs int) { -} - -// init initializes architecture-specific state. -func (c *CPU) init(cpuID int) { - // Set the kernel stack pointer(virtual address). - c.registers.Sp = uint64(c.StackTop()) - -} - -// StackTop returns the kernel's stack address. -// -//go:nosplit -func (c *CPU) StackTop() uint64 { - return uint64(kernelAddr(&c.stack[0])) + uint64(len(c.stack)) -} - -// IsCanonical indicates whether addr is canonical per the arm64 spec. -// -//go:nosplit -func IsCanonical(addr uint64) bool { - return addr <= 0x0000ffffffffffff || addr > 0xffff000000000000 -} - -// SwitchToUser performs an eret. -// -// The return value is the exception vector. -// -// +checkescape:all -// -//go:nosplit -func (c *CPU) SwitchToUser(switchOpts SwitchOpts) (vector Vector) { - storeAppASID(uintptr(switchOpts.UserASID)) - if switchOpts.Flush { - FlushTlbByASID(uintptr(switchOpts.UserASID)) - } - - regs := switchOpts.Registers - - regs.Pstate &= ^uint64(PsrFlagsClear) - regs.Pstate |= UserFlagsSet - - EnableVFP() - LoadFloatingPoint(switchOpts.FloatingPointState) - - kernelExitToEl0() - - SaveFloatingPoint(switchOpts.FloatingPointState) - DisableVFP() - - vector = c.vecCode - - return -} diff --git a/pkg/sentry/platform/ring0/kernel_unsafe.go b/pkg/sentry/platform/ring0/kernel_unsafe.go deleted file mode 100644 index 16955ad91..000000000 --- a/pkg/sentry/platform/ring0/kernel_unsafe.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ring0 - -import ( - "unsafe" -) - -// eface mirrors runtime.eface. -type eface struct { - typ uintptr - data unsafe.Pointer -} - -// kernelAddr returns the kernel virtual address for the given object. -// -//go:nosplit -func kernelAddr(obj interface{}) uintptr { - e := (*eface)(unsafe.Pointer(&obj)) - return KernelStartAddress | uintptr(e.data) -} - -// kernelFunc returns the address of the given function. -// -//go:nosplit -func kernelFunc(fn func()) uintptr { - fnptr := (**uintptr)(unsafe.Pointer(&fn)) - return KernelStartAddress | **fnptr -} diff --git a/pkg/sentry/platform/ring0/lib_amd64.go b/pkg/sentry/platform/ring0/lib_amd64.go deleted file mode 100644 index 0ec5c3bc5..000000000 --- a/pkg/sentry/platform/ring0/lib_amd64.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package ring0 - -import ( - "gvisor.dev/gvisor/pkg/cpuid" -) - -// LoadFloatingPoint loads floating point state by the most efficient mechanism -// available (set by Init). -var LoadFloatingPoint func(*byte) - -// SaveFloatingPoint saves floating point state by the most efficient mechanism -// available (set by Init). -var SaveFloatingPoint func(*byte) - -// fxrstor uses fxrstor64 to load floating point state. -func fxrstor(*byte) - -// xrstor uses xrstor to load floating point state. -func xrstor(*byte) - -// fxsave uses fxsave64 to save floating point state. -func fxsave(*byte) - -// xsave uses xsave to save floating point state. -func xsave(*byte) - -// xsaveopt uses xsaveopt to save floating point state. -func xsaveopt(*byte) - -// WriteFS sets the GS address (set by init). -var WriteFS func(addr uintptr) - -// wrfsbase writes to the GS base address. -func wrfsbase(addr uintptr) - -// wrfsmsr writes to the GS_BASE MSR. -func wrfsmsr(addr uintptr) - -// WriteGS sets the GS address (set by init). -var WriteGS func(addr uintptr) - -// wrgsbase writes to the GS base address. -func wrgsbase(addr uintptr) - -// wrgsmsr writes to the GS_BASE MSR. -func wrgsmsr(addr uintptr) - -// readCR2 reads the current CR2 value. -func readCR2() uintptr - -// fninit initializes the floating point unit. -func fninit() - -// xsetbv writes to an extended control register. -func xsetbv(reg, value uintptr) - -// xgetbv reads an extended control register. -func xgetbv(reg uintptr) uintptr - -// wrmsr reads to the given MSR. -func wrmsr(reg, value uintptr) - -// rdmsr reads the given MSR. -func rdmsr(reg uintptr) uintptr - -// Mostly-constants set by Init. -var ( - hasSMEP bool - hasPCID bool - hasXSAVEOPT bool - hasXSAVE bool - hasFSGSBASE bool - validXCR0Mask uintptr -) - -// Init sets function pointers based on architectural features. -// -// This must be called prior to using ring0. -func Init(featureSet *cpuid.FeatureSet) { - hasSMEP = featureSet.HasFeature(cpuid.X86FeatureSMEP) - hasPCID = featureSet.HasFeature(cpuid.X86FeaturePCID) - hasXSAVEOPT = featureSet.UseXsaveopt() - hasXSAVE = featureSet.UseXsave() - hasFSGSBASE = featureSet.HasFeature(cpuid.X86FeatureFSGSBase) - validXCR0Mask = uintptr(featureSet.ValidXCR0Mask()) - if hasXSAVEOPT { - SaveFloatingPoint = xsaveopt - LoadFloatingPoint = xrstor - } else if hasXSAVE { - SaveFloatingPoint = xsave - LoadFloatingPoint = xrstor - } else { - SaveFloatingPoint = fxsave - LoadFloatingPoint = fxrstor - } - if hasFSGSBASE { - WriteFS = wrfsbase - WriteGS = wrgsbase - } else { - WriteFS = wrfsmsr - WriteGS = wrgsmsr - } -} diff --git a/pkg/sentry/platform/ring0/lib_amd64.s b/pkg/sentry/platform/ring0/lib_amd64.s deleted file mode 100644 index 2fe83568a..000000000 --- a/pkg/sentry/platform/ring0/lib_amd64.s +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "funcdata.h" -#include "textflag.h" - -// fxrstor loads floating point state. -// -// The code corresponds to: -// -// fxrstor64 (%rbx) -// -TEXT ·fxrstor(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), BX - MOVL $0xffffffff, AX - MOVL $0xffffffff, DX - BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x0b; - RET - -// xrstor loads floating point state. -// -// The code corresponds to: -// -// xrstor (%rdi) -// -TEXT ·xrstor(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), DI - MOVL $0xffffffff, AX - MOVL $0xffffffff, DX - BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x2f; - RET - -// fxsave saves floating point state. -// -// The code corresponds to: -// -// fxsave64 (%rbx) -// -TEXT ·fxsave(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), BX - MOVL $0xffffffff, AX - MOVL $0xffffffff, DX - BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x03; - RET - -// xsave saves floating point state. -// -// The code corresponds to: -// -// xsave (%rdi) -// -TEXT ·xsave(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), DI - MOVL $0xffffffff, AX - MOVL $0xffffffff, DX - BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x27; - RET - -// xsaveopt saves floating point state. -// -// The code corresponds to: -// -// xsaveopt (%rdi) -// -TEXT ·xsaveopt(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), DI - MOVL $0xffffffff, AX - MOVL $0xffffffff, DX - BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x37; - RET - -// wrfsbase writes to the FS base. -// -// The code corresponds to: -// -// wrfsbase %rax -// -TEXT ·wrfsbase(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), AX - BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0xd0; - RET - -// wrfsmsr writes to the FSBASE MSR. -// -// The code corresponds to: -// -// wrmsr (writes EDX:EAX to the MSR in ECX) -// -TEXT ·wrfsmsr(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), AX - MOVQ AX, DX - SHRQ $32, DX - MOVQ $0xc0000100, CX // MSR_FS_BASE - BYTE $0x0f; BYTE $0x30; - RET - -// wrgsbase writes to the GS base. -// -// The code corresponds to: -// -// wrgsbase %rax -// -TEXT ·wrgsbase(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), AX - BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0xd8; - RET - -// wrgsmsr writes to the GSBASE MSR. -// -// See wrfsmsr. -TEXT ·wrgsmsr(SB),NOSPLIT,$0-8 - MOVQ addr+0(FP), AX - MOVQ AX, DX - SHRQ $32, DX - MOVQ $0xc0000101, CX // MSR_GS_BASE - BYTE $0x0f; BYTE $0x30; // WRMSR - RET - -// readCR2 reads the current CR2 value. -// -// The code corresponds to: -// -// mov %cr2, %rax -// -TEXT ·readCR2(SB),NOSPLIT,$0-8 - BYTE $0x0f; BYTE $0x20; BYTE $0xd0; - MOVQ AX, ret+0(FP) - RET - -// fninit initializes the floating point unit. -// -// The code corresponds to: -// -// fninit -TEXT ·fninit(SB),NOSPLIT,$0 - BYTE $0xdb; BYTE $0xe3; - RET - -// xsetbv writes to an extended control register. -// -// The code corresponds to: -// -// xsetbv -// -TEXT ·xsetbv(SB),NOSPLIT,$0-16 - MOVL reg+0(FP), CX - MOVL value+8(FP), AX - MOVL value+12(FP), DX - BYTE $0x0f; BYTE $0x01; BYTE $0xd1; - RET - -// xgetbv reads an extended control register. -// -// The code corresponds to: -// -// xgetbv -// -TEXT ·xgetbv(SB),NOSPLIT,$0-16 - MOVL reg+0(FP), CX - BYTE $0x0f; BYTE $0x01; BYTE $0xd0; - MOVL AX, ret+8(FP) - MOVL DX, ret+12(FP) - RET - -// wrmsr writes to a control register. -// -// The code corresponds to: -// -// wrmsr -// -TEXT ·wrmsr(SB),NOSPLIT,$0-16 - MOVL reg+0(FP), CX - MOVL value+8(FP), AX - MOVL value+12(FP), DX - BYTE $0x0f; BYTE $0x30; - RET - -// rdmsr reads a control register. -// -// The code corresponds to: -// -// rdmsr -// -TEXT ·rdmsr(SB),NOSPLIT,$0-16 - MOVL reg+0(FP), CX - BYTE $0x0f; BYTE $0x32; - MOVL AX, ret+8(FP) - MOVL DX, ret+12(FP) - RET diff --git a/pkg/sentry/platform/ring0/lib_arm64.go b/pkg/sentry/platform/ring0/lib_arm64.go deleted file mode 100644 index a490bf3af..000000000 --- a/pkg/sentry/platform/ring0/lib_arm64.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -package ring0 - -// storeAppASID writes the application's asid value. -func storeAppASID(asid uintptr) - -// LocalFlushTlbAll same as FlushTlbAll, but only applies to the calling CPU. -func LocalFlushTlbAll() - -// FlushTlbByVA invalidates tlb by VA/Last-level/Inner-Shareable. -func FlushTlbByVA(addr uintptr) - -// FlushTlbByASID invalidates tlb by ASID/Inner-Shareable. -func FlushTlbByASID(asid uintptr) - -// FlushTlbAll invalidates all tlb. -func FlushTlbAll() - -// CPACREL1 returns the value of the CPACR_EL1 register. -func CPACREL1() (value uintptr) - -// GetFPCR returns the value of FPCR register. -func GetFPCR() (value uintptr) - -// SetFPCR writes the FPCR value. -func SetFPCR(value uintptr) - -// GetFPSR returns the value of FPSR register. -func GetFPSR() (value uintptr) - -// SetFPSR writes the FPSR value. -func SetFPSR(value uintptr) - -// SaveVRegs saves V0-V31 registers. -// V0-V31: 32 128-bit registers for floating point and simd. -func SaveVRegs(*byte) - -// LoadVRegs loads V0-V31 registers. -func LoadVRegs(*byte) - -// LoadFloatingPoint loads floating point state. -func LoadFloatingPoint(*byte) - -// SaveFloatingPoint saves floating point state. -func SaveFloatingPoint(*byte) - -// EnableVFP enables fpsimd. -func EnableVFP() - -// DisableVFP disables fpsimd. -func DisableVFP() - -// Init sets function pointers based on architectural features. -// -// This must be called prior to using ring0. -func Init() {} diff --git a/pkg/sentry/platform/ring0/lib_arm64.s b/pkg/sentry/platform/ring0/lib_arm64.s deleted file mode 100644 index e39b32841..000000000 --- a/pkg/sentry/platform/ring0/lib_arm64.s +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "funcdata.h" -#include "textflag.h" - -#define TLBI_ASID_SHIFT 48 - -TEXT ·FlushTlbByVA(SB),NOSPLIT,$0-8 - MOVD addr+0(FP), R1 - DSB $10 // dsb(ishst) - WORD $0xd50883a1 // tlbi vale1is, x1 - DSB $11 // dsb(ish) - RET - -TEXT ·FlushTlbByASID(SB),NOSPLIT,$0-8 - MOVD asid+0(FP), R1 - LSL $TLBI_ASID_SHIFT, R1, R1 - DSB $10 // dsb(ishst) - WORD $0xd5088341 // tlbi aside1is, x1 - DSB $11 // dsb(ish) - RET - -TEXT ·LocalFlushTlbAll(SB),NOSPLIT,$0 - DSB $6 // dsb(nshst) - WORD $0xd508871f // __tlbi(vmalle1) - DSB $7 // dsb(nsh) - ISB $15 - RET - -TEXT ·FlushTlbAll(SB),NOSPLIT,$0 - DSB $10 // dsb(ishst) - WORD $0xd508831f // __tlbi(vmalle1is) - DSB $11 // dsb(ish) - ISB $15 - RET - -TEXT ·CPACREL1(SB),NOSPLIT,$0-8 - WORD $0xd5381041 // MRS CPACR_EL1, R1 - MOVD R1, ret+0(FP) - RET - -TEXT ·GetFPCR(SB),NOSPLIT,$0-8 - MOVD FPCR, R1 - MOVD R1, ret+0(FP) - RET - -TEXT ·GetFPSR(SB),NOSPLIT,$0-8 - MOVD FPSR, R1 - MOVD R1, ret+0(FP) - RET - -TEXT ·SetFPCR(SB),NOSPLIT,$0-8 - MOVD addr+0(FP), R1 - MOVD R1, FPCR - RET - -TEXT ·SetFPSR(SB),NOSPLIT,$0-8 - MOVD addr+0(FP), R1 - MOVD R1, FPSR - RET - -TEXT ·SaveVRegs(SB),NOSPLIT,$0-8 - MOVD addr+0(FP), R0 - - // Skip aarch64_ctx, fpsr, fpcr. - ADD $16, R0, R0 - - WORD $0xad000400 // stp q0, q1, [x0] - WORD $0xad010c02 // stp q2, q3, [x0, #32] - WORD $0xad021404 // stp q4, q5, [x0, #64] - WORD $0xad031c06 // stp q6, q7, [x0, #96] - WORD $0xad042408 // stp q8, q9, [x0, #128] - WORD $0xad052c0a // stp q10, q11, [x0, #160] - WORD $0xad06340c // stp q12, q13, [x0, #192] - WORD $0xad073c0e // stp q14, q15, [x0, #224] - WORD $0xad084410 // stp q16, q17, [x0, #256] - WORD $0xad094c12 // stp q18, q19, [x0, #288] - WORD $0xad0a5414 // stp q20, q21, [x0, #320] - WORD $0xad0b5c16 // stp q22, q23, [x0, #352] - WORD $0xad0c6418 // stp q24, q25, [x0, #384] - WORD $0xad0d6c1a // stp q26, q27, [x0, #416] - WORD $0xad0e741c // stp q28, q29, [x0, #448] - WORD $0xad0f7c1e // stp q30, q31, [x0, #480] - - RET - -TEXT ·LoadVRegs(SB),NOSPLIT,$0-8 - MOVD addr+0(FP), R0 - - // Skip aarch64_ctx, fpsr, fpcr. - ADD $16, R0, R0 - - WORD $0xad400400 // ldp q0, q1, [x0] - WORD $0xad410c02 // ldp q2, q3, [x0, #32] - WORD $0xad421404 // ldp q4, q5, [x0, #64] - WORD $0xad431c06 // ldp q6, q7, [x0, #96] - WORD $0xad442408 // ldp q8, q9, [x0, #128] - WORD $0xad452c0a // ldp q10, q11, [x0, #160] - WORD $0xad46340c // ldp q12, q13, [x0, #192] - WORD $0xad473c0e // ldp q14, q15, [x0, #224] - WORD $0xad484410 // ldp q16, q17, [x0, #256] - WORD $0xad494c12 // ldp q18, q19, [x0, #288] - WORD $0xad4a5414 // ldp q20, q21, [x0, #320] - WORD $0xad4b5c16 // ldp q22, q23, [x0, #352] - WORD $0xad4c6418 // ldp q24, q25, [x0, #384] - WORD $0xad4d6c1a // ldp q26, q27, [x0, #416] - WORD $0xad4e741c // ldp q28, q29, [x0, #448] - WORD $0xad4f7c1e // ldp q30, q31, [x0, #480] - - RET - -TEXT ·LoadFloatingPoint(SB),NOSPLIT,$0-8 - MOVD addr+0(FP), R0 - - MOVD 0(R0), R1 - MOVD R1, FPSR - MOVD 8(R0), R1 - MOVD R1, FPCR - - ADD $16, R0, R0 - - WORD $0xad400400 // ldp q0, q1, [x0] - WORD $0xad410c02 // ldp q2, q3, [x0, #32] - WORD $0xad421404 // ldp q4, q5, [x0, #64] - WORD $0xad431c06 // ldp q6, q7, [x0, #96] - WORD $0xad442408 // ldp q8, q9, [x0, #128] - WORD $0xad452c0a // ldp q10, q11, [x0, #160] - WORD $0xad46340c // ldp q12, q13, [x0, #192] - WORD $0xad473c0e // ldp q14, q15, [x0, #224] - WORD $0xad484410 // ldp q16, q17, [x0, #256] - WORD $0xad494c12 // ldp q18, q19, [x0, #288] - WORD $0xad4a5414 // ldp q20, q21, [x0, #320] - WORD $0xad4b5c16 // ldp q22, q23, [x0, #352] - WORD $0xad4c6418 // ldp q24, q25, [x0, #384] - WORD $0xad4d6c1a // ldp q26, q27, [x0, #416] - WORD $0xad4e741c // ldp q28, q29, [x0, #448] - WORD $0xad4f7c1e // ldp q30, q31, [x0, #480] - - RET - -TEXT ·SaveFloatingPoint(SB),NOSPLIT,$0-8 - MOVD addr+0(FP), R0 - - MOVD FPSR, R1 - MOVD R1, 0(R0) - MOVD FPCR, R1 - MOVD R1, 8(R0) - - ADD $16, R0, R0 - - WORD $0xad000400 // stp q0, q1, [x0] - WORD $0xad010c02 // stp q2, q3, [x0, #32] - WORD $0xad021404 // stp q4, q5, [x0, #64] - WORD $0xad031c06 // stp q6, q7, [x0, #96] - WORD $0xad042408 // stp q8, q9, [x0, #128] - WORD $0xad052c0a // stp q10, q11, [x0, #160] - WORD $0xad06340c // stp q12, q13, [x0, #192] - WORD $0xad073c0e // stp q14, q15, [x0, #224] - WORD $0xad084410 // stp q16, q17, [x0, #256] - WORD $0xad094c12 // stp q18, q19, [x0, #288] - WORD $0xad0a5414 // stp q20, q21, [x0, #320] - WORD $0xad0b5c16 // stp q22, q23, [x0, #352] - WORD $0xad0c6418 // stp q24, q25, [x0, #384] - WORD $0xad0d6c1a // stp q26, q27, [x0, #416] - WORD $0xad0e741c // stp q28, q29, [x0, #448] - WORD $0xad0f7c1e // stp q30, q31, [x0, #480] - - RET diff --git a/pkg/sentry/platform/ring0/pagetables/allocator.go b/pkg/sentry/platform/ring0/pagetables/allocator.go deleted file mode 100644 index 8d75b7599..000000000 --- a/pkg/sentry/platform/ring0/pagetables/allocator.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagetables - -// Allocator is used to allocate and map PTEs. -// -// Note that allocators may be called concurrently. -type Allocator interface { - // NewPTEs returns a new set of PTEs and their physical address. - NewPTEs() *PTEs - - // PhysicalFor gives the physical address for a set of PTEs. - PhysicalFor(ptes *PTEs) uintptr - - // LookupPTEs looks up PTEs by physical address. - LookupPTEs(physical uintptr) *PTEs - - // FreePTEs marks a set of PTEs a freed, although they may not be available - // for use again until Recycle is called, below. - FreePTEs(ptes *PTEs) - - // Recycle makes freed PTEs available for use again. - Recycle() -} - -// RuntimeAllocator is a trivial allocator. -type RuntimeAllocator struct { - // used is the set of PTEs that have been allocated. This includes any - // PTEs that may be in the pool below. PTEs are only freed from this - // map by the Drain call. - // - // This exists to prevent accidental garbage collection. - used map[*PTEs]struct{} - - // pool is the set of free-to-use PTEs. - pool []*PTEs - - // freed is the set of recently-freed PTEs. - freed []*PTEs -} - -// NewRuntimeAllocator returns an allocator that uses runtime allocation. -func NewRuntimeAllocator() *RuntimeAllocator { - r := new(RuntimeAllocator) - r.Init() - return r -} - -// Init initializes a RuntimeAllocator. -func (r *RuntimeAllocator) Init() { - r.used = make(map[*PTEs]struct{}) -} - -// Recycle returns freed pages to the pool. -func (r *RuntimeAllocator) Recycle() { - r.pool = append(r.pool, r.freed...) - r.freed = r.freed[:0] -} - -// Drain empties the pool. -func (r *RuntimeAllocator) Drain() { - r.Recycle() - for i, ptes := range r.pool { - // Zap the entry in the underlying array to ensure that it can - // be properly garbage collected. - r.pool[i] = nil - // Similarly, free the reference held by the used map (these - // also apply for the pool entries). - delete(r.used, ptes) - } - r.pool = r.pool[:0] -} - -// NewPTEs implements Allocator.NewPTEs. -// -// Note that the "physical" address here is actually the virtual address of the -// PTEs structure. The entries are tracked only to avoid garbage collection. -// -// This is guaranteed not to split as long as the pool is sufficiently full. -// -//go:nosplit -func (r *RuntimeAllocator) NewPTEs() *PTEs { - // Pull from the pool if we can. - if len(r.pool) > 0 { - ptes := r.pool[len(r.pool)-1] - r.pool = r.pool[:len(r.pool)-1] - return ptes - } - - // Allocate a new entry. - ptes := newAlignedPTEs() - r.used[ptes] = struct{}{} - return ptes -} - -// PhysicalFor returns the physical address for the given PTEs. -// -//go:nosplit -func (r *RuntimeAllocator) PhysicalFor(ptes *PTEs) uintptr { - return physicalFor(ptes) -} - -// LookupPTEs implements Allocator.LookupPTEs. -// -//go:nosplit -func (r *RuntimeAllocator) LookupPTEs(physical uintptr) *PTEs { - return fromPhysical(physical) -} - -// FreePTEs implements Allocator.FreePTEs. -// -//go:nosplit -func (r *RuntimeAllocator) FreePTEs(ptes *PTEs) { - r.freed = append(r.freed, ptes) -} diff --git a/pkg/sentry/platform/ring0/pagetables/allocator_unsafe.go b/pkg/sentry/platform/ring0/pagetables/allocator_unsafe.go deleted file mode 100644 index d08bfdeb3..000000000 --- a/pkg/sentry/platform/ring0/pagetables/allocator_unsafe.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagetables - -import ( - "unsafe" - - "gvisor.dev/gvisor/pkg/usermem" -) - -// newAlignedPTEs returns a set of aligned PTEs. -func newAlignedPTEs() *PTEs { - ptes := new(PTEs) - offset := physicalFor(ptes) & (usermem.PageSize - 1) - if offset == 0 { - // Already aligned. - return ptes - } - - // Need to force an aligned allocation. - unaligned := make([]byte, (2*usermem.PageSize)-1) - offset = uintptr(unsafe.Pointer(&unaligned[0])) & (usermem.PageSize - 1) - if offset != 0 { - offset = usermem.PageSize - offset - } - return (*PTEs)(unsafe.Pointer(&unaligned[offset])) -} - -// physicalFor returns the "physical" address for PTEs. -// -//go:nosplit -func physicalFor(ptes *PTEs) uintptr { - return uintptr(unsafe.Pointer(ptes)) -} - -// fromPhysical returns the PTEs from the "physical" address. -// -//go:nosplit -func fromPhysical(physical uintptr) *PTEs { - return (*PTEs)(unsafe.Pointer(physical)) -} diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables.go b/pkg/sentry/platform/ring0/pagetables/pagetables.go deleted file mode 100644 index 8c0a6aa82..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables.go +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package pagetables provides a generic implementation of pagetables. -// -// The core functions must be safe to call from a nosplit context. Furthermore, -// this pagetables implementation goes to lengths to ensure that all functions -// are free from runtime allocation. Calls to NewPTEs/FreePTEs may be made -// during walks, but these can be cached elsewhere if required. -package pagetables - -import ( - "gvisor.dev/gvisor/pkg/usermem" -) - -// PageTables is a set of page tables. -type PageTables struct { - // Allocator is used to allocate nodes. - Allocator Allocator - - // root is the pagetable root. - // - // For same archs such as amd64, the upper of the PTEs is cloned - // from and owned by upperSharedPageTables which are shared among - // many PageTables if upperSharedPageTables is not nil. - root *PTEs - - // rootPhysical is the cached physical address of the root. - // - // This is saved only to prevent constant translation. - rootPhysical uintptr - - // archPageTables includes architecture-specific features. - archPageTables - - // upperSharedPageTables represents a read-only shared upper - // of the Pagetable. When it is not nil, the upper is not - // allowed to be modified. - upperSharedPageTables *PageTables - - // upperStart is the start address of the upper portion that - // are shared from upperSharedPageTables - upperStart uintptr - - // readOnlyShared indicates the Pagetables are read-only and - // own the ranges that are shared with other Pagetables. - readOnlyShared bool -} - -// Init initializes a set of PageTables. -// -// +checkescape:hard,stack -//go:nosplit -func (p *PageTables) Init(allocator Allocator) { - p.Allocator = allocator - p.root = p.Allocator.NewPTEs() - p.rootPhysical = p.Allocator.PhysicalFor(p.root) -} - -// NewWithUpper returns new PageTables. -// -// upperSharedPageTables are used for mapping the upper of addresses, -// starting at upperStart. These pageTables should not be touched (as -// invalidations may be incorrect) after they are passed as an -// upperSharedPageTables. Only when all dependent PageTables are gone -// may they be used. The intenteded use case is for kernel page tables, -// which are static and fixed. -// -// Precondition: upperStart must be between canonical ranges. -// Precondition: upperStart must be pgdSize aligned. -// precondition: upperSharedPageTables must be marked read-only shared. -func NewWithUpper(a Allocator, upperSharedPageTables *PageTables, upperStart uintptr) *PageTables { - p := new(PageTables) - p.Init(a) - - if upperSharedPageTables != nil { - if !upperSharedPageTables.readOnlyShared { - panic("Only read-only shared pagetables can be used as upper") - } - p.upperSharedPageTables = upperSharedPageTables - p.upperStart = upperStart - } - - p.InitArch(a) - return p -} - -// New returns new PageTables. -func New(a Allocator) *PageTables { - return NewWithUpper(a, nil, 0) -} - -// mapVisitor is used for map. -type mapVisitor struct { - target uintptr // Input. - physical uintptr // Input. - opts MapOpts // Input. - prev bool // Output. -} - -// visit is used for map. -// -//go:nosplit -func (v *mapVisitor) visit(start uintptr, pte *PTE, align uintptr) bool { - p := v.physical + (start - uintptr(v.target)) - if pte.Valid() && (pte.Address() != p || pte.Opts() != v.opts) { - v.prev = true - } - if p&align != 0 { - // We will install entries at a smaller granulaity if we don't - // install a valid entry here, however we must zap any existing - // entry to ensure this happens. - pte.Clear() - return true - } - pte.Set(p, v.opts) - return true -} - -//go:nosplit -func (*mapVisitor) requiresAlloc() bool { return true } - -//go:nosplit -func (*mapVisitor) requiresSplit() bool { return true } - -// Map installs a mapping with the given physical address. -// -// True is returned iff there was a previous mapping in the range. -// -// Precondition: addr & length must be page-aligned, their sum must not overflow. -// -// +checkescape:hard,stack -//go:nosplit -func (p *PageTables) Map(addr usermem.Addr, length uintptr, opts MapOpts, physical uintptr) bool { - if p.readOnlyShared { - panic("Should not modify read-only shared pagetables.") - } - if uintptr(addr)+length < uintptr(addr) { - panic("addr & length overflow") - } - if p.upperSharedPageTables != nil { - // ignore change to the read-only upper shared portion. - if uintptr(addr) >= p.upperStart { - return false - } - if uintptr(addr)+length > p.upperStart { - length = p.upperStart - uintptr(addr) - } - } - w := mapWalker{ - pageTables: p, - visitor: mapVisitor{ - target: uintptr(addr), - physical: physical, - opts: opts, - }, - } - w.iterateRange(uintptr(addr), uintptr(addr)+length) - return w.visitor.prev -} - -// unmapVisitor is used for unmap. -type unmapVisitor struct { - count int -} - -//go:nosplit -func (*unmapVisitor) requiresAlloc() bool { return false } - -//go:nosplit -func (*unmapVisitor) requiresSplit() bool { return true } - -// visit unmaps the given entry. -// -//go:nosplit -func (v *unmapVisitor) visit(start uintptr, pte *PTE, align uintptr) bool { - pte.Clear() - v.count++ - return true -} - -// Unmap unmaps the given range. -// -// True is returned iff there was a previous mapping in the range. -// -// Precondition: addr & length must be page-aligned, their sum must not overflow. -// -// +checkescape:hard,stack -//go:nosplit -func (p *PageTables) Unmap(addr usermem.Addr, length uintptr) bool { - if p.readOnlyShared { - panic("Should not modify read-only shared pagetables.") - } - if uintptr(addr)+length < uintptr(addr) { - panic("addr & length overflow") - } - if p.upperSharedPageTables != nil { - // ignore change to the read-only upper shared portion. - if uintptr(addr) >= p.upperStart { - return false - } - if uintptr(addr)+length > p.upperStart { - length = p.upperStart - uintptr(addr) - } - } - w := unmapWalker{ - pageTables: p, - visitor: unmapVisitor{ - count: 0, - }, - } - w.iterateRange(uintptr(addr), uintptr(addr)+length) - return w.visitor.count > 0 -} - -// emptyVisitor is used for emptiness checks. -type emptyVisitor struct { - count int -} - -//go:nosplit -func (*emptyVisitor) requiresAlloc() bool { return false } - -//go:nosplit -func (*emptyVisitor) requiresSplit() bool { return false } - -// visit unmaps the given entry. -// -//go:nosplit -func (v *emptyVisitor) visit(start uintptr, pte *PTE, align uintptr) bool { - v.count++ - return true -} - -// IsEmpty checks if the given range is empty. -// -// Precondition: addr & length must be page-aligned. -// -// +checkescape:hard,stack -//go:nosplit -func (p *PageTables) IsEmpty(addr usermem.Addr, length uintptr) bool { - w := emptyWalker{ - pageTables: p, - } - w.iterateRange(uintptr(addr), uintptr(addr)+length) - return w.visitor.count == 0 -} - -// lookupVisitor is used for lookup. -type lookupVisitor struct { - target uintptr // Input & Output. - findFirst bool // Input. - physical uintptr // Output. - size uintptr // Output. - opts MapOpts // Output. -} - -// visit matches the given address. -// -//go:nosplit -func (v *lookupVisitor) visit(start uintptr, pte *PTE, align uintptr) bool { - if !pte.Valid() { - // If looking for the first, then we just keep iterating until - // we find a valid entry. - return v.findFirst - } - // Is this within the current range? - v.target = start - v.physical = pte.Address() - v.size = (align + 1) - v.opts = pte.Opts() - return false -} - -//go:nosplit -func (*lookupVisitor) requiresAlloc() bool { return false } - -//go:nosplit -func (*lookupVisitor) requiresSplit() bool { return false } - -// Lookup returns the physical address for the given virtual address. -// -// If findFirst is true, then the next valid address after addr is returned. -// If findFirst is false, then only a mapping for addr will be returned. -// -// Note that if size is zero, then no matching entry was found. -// -// +checkescape:hard,stack -//go:nosplit -func (p *PageTables) Lookup(addr usermem.Addr, findFirst bool) (virtual usermem.Addr, physical, size uintptr, opts MapOpts) { - mask := uintptr(usermem.PageSize - 1) - addr &^= usermem.Addr(mask) - w := lookupWalker{ - pageTables: p, - visitor: lookupVisitor{ - target: uintptr(addr), - findFirst: findFirst, - }, - } - end := ^usermem.Addr(0) &^ usermem.Addr(mask) - if !findFirst { - end = addr + 1 - } - w.iterateRange(uintptr(addr), uintptr(end)) - return usermem.Addr(w.visitor.target), w.visitor.physical, w.visitor.size, w.visitor.opts -} - -// MarkReadOnlyShared marks the pagetables read-only and can be shared. -// -// It is usually used on the pagetables that are used as the upper -func (p *PageTables) MarkReadOnlyShared() { - p.readOnlyShared = true -} diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_aarch64.go b/pkg/sentry/platform/ring0/pagetables/pagetables_aarch64.go deleted file mode 100644 index 163a3aea3..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_aarch64.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -package pagetables - -import ( - "sync/atomic" - - "gvisor.dev/gvisor/pkg/usermem" -) - -// archPageTables is architecture-specific data. -type archPageTables struct { - // root is the pagetable root for kernel space. - root *PTEs - - // rootPhysical is the cached physical address of the root. - // - // This is saved only to prevent constant translation. - rootPhysical uintptr - - asid uint16 -} - -// TTBR0_EL1 returns the translation table base register 0. -// -//go:nosplit -func (p *PageTables) TTBR0_EL1(noFlush bool, asid uint16) uint64 { - return uint64(p.rootPhysical) | (uint64(asid)&ttbrASIDMask)<<ttbrASIDOffset -} - -// TTBR1_EL1 returns the translation table base register 1. -// -//go:nosplit -func (p *PageTables) TTBR1_EL1(noFlush bool, asid uint16) uint64 { - return uint64(p.archPageTables.rootPhysical) | (uint64(asid)&ttbrASIDMask)<<ttbrASIDOffset -} - -// Bits in page table entries. -const ( - typeTable = 0x3 << 0 - typeSect = 0x1 << 0 - typePage = 0x3 << 0 - pteValid = 0x1 << 0 - pteTableBit = 0x1 << 1 - pteTypeMask = 0x3 << 0 - present = pteValid | pteTableBit - user = 0x1 << 6 /* AP[1] */ - readOnly = 0x1 << 7 /* AP[2] */ - accessed = 0x1 << 10 - dbm = 0x1 << 51 - writable = dbm - cont = 0x1 << 52 - pxn = 0x1 << 53 - xn = 0x1 << 54 - dirty = 0x1 << 55 - nG = 0x1 << 11 - shared = 0x3 << 8 -) - -const ( - mtDevicenGnRE = 0x1 << 2 - mtNormal = 0x4 << 2 -) - -const ( - executeDisable = xn - optionMask = 0xfff | 0xffff<<48 - protDefault = accessed | shared -) - -// MapOpts are x86 options. -type MapOpts struct { - // AccessType defines permissions. - AccessType usermem.AccessType - - // Global indicates the page is globally accessible. - Global bool - - // User indicates the page is a user page. - User bool -} - -// PTE is a page table entry. -type PTE uintptr - -// Clear clears this PTE, including sect page information. -// -//go:nosplit -func (p *PTE) Clear() { - atomic.StoreUintptr((*uintptr)(p), 0) -} - -// Valid returns true iff this entry is valid. -// -//go:nosplit -func (p *PTE) Valid() bool { - return atomic.LoadUintptr((*uintptr)(p))&present != 0 -} - -// Opts returns the PTE options. -// -// These are all options except Valid and Sect. -// -//go:nosplit -func (p *PTE) Opts() MapOpts { - v := atomic.LoadUintptr((*uintptr)(p)) - - return MapOpts{ - AccessType: usermem.AccessType{ - Read: true, - Write: v&readOnly == 0, - Execute: v&xn == 0, - }, - Global: v&nG == 0, - User: v&user != 0, - } -} - -// SetSect sets this page as a sect page. -// -// The page must not be valid or a panic will result. -// -//go:nosplit -func (p *PTE) SetSect() { - if p.Valid() { - // This is not allowed. - panic("SetSect called on valid page!") - } - atomic.StoreUintptr((*uintptr)(p), typeSect) -} - -// IsSect returns true iff this page is a sect page. -// -//go:nosplit -func (p *PTE) IsSect() bool { - return atomic.LoadUintptr((*uintptr)(p))&pteTypeMask == typeSect -} - -// Set sets this PTE value. -// -// This does not change the sect page property. -// -//go:nosplit -func (p *PTE) Set(addr uintptr, opts MapOpts) { - v := (addr &^ optionMask) | nG | readOnly | protDefault - if p.IsSect() { - // Note that this is inherited from the previous instance. Set - // does not change the value of Sect. See above. - v |= typeSect - } else { - v |= typePage - } - if !opts.AccessType.Any() { - // Leave as non-valid if no access is available. - v &^= pteValid - } - - if opts.Global { - v = v &^ nG - } - - if opts.AccessType.Execute { - v = v &^ executeDisable - } else { - v |= executeDisable - } - if opts.AccessType.Write { - v = v &^ readOnly - } - - if opts.User { - v |= user - v |= mtNormal - } else { - v = v &^ user - v |= mtNormal - } - atomic.StoreUintptr((*uintptr)(p), v) -} - -// setPageTable sets this PTE value and forces the write bit and sect bit to -// be cleared. This is used explicitly for breaking sect pages. -// -//go:nosplit -func (p *PTE) setPageTable(pt *PageTables, ptes *PTEs) { - addr := pt.Allocator.PhysicalFor(ptes) - if addr&^optionMask != addr { - // This should never happen. - panic("unaligned physical address!") - } - v := addr | typeTable | protDefault | mtNormal - atomic.StoreUintptr((*uintptr)(p), v) -} - -// Address extracts the address. This should only be used if Valid returns true. -// -//go:nosplit -func (p *PTE) Address() uintptr { - return atomic.LoadUintptr((*uintptr)(p)) &^ optionMask -} diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_aarch64_state_autogen.go b/pkg/sentry/platform/ring0/pagetables/pagetables_aarch64_state_autogen.go deleted file mode 100644 index a24523f87..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_aarch64_state_autogen.go +++ /dev/null @@ -1,6 +0,0 @@ -// automatically generated by stateify. - -// +build arm64 -// +build arm64 - -package pagetables diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64.go b/pkg/sentry/platform/ring0/pagetables/pagetables_amd64.go deleted file mode 100644 index a217f404c..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagetables - -// Address constraints. -// -// The lowerTop and upperBottom currently apply to four-level pagetables; -// additional refactoring would be necessary to support five-level pagetables. -const ( - lowerTop = 0x00007fffffffffff - upperBottom = 0xffff800000000000 - - pteShift = 12 - pmdShift = 21 - pudShift = 30 - pgdShift = 39 - - pteMask = 0x1ff << pteShift - pmdMask = 0x1ff << pmdShift - pudMask = 0x1ff << pudShift - pgdMask = 0x1ff << pgdShift - - pteSize = 1 << pteShift - pmdSize = 1 << pmdShift - pudSize = 1 << pudShift - pgdSize = 1 << pgdShift - - executeDisable = 1 << 63 - entriesPerPage = 512 -) - -// InitArch does some additional initialization related to the architecture. -// -// +checkescape:hard,stack -//go:nosplit -func (p *PageTables) InitArch(allocator Allocator) { - if p.upperSharedPageTables != nil { - p.cloneUpperShared() - } -} - -//go:nosplit -func pgdIndex(upperStart uintptr) uintptr { - if upperStart&(pgdSize-1) != 0 { - panic("upperStart should be pgd size aligned") - } - if upperStart >= upperBottom { - return entriesPerPage/2 + (upperStart-upperBottom)/pgdSize - } - if upperStart < lowerTop { - return upperStart / pgdSize - } - panic("upperStart should be in canonical range") -} - -// cloneUpperShared clone the upper from the upper shared page tables. -// -//go:nosplit -func (p *PageTables) cloneUpperShared() { - start := pgdIndex(p.upperStart) - copy(p.root[start:entriesPerPage], p.upperSharedPageTables.root[start:entriesPerPage]) -} - -// PTEs is a collection of entries. -type PTEs [entriesPerPage]PTE diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_state_autogen.go b/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_state_autogen.go deleted file mode 100644 index f48a8acd1..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_amd64_state_autogen.go +++ /dev/null @@ -1,5 +0,0 @@ -// automatically generated by stateify. - -// +build amd64 - -package pagetables diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_arm64.go b/pkg/sentry/platform/ring0/pagetables/pagetables_arm64.go deleted file mode 100644 index fef7a0fd1..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_arm64.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagetables - -// Address constraints. -// -// The lowerTop and upperBottom currently apply to four-level pagetables; -// additional refactoring would be necessary to support five-level pagetables. -const ( - lowerTop = 0x0000ffffffffffff - upperBottom = 0xffff000000000000 - pteShift = 12 - pmdShift = 21 - pudShift = 30 - pgdShift = 39 - - pteMask = 0x1ff << pteShift - pmdMask = 0x1ff << pmdShift - pudMask = 0x1ff << pudShift - pgdMask = 0x1ff << pgdShift - - pteSize = 1 << pteShift - pmdSize = 1 << pmdShift - pudSize = 1 << pudShift - pgdSize = 1 << pgdShift - - ttbrASIDOffset = 48 - ttbrASIDMask = 0xff - - entriesPerPage = 512 -) - -// InitArch does some additional initialization related to the architecture. -// -// +checkescape:hard,stack -//go:nosplit -func (p *PageTables) InitArch(allocator Allocator) { - if p.upperSharedPageTables != nil { - p.cloneUpperShared() - } else { - p.archPageTables.root = p.Allocator.NewPTEs() - p.archPageTables.rootPhysical = p.Allocator.PhysicalFor(p.archPageTables.root) - } -} - -// cloneUpperShared clone the upper from the upper shared page tables. -// -//go:nosplit -func (p *PageTables) cloneUpperShared() { - if p.upperStart != upperBottom { - panic("upperStart should be the same as upperBottom") - } - - p.archPageTables.root = p.upperSharedPageTables.archPageTables.root - p.archPageTables.rootPhysical = p.upperSharedPageTables.archPageTables.rootPhysical -} - -// PTEs is a collection of entries. -type PTEs [entriesPerPage]PTE diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_arm64_state_autogen.go b/pkg/sentry/platform/ring0/pagetables/pagetables_arm64_state_autogen.go deleted file mode 100644 index ae9d2b272..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_arm64_state_autogen.go +++ /dev/null @@ -1,5 +0,0 @@ -// automatically generated by stateify. - -// +build arm64 - -package pagetables diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_state_autogen.go b/pkg/sentry/platform/ring0/pagetables/pagetables_state_autogen.go deleted file mode 100644 index 4c4540603..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_state_autogen.go +++ /dev/null @@ -1,3 +0,0 @@ -// automatically generated by stateify. - -package pagetables diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_unsafe_state_autogen.go b/pkg/sentry/platform/ring0/pagetables/pagetables_unsafe_state_autogen.go deleted file mode 100644 index 4c4540603..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_unsafe_state_autogen.go +++ /dev/null @@ -1,3 +0,0 @@ -// automatically generated by stateify. - -package pagetables diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_x86.go b/pkg/sentry/platform/ring0/pagetables/pagetables_x86.go deleted file mode 100644 index 32edd2f0a..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_x86.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build 386 amd64 - -package pagetables - -import ( - "sync/atomic" - - "gvisor.dev/gvisor/pkg/usermem" -) - -// archPageTables is architecture-specific data. -type archPageTables struct { - // pcid is the value assigned by PCIDs.Assign. - // - // Note that zero is a valid PCID. - pcid uint16 -} - -// CR3 returns the CR3 value for these tables. -// -// This may be called in interrupt contexts. A PCID of zero always implies a -// flush and should be passed when PCIDs are not enabled. See pcids_x86.go for -// more information. -// -//go:nosplit -func (p *PageTables) CR3(noFlush bool, pcid uint16) uint64 { - // Bit 63 is set to avoid flushing the PCID (per SDM 4.10.4.1). - const noFlushBit uint64 = 0x8000000000000000 - if noFlush && pcid != 0 { - return noFlushBit | uint64(p.rootPhysical) | uint64(pcid) - } - return uint64(p.rootPhysical) | uint64(pcid) -} - -// Bits in page table entries. -const ( - present = 0x001 - writable = 0x002 - user = 0x004 - writeThrough = 0x008 - cacheDisable = 0x010 - accessed = 0x020 - dirty = 0x040 - super = 0x080 - global = 0x100 - optionMask = executeDisable | 0xfff -) - -// MapOpts are x86 options. -type MapOpts struct { - // AccessType defines permissions. - AccessType usermem.AccessType - - // Global indicates the page is globally accessible. - Global bool - - // User indicates the page is a user page. - User bool -} - -// PTE is a page table entry. -type PTE uintptr - -// Clear clears this PTE, including super page information. -// -//go:nosplit -func (p *PTE) Clear() { - atomic.StoreUintptr((*uintptr)(p), 0) -} - -// Valid returns true iff this entry is valid. -// -//go:nosplit -func (p *PTE) Valid() bool { - return atomic.LoadUintptr((*uintptr)(p))&present != 0 -} - -// Opts returns the PTE options. -// -// These are all options except Valid and Super. -// -//go:nosplit -func (p *PTE) Opts() MapOpts { - v := atomic.LoadUintptr((*uintptr)(p)) - return MapOpts{ - AccessType: usermem.AccessType{ - Read: v&present != 0, - Write: v&writable != 0, - Execute: v&executeDisable == 0, - }, - Global: v&global != 0, - User: v&user != 0, - } -} - -// SetSuper sets this page as a super page. -// -// The page must not be valid or a panic will result. -// -//go:nosplit -func (p *PTE) SetSuper() { - if p.Valid() { - // This is not allowed. - panic("SetSuper called on valid page!") - } - atomic.StoreUintptr((*uintptr)(p), super) -} - -// IsSuper returns true iff this page is a super page. -// -//go:nosplit -func (p *PTE) IsSuper() bool { - return atomic.LoadUintptr((*uintptr)(p))&super != 0 -} - -// Set sets this PTE value. -// -// This does not change the super page property. -// -//go:nosplit -func (p *PTE) Set(addr uintptr, opts MapOpts) { - if !opts.AccessType.Any() { - p.Clear() - return - } - v := (addr &^ optionMask) - if opts.AccessType.Any() { - v |= present | accessed - } - if opts.User { - v |= user - } - if opts.Global { - v |= global - } - if !opts.AccessType.Execute { - v |= executeDisable - } - if opts.AccessType.Write { - v |= writable | dirty - } - if p.IsSuper() { - // Note that this is inherited from the previous instance. Set - // does not change the value of Super. See above. - v |= super - } - atomic.StoreUintptr((*uintptr)(p), v) -} - -// setPageTable sets this PTE value and forces the write bit and super bit to -// be cleared. This is used explicitly for breaking super pages. -// -//go:nosplit -func (p *PTE) setPageTable(pt *PageTables, ptes *PTEs) { - addr := pt.Allocator.PhysicalFor(ptes) - if addr&^optionMask != addr { - // This should never happen. - panic("unaligned physical address!") - } - v := addr | present | user | writable | accessed | dirty - atomic.StoreUintptr((*uintptr)(p), v) -} - -// Address extracts the address. This should only be used if Valid returns true. -// -//go:nosplit -func (p *PTE) Address() uintptr { - return atomic.LoadUintptr((*uintptr)(p)) &^ optionMask -} diff --git a/pkg/sentry/platform/ring0/pagetables/pagetables_x86_state_autogen.go b/pkg/sentry/platform/ring0/pagetables/pagetables_x86_state_autogen.go deleted file mode 100644 index 6fe78c51c..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pagetables_x86_state_autogen.go +++ /dev/null @@ -1,6 +0,0 @@ -// automatically generated by stateify. - -// +build 386 amd64 -// +build i386 amd64 - -package pagetables diff --git a/pkg/sentry/platform/ring0/pagetables/pcids.go b/pkg/sentry/platform/ring0/pagetables/pcids.go deleted file mode 100644 index 964496aac..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pcids.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagetables - -import ( - "gvisor.dev/gvisor/pkg/sync" -) - -// PCIDs is a simple PCID database. -// -// This is not protected by locks and is thus suitable for use only with a -// single CPU at a time. -type PCIDs struct { - // mu protects below. - mu sync.Mutex - - // cache are the assigned page tables. - cache map[*PageTables]uint16 - - // avail are available PCIDs. - avail []uint16 -} - -// NewPCIDs returns a new PCID database. -// -// start is the first index to assign. Typically this will be one, as the zero -// pcid will always be flushed on transition (see pagetables_x86.go). This may -// be more than one if specific PCIDs are reserved. -// -// Nil is returned iff the start and size are out of range. -func NewPCIDs(start, size uint16) *PCIDs { - if start+uint16(size) > limitPCID { - return nil // See comment. - } - p := &PCIDs{ - cache: make(map[*PageTables]uint16), - } - for pcid := start; pcid < start+size; pcid++ { - p.avail = append(p.avail, pcid) - } - return p -} - -// Assign assigns a PCID to the given PageTables. -// -// This may overwrite any previous assignment provided. If this in the case, -// true is returned to indicate that the PCID should be flushed. -func (p *PCIDs) Assign(pt *PageTables) (uint16, bool) { - p.mu.Lock() - if pcid, ok := p.cache[pt]; ok { - p.mu.Unlock() - return pcid, false // No flush. - } - - // Is there something available? - if len(p.avail) > 0 { - pcid := p.avail[len(p.avail)-1] - p.avail = p.avail[:len(p.avail)-1] - p.cache[pt] = pcid - - // We need to flush because while this is in the available - // pool, it may have been used previously. - p.mu.Unlock() - return pcid, true - } - - // Evict an existing table. - for old, pcid := range p.cache { - delete(p.cache, old) - p.cache[pt] = pcid - - // A flush is definitely required in this case, these page - // tables may still be active. (They will just be assigned some - // other PCID if and when they hit the given CPU again.) - p.mu.Unlock() - return pcid, true - } - - // No PCID. - p.mu.Unlock() - return 0, false -} - -// Drop drops references to a set of page tables. -func (p *PCIDs) Drop(pt *PageTables) { - p.mu.Lock() - if pcid, ok := p.cache[pt]; ok { - delete(p.cache, pt) - p.avail = append(p.avail, pcid) - } - p.mu.Unlock() -} diff --git a/pkg/sentry/platform/ring0/pagetables/pcids_aarch64.go b/pkg/sentry/platform/ring0/pagetables/pcids_aarch64.go deleted file mode 100644 index fbfd41d83..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pcids_aarch64.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -package pagetables - -// limitPCID is the maximum value of PCIDs. -// -// In VMSAv8-64, the PCID(ASID) size is an IMPLEMENTATION DEFINED choice -// of 8 bits or 16 bits, and ID_AA64MMFR0_EL1.ASIDBits identifies the -// supported size. When an implementation supports a 16-bit ASID, TCR_ELx.AS -// selects whether the top 8 bits of the ASID are used. -var limitPCID uint16 - -// GetASIDBits return the system ASID bits, 8 or 16 bits. -func GetASIDBits() uint8 - -func init() { - limitPCID = uint16(1)<<GetASIDBits() - 1 -} diff --git a/pkg/sentry/platform/ring0/pagetables/pcids_aarch64.s b/pkg/sentry/platform/ring0/pagetables/pcids_aarch64.s deleted file mode 100644 index e9d62d768..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pcids_aarch64.s +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -#include "funcdata.h" -#include "textflag.h" - -#define ID_AA64MMFR0_ASIDBITS_SHIFT 4 -#define ID_AA64MMFR0_ASIDBITS_16 2 -#define TCR_EL1_AS_BIT 36 - -// GetASIDBits return the system ASID bits, 8 or 16 bits. -// -// func GetASIDBits() uint8 -TEXT ·GetASIDBits(SB),NOSPLIT,$0-1 - // First, check whether 16bits ASID is supported. - // ID_AA64MMFR0_EL1.ASIDBITS[7:4] == 0010. - WORD $0xd5380700 // MRS ID_AA64MMFR0_EL1, R0 - UBFX $ID_AA64MMFR0_ASIDBITS_SHIFT, R0, $4, R0 - CMPW $ID_AA64MMFR0_ASIDBITS_16, R0 - BNE bits_8 - - // Second, check whether 16bits ASID is enabled. - // TCR_EL1.AS[36] == 1. - WORD $0xd5382040 // MRS TCR_EL1, R0 - TBZ $TCR_EL1_AS_BIT, R0, bits_8 - MOVD $16, R0 - B done -bits_8: - MOVD $8, R0 -done: - MOVB R0, ret+0(FP) - RET diff --git a/pkg/sentry/platform/ring0/pagetables/pcids_x86.go b/pkg/sentry/platform/ring0/pagetables/pcids_x86.go deleted file mode 100644 index 91fc5e8dd..000000000 --- a/pkg/sentry/platform/ring0/pagetables/pcids_x86.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build i386 amd64 - -package pagetables - -// limitPCID is the maximum value of valid PCIDs. -const limitPCID = 4095 diff --git a/pkg/sentry/platform/ring0/pagetables/walker_amd64.go b/pkg/sentry/platform/ring0/pagetables/walker_amd64.go deleted file mode 100644 index eb4fbcc31..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_amd64.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *Walker) iterateRangeCanonical(start, end uintptr) bool { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - // Skip over this entry. - start = next(start, pgdSize) - continue - } - - // Allocate a new pgd. - pudEntries = w.pageTables.Allocator.NewPTEs() // escapes: depends on allocator. - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) // escapes: see above. - } - - // Map the next level. - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - // Skip over this entry. - clearPUDEntries++ - start = next(start, pudSize) - continue - } - - // This level has 1-GB super pages. Is this - // entire region at least as large as a single - // PUD entry? If so, we can skip allocating a - // new page for the pmd. - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = next(start, pudSize) - continue - } - } - - // Allocate a new pud. - pmdEntries = w.pageTables.Allocator.NewPTEs() // escapes: see above. - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - // Does this page need to be split? - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < next(start, pudSize)) { - // Install the relevant entries. - pmdEntries = w.pageTables.Allocator.NewPTEs() // escapes: see above. - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - // A super page to be checked directly. - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - - // Might have been cleared. - if !pudEntry.Valid() { - clearPUDEntries++ - } - - // Note that the super page was changed. - start = next(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) // escapes: see above. - } - - // Map the next level, since this is valid. - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - // Skip over this entry. - clearPMDEntries++ - start = next(start, pmdSize) - continue - } - - // This level has 2-MB huge pages. If this - // region is contined in a single PMD entry? - // As above, we can skip allocating a new page. - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = next(start, pmdSize) - continue - } - } - - // Allocate a new pmd. - pteEntries = w.pageTables.Allocator.NewPTEs() // escapes: see above. - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - // Does this page need to be split? - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < next(start, pmdSize)) { - // Install the relevant entries. - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - // A huge page to be checked directly. - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - - // Might have been cleared. - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - // Note that the huge page was changed. - start = next(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) // escapes: see above. - } - - // Map the next level, since this is valid. - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - // At this point, we are guaranteed that start%pteSize == 0. - if !w.visitor.visit(uintptr(start&^(pteSize-1)), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - } - - // Note that the pte was changed. - start += pteSize - continue - } - - // Check if we no longer need this page. - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) // escapes: see above. - clearPMDEntries++ - } - } - - // Check if we no longer need this page. - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) // escapes: see above. - clearPUDEntries++ - } - } - - // Check if we no longer need this page. - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) // escapes: see above. - } - } - return true -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_arm64.go b/pkg/sentry/platform/ring0/pagetables/walker_arm64.go deleted file mode 100644 index 5ed881c7a..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_arm64.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *Walker) iterateRangeCanonical(start, end uintptr) bool { - pgdEntryIndex := w.pageTables.root - if start >= upperBottom { - pgdEntryIndex = w.pageTables.archPageTables.root - } - - for pgdIndex := (uint16((start & pgdMask) >> pgdShift)); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &pgdEntryIndex[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - // Skip over this entry. - start = next(start, pgdSize) - continue - } - - // Allocate a new pgd. - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - // Map the next level. - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - // Skip over this entry. - clearPUDEntries++ - start = next(start, pudSize) - continue - } - - // This level has 1-GB sect pages. Is this - // entire region at least as large as a single - // PUD entry? If so, we can skip allocating a - // new page for the pmd. - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSect() - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = next(start, pudSize) - continue - } - } - - // Allocate a new pud. - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSect() { - // Does this page need to be split? - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < next(start, pudSize)) { - // Install the relevant entries. - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSect() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - // A sect page to be checked directly. - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - - // Might have been cleared. - if !pudEntry.Valid() { - clearPUDEntries++ - } - - // Note that the sect page was changed. - start = next(start, pudSize) - continue - } - - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - // Map the next level, since this is valid. - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - // Skip over this entry. - clearPMDEntries++ - start = next(start, pmdSize) - continue - } - - // This level has 2-MB huge pages. If this - // region is contined in a single PMD entry? - // As above, we can skip allocating a new page. - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSect() - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = next(start, pmdSize) - continue - } - } - - // Allocate a new pmd. - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSect() { - // Does this page need to be split? - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < next(start, pmdSize)) { - // Install the relevant entries. - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - // A huge page to be checked directly. - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - - // Might have been cleared. - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - // Note that the huge page was changed. - start = next(start, pmdSize) - continue - } - - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - // Map the next level, since this is valid. - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - // At this point, we are guaranteed that start%pteSize == 0. - if !w.visitor.visit(uintptr(start), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - // Note that the pte was changed. - start += pteSize - continue - } - - // Check if we no longer need this page. - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - // Check if we no longer need this page. - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - // Check if we no longer need this page. - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_empty_amd64.go b/pkg/sentry/platform/ring0/pagetables/walker_empty_amd64.go deleted file mode 100644 index a3cd7a1a2..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_empty_amd64.go +++ /dev/null @@ -1,265 +0,0 @@ -// +build amd64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *emptyWalker) iterateRangeCanonical(start, end uintptr) bool { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = emptynext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = emptynext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = emptynext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < emptynext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = emptynext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = emptynext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = emptynext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < emptynext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = emptynext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start&^(pteSize-1)), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type emptyWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor emptyVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *emptyWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func emptynext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_empty_arm64.go b/pkg/sentry/platform/ring0/pagetables/walker_empty_arm64.go deleted file mode 100644 index d61b44b65..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_empty_arm64.go +++ /dev/null @@ -1,275 +0,0 @@ -// +build arm64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *emptyWalker) iterateRangeCanonical(start, end uintptr) bool { - pgdEntryIndex := w.pageTables.root - if start >= upperBottom { - pgdEntryIndex = w.pageTables.archPageTables.root - } - - for pgdIndex := (uint16((start & pgdMask) >> pgdShift)); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &pgdEntryIndex[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = emptynext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = emptynext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSect() - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = emptynext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < emptynext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSect() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = emptynext(start, pudSize) - continue - } - - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = emptynext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSect() - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = emptynext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < emptynext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = emptynext(start, pmdSize) - continue - } - - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type emptyWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor emptyVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *emptyWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func emptynext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_generic.go b/pkg/sentry/platform/ring0/pagetables/walker_generic.go deleted file mode 100644 index 34fba7b84..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_generic.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagetables - -// Visitor is a generic type. -type Visitor interface { - // visit is called on each PTE. The returned boolean indicates whether - // the walk should continue. - visit(start uintptr, pte *PTE, align uintptr) bool - - // requiresAlloc indicates that new entries should be allocated within - // the walked range. - requiresAlloc() bool - - // requiresSplit indicates that entries in the given range should be - // split if they are huge or jumbo pages. - requiresSplit() bool -} - -// Walker walks page tables. -type Walker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor Visitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *Walker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func next(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_lookup_amd64.go b/pkg/sentry/platform/ring0/pagetables/walker_lookup_amd64.go deleted file mode 100644 index c92c1cb44..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_lookup_amd64.go +++ /dev/null @@ -1,265 +0,0 @@ -// +build amd64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *lookupWalker) iterateRangeCanonical(start, end uintptr) bool { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = lookupnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = lookupnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = lookupnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < lookupnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = lookupnext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = lookupnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = lookupnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < lookupnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = lookupnext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start&^(pteSize-1)), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type lookupWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor lookupVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *lookupWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func lookupnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_lookup_arm64.go b/pkg/sentry/platform/ring0/pagetables/walker_lookup_arm64.go deleted file mode 100644 index 74062a00a..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_lookup_arm64.go +++ /dev/null @@ -1,275 +0,0 @@ -// +build arm64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *lookupWalker) iterateRangeCanonical(start, end uintptr) bool { - pgdEntryIndex := w.pageTables.root - if start >= upperBottom { - pgdEntryIndex = w.pageTables.archPageTables.root - } - - for pgdIndex := (uint16((start & pgdMask) >> pgdShift)); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &pgdEntryIndex[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = lookupnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = lookupnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSect() - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = lookupnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < lookupnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSect() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = lookupnext(start, pudSize) - continue - } - - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = lookupnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSect() - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = lookupnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < lookupnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = lookupnext(start, pmdSize) - continue - } - - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type lookupWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor lookupVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *lookupWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func lookupnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_map_amd64.go b/pkg/sentry/platform/ring0/pagetables/walker_map_amd64.go deleted file mode 100644 index 1c6c1a032..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_map_amd64.go +++ /dev/null @@ -1,265 +0,0 @@ -// +build amd64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *mapWalker) iterateRangeCanonical(start, end uintptr) bool { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = mapnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = mapnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = mapnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < mapnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = mapnext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = mapnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = mapnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < mapnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = mapnext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start&^(pteSize-1)), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type mapWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor mapVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *mapWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func mapnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_map_arm64.go b/pkg/sentry/platform/ring0/pagetables/walker_map_arm64.go deleted file mode 100644 index 8223de306..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_map_arm64.go +++ /dev/null @@ -1,275 +0,0 @@ -// +build arm64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *mapWalker) iterateRangeCanonical(start, end uintptr) bool { - pgdEntryIndex := w.pageTables.root - if start >= upperBottom { - pgdEntryIndex = w.pageTables.archPageTables.root - } - - for pgdIndex := (uint16((start & pgdMask) >> pgdShift)); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &pgdEntryIndex[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = mapnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = mapnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSect() - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = mapnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < mapnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSect() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = mapnext(start, pudSize) - continue - } - - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = mapnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSect() - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = mapnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < mapnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = mapnext(start, pmdSize) - continue - } - - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type mapWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor mapVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *mapWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func mapnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_unmap_amd64.go b/pkg/sentry/platform/ring0/pagetables/walker_unmap_amd64.go deleted file mode 100644 index 82b27ab64..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_unmap_amd64.go +++ /dev/null @@ -1,265 +0,0 @@ -// +build amd64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *unmapWalker) iterateRangeCanonical(start, end uintptr) bool { - for pgdIndex := uint16((start & pgdMask) >> pgdShift); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &w.pageTables.root[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = unmapnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = unmapnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = unmapnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < unmapnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSuper() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pudSize-1)), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = unmapnext(start, pudSize) - continue - } - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = unmapnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSuper() - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = unmapnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSuper() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < unmapnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start&^(pmdSize-1)), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = unmapnext(start, pmdSize) - continue - } - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start&^(pteSize-1)), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type unmapWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor unmapVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *unmapWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func unmapnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/pagetables/walker_unmap_arm64.go b/pkg/sentry/platform/ring0/pagetables/walker_unmap_arm64.go deleted file mode 100644 index 1ecccbf27..000000000 --- a/pkg/sentry/platform/ring0/pagetables/walker_unmap_arm64.go +++ /dev/null @@ -1,275 +0,0 @@ -// +build arm64 - -package pagetables - -// iterateRangeCanonical walks a canonical range. -// -//go:nosplit -func (w *unmapWalker) iterateRangeCanonical(start, end uintptr) bool { - pgdEntryIndex := w.pageTables.root - if start >= upperBottom { - pgdEntryIndex = w.pageTables.archPageTables.root - } - - for pgdIndex := (uint16((start & pgdMask) >> pgdShift)); start < end && pgdIndex < entriesPerPage; pgdIndex++ { - var ( - pgdEntry = &pgdEntryIndex[pgdIndex] - pudEntries *PTEs - ) - if !pgdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - start = unmapnext(start, pgdSize) - continue - } - - pudEntries = w.pageTables.Allocator.NewPTEs() - pgdEntry.setPageTable(w.pageTables, pudEntries) - } else { - pudEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address()) - } - - clearPUDEntries := uint16(0) - - for pudIndex := uint16((start & pudMask) >> pudShift); start < end && pudIndex < entriesPerPage; pudIndex++ { - var ( - pudEntry = &pudEntries[pudIndex] - pmdEntries *PTEs - ) - if !pudEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPUDEntries++ - start = unmapnext(start, pudSize) - continue - } - - if start&(pudSize-1) == 0 && end-start >= pudSize { - pudEntry.SetSect() - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - if pudEntry.Valid() { - start = unmapnext(start, pudSize) - continue - } - } - - pmdEntries = w.pageTables.Allocator.NewPTEs() - pudEntry.setPageTable(w.pageTables, pmdEntries) - - } else if pudEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pudSize-1) != 0 || end < unmapnext(start, pudSize)) { - - pmdEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pmdEntries[index].SetSect() - pmdEntries[index].Set( - pudEntry.Address()+(pmdSize*uintptr(index)), - pudEntry.Opts()) - } - pudEntry.setPageTable(w.pageTables, pmdEntries) - } else { - - if !w.visitor.visit(uintptr(start), pudEntry, pudSize-1) { - return false - } - - if !pudEntry.Valid() { - clearPUDEntries++ - } - - start = unmapnext(start, pudSize) - continue - } - - } else { - pmdEntries = w.pageTables.Allocator.LookupPTEs(pudEntry.Address()) - } - - clearPMDEntries := uint16(0) - - for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ { - var ( - pmdEntry = &pmdEntries[pmdIndex] - pteEntries *PTEs - ) - if !pmdEntry.Valid() { - if !w.visitor.requiresAlloc() { - - clearPMDEntries++ - start = unmapnext(start, pmdSize) - continue - } - - if start&(pmdSize-1) == 0 && end-start >= pmdSize { - pmdEntry.SetSect() - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - if pmdEntry.Valid() { - start = unmapnext(start, pmdSize) - continue - } - } - - pteEntries = w.pageTables.Allocator.NewPTEs() - pmdEntry.setPageTable(w.pageTables, pteEntries) - - } else if pmdEntry.IsSect() { - - if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < unmapnext(start, pmdSize)) { - - pteEntries = w.pageTables.Allocator.NewPTEs() - for index := uint16(0); index < entriesPerPage; index++ { - pteEntries[index].Set( - pmdEntry.Address()+(pteSize*uintptr(index)), - pmdEntry.Opts()) - } - pmdEntry.setPageTable(w.pageTables, pteEntries) - } else { - - if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) { - return false - } - - if !pmdEntry.Valid() { - clearPMDEntries++ - } - - start = unmapnext(start, pmdSize) - continue - } - - } else { - pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address()) - } - - clearPTEEntries := uint16(0) - - for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ { - var ( - pteEntry = &pteEntries[pteIndex] - ) - if !pteEntry.Valid() && !w.visitor.requiresAlloc() { - clearPTEEntries++ - start += pteSize - continue - } - - if !w.visitor.visit(uintptr(start), pteEntry, pteSize-1) { - return false - } - if !pteEntry.Valid() { - if w.visitor.requiresAlloc() { - panic("PTE not set after iteration with requiresAlloc!") - } - clearPTEEntries++ - } - - start += pteSize - continue - } - - if clearPTEEntries == entriesPerPage { - pmdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pteEntries) - clearPMDEntries++ - } - } - - if clearPMDEntries == entriesPerPage { - pudEntry.Clear() - w.pageTables.Allocator.FreePTEs(pmdEntries) - clearPUDEntries++ - } - } - - if clearPUDEntries == entriesPerPage { - pgdEntry.Clear() - w.pageTables.Allocator.FreePTEs(pudEntries) - } - } - return true -} - -// Walker walks page tables. -type unmapWalker struct { - // pageTables are the tables to walk. - pageTables *PageTables - - // Visitor is the set of arguments. - visitor unmapVisitor -} - -// iterateRange iterates over all appropriate levels of page tables for the given range. -// -// If requiresAlloc is true, then Set _must_ be called on all given PTEs. The -// exception is super pages. If a valid super page (huge or jumbo) cannot be -// installed, then the walk will continue to individual entries. -// -// This algorithm will attempt to maximize the use of super/sect pages whenever -// possible. Whether a super page is provided will be clear through the range -// provided in the callback. -// -// Note that if requiresAlloc is true, then no gaps will be present. However, -// if alloc is not set, then the iteration will likely be full of gaps. -// -// Note that this function should generally be avoided in favor of Map, Unmap, -// etc. when not necessary. -// -// Precondition: start must be page-aligned. -// Precondition: start must be less than end. -// Precondition: If requiresAlloc is true, then start and end should not span -// non-canonical ranges. If they do, a panic will result. -// -//go:nosplit -func (w *unmapWalker) iterateRange(start, end uintptr) { - if start%pteSize != 0 { - panic("unaligned start") - } - if end < start { - panic("start > end") - } - if start < lowerTop { - if end <= lowerTop { - w.iterateRangeCanonical(start, end) - } else if end > lowerTop && end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(start, lowerTop) - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - if !w.iterateRangeCanonical(start, lowerTop) { - return - } - w.iterateRangeCanonical(upperBottom, end) - } - } else if start < upperBottom { - if end <= upperBottom { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - } else { - if w.visitor.requiresAlloc() { - panic("alloc spans non-canonical range") - } - w.iterateRangeCanonical(upperBottom, end) - } - } else { - w.iterateRangeCanonical(start, end) - } -} - -// next returns the next address quantized by the given size. -// -//go:nosplit -func unmapnext(start uintptr, size uintptr) uintptr { - start &= ^(size - 1) - start += size - return start -} diff --git a/pkg/sentry/platform/ring0/ring0.go b/pkg/sentry/platform/ring0/ring0.go deleted file mode 100644 index cdeb1b43a..000000000 --- a/pkg/sentry/platform/ring0/ring0.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ring0 provides basic operating system-level stubs. -package ring0 diff --git a/pkg/sentry/platform/ring0/ring0_amd64_state_autogen.go b/pkg/sentry/platform/ring0/ring0_amd64_state_autogen.go deleted file mode 100644 index 96cf5d331..000000000 --- a/pkg/sentry/platform/ring0/ring0_amd64_state_autogen.go +++ /dev/null @@ -1,7 +0,0 @@ -// automatically generated by stateify. - -// +build amd64 -// +build amd64 -// +build amd64 - -package ring0 diff --git a/pkg/sentry/platform/ring0/ring0_arm64_state_autogen.go b/pkg/sentry/platform/ring0/ring0_arm64_state_autogen.go deleted file mode 100644 index 7f2ab3537..000000000 --- a/pkg/sentry/platform/ring0/ring0_arm64_state_autogen.go +++ /dev/null @@ -1,7 +0,0 @@ -// automatically generated by stateify. - -// +build arm64 -// +build arm64 -// +build arm64 - -package ring0 diff --git a/pkg/sentry/platform/ring0/ring0_impl_amd64_state_autogen.go b/pkg/sentry/platform/ring0/ring0_impl_amd64_state_autogen.go deleted file mode 100644 index 770a13dfa..000000000 --- a/pkg/sentry/platform/ring0/ring0_impl_amd64_state_autogen.go +++ /dev/null @@ -1,7 +0,0 @@ -// automatically generated by stateify. - -// +build amd64 -// +build amd64 -// +build 386 amd64 - -package ring0 diff --git a/pkg/sentry/platform/ring0/ring0_impl_arm64_state_autogen.go b/pkg/sentry/platform/ring0/ring0_impl_arm64_state_autogen.go deleted file mode 100644 index 7f2ab3537..000000000 --- a/pkg/sentry/platform/ring0/ring0_impl_arm64_state_autogen.go +++ /dev/null @@ -1,7 +0,0 @@ -// automatically generated by stateify. - -// +build arm64 -// +build arm64 -// +build arm64 - -package ring0 diff --git a/pkg/sentry/platform/ring0/ring0_state_autogen.go b/pkg/sentry/platform/ring0/ring0_state_autogen.go deleted file mode 100644 index 327aba163..000000000 --- a/pkg/sentry/platform/ring0/ring0_state_autogen.go +++ /dev/null @@ -1,3 +0,0 @@ -// automatically generated by stateify. - -package ring0 diff --git a/pkg/sentry/platform/ring0/ring0_unsafe_state_autogen.go b/pkg/sentry/platform/ring0/ring0_unsafe_state_autogen.go deleted file mode 100644 index 327aba163..000000000 --- a/pkg/sentry/platform/ring0/ring0_unsafe_state_autogen.go +++ /dev/null @@ -1,3 +0,0 @@ -// automatically generated by stateify. - -package ring0 |