Merge pull request #4933 from lubinszARM:pr_kvm_el0_exceptions

PiperOrigin-RevId: 350862699

4 files changed, 57 insertions, 56 deletions

diff --git a/pkg/sentry/platform/kvm/machine_arm64_unsafe.go b/pkg/sentry/platform/kvm/machine_arm64_unsafe.go
index 67f7f88f2..a466acf4d 100644
--- a/pkg/sentry/platform/kvm/machine_arm64_unsafe.go
+++ b/pkg/sentry/platform/kvm/machine_arm64_unsafe.go
@@ -235,12 +235,12 @@ func (c *vCPU) SwitchToUser(switchOpts ring0.SwitchOpts, info *arch.SignalInfo)
 	ttbr0App := switchOpts.PageTables.TTBR0_EL1(false, 0)
 	c.SetTtbr0App(uintptr(ttbr0App))
 
-	// TODO(gvisor.dev/issue/1238): full context-switch supporting for Arm64.
+	// Full context-switch supporting for Arm64.
 	// The Arm64 user-mode execution state consists of:
 	// x0-x30
 	// PC, SP, PSTATE
 	// V0-V31: 32 128-bit registers for floating point, and simd
-	// FPSR
+	// FPSR, FPCR
 	// TPIDR_EL0, used for TLS
 	appRegs := switchOpts.Registers
 	c.SetAppAddr(ring0.KernelStartAddress | uintptr(unsafe.Pointer(appRegs)))
@@ -254,15 +254,30 @@ func (c *vCPU) SwitchToUser(switchOpts ring0.SwitchOpts, info *arch.SignalInfo)
 	case ring0.Syscall:
 		// Fast path: system call executed.
 		return usermem.NoAccess, nil
-
 	case ring0.PageFault:
 		return c.fault(int32(syscall.SIGSEGV), info)
 	case ring0.El0ErrNMI:
 		return c.fault(int32(syscall.SIGBUS), info)
-	case ring0.Vector(bounce): // ring0.VirtualizationException
+	case ring0.Vector(bounce): // ring0.VirtualizationException.
 		return usermem.NoAccess, platform.ErrContextInterrupt
 	case ring0.El0SyncUndef:
 		return c.fault(int32(syscall.SIGILL), info)
+	case ring0.El0SyncDbg:
+		*info = arch.SignalInfo{
+			Signo: int32(syscall.SIGTRAP),
+			Code:  1, // TRAP_BRKPT (breakpoint).
+		}
+		info.SetAddr(switchOpts.Registers.Pc) // Include address.
+		return usermem.AccessType{}, platform.ErrContextSignal
+	case ring0.El0SyncSpPc:
+		*info = arch.SignalInfo{
+			Signo: int32(syscall.SIGBUS),
+			Code:  2, // BUS_ADRERR (physical address does not exist).
+		}
+		return usermem.NoAccess, platform.ErrContextSignal
+	case ring0.El0SyncSys,
+		ring0.El0SyncWfx:
+		return usermem.NoAccess, nil // skip for now.
 	default:
 		panic(fmt.Sprintf("unexpected vector: 0x%x", vector))
 	}
diff --git a/pkg/sentry/platform/ring0/aarch64.go b/pkg/sentry/platform/ring0/aarch64.go
index c51df2811..3bda594f9 100644
--- a/pkg/sentry/platform/ring0/aarch64.go
+++ b/pkg/sentry/platform/ring0/aarch64.go
@@ -95,6 +95,7 @@ const (
 	El0SyncSpPc
 	El0SyncUndef
 	El0SyncDbg
+	El0SyncWfx
 	El0SyncInv
 
 	El0ErrNMI
diff --git a/pkg/sentry/platform/ring0/entry_arm64.s b/pkg/sentry/platform/ring0/entry_arm64.s
index 7529b3362..b2bb18257 100644
--- a/pkg/sentry/platform/ring0/entry_arm64.s
+++ b/pkg/sentry/platform/ring0/entry_arm64.s
@@ -576,26 +576,26 @@ TEXT ·El1_error_invalid(SB),NOSPLIT,$0
 // El1_sync is the handler for El1_sync.
 TEXT ·El1_sync(SB),NOSPLIT,$0
 	KERNEL_ENTRY_FROM_EL1
-	WORD $0xd5385219        // MRS ESR_EL1, R25
-	LSR  $ESR_ELx_EC_SHIFT, R25, R24
+	MRS ESR_EL1, R25                  // read the syndrome register
+	LSR  $ESR_ELx_EC_SHIFT, R25, R24  // exception class
 	CMP $ESR_ELx_EC_DABT_CUR, R24
-	BEQ el1_da
+	BEQ el1_da                        // data abort in EL1
 	CMP $ESR_ELx_EC_IABT_CUR, R24
-	BEQ el1_ia
-	CMP $ESR_ELx_EC_SYS64, R24
-	BEQ el1_undef
+	BEQ el1_ia                        // instruction abort in EL1
 	CMP $ESR_ELx_EC_SP_ALIGN, R24
-	BEQ el1_sp_pc
+	BEQ el1_sp_pc                     // stack alignment exception
 	CMP $ESR_ELx_EC_PC_ALIGN, R24
-	BEQ el1_sp_pc
+	BEQ el1_sp_pc                     // pc alignment exception
 	CMP $ESR_ELx_EC_UNKNOWN, R24
-	BEQ el1_undef
+	BEQ el1_undef                     // unknown exception in EL1
 	CMP $ESR_ELx_EC_SVC64, R24
-	BEQ el1_svc
+	BEQ el1_svc                       // SVC in 64-bit state
 	CMP $ESR_ELx_EC_BREAKPT_CUR, R24
-	BGE el1_dbg
+	BEQ el1_dbg                       // debug exception in EL1
 	CMP $ESR_ELx_EC_FP_ASIMD, R24
-	BEQ el1_fpsimd_acc
+	BEQ el1_fpsimd_acc                // FP/ASIMD access
+	CMP $ESR_ELx_EC_SVE, R24
+	BEQ el1_sve_acc                   // SVE access
 	B el1_invalid
 
 el1_da:
@@ -611,6 +611,7 @@ el1_svc:
 el1_dbg:
 	EXCEPTION_EL1(El1SyncDbg)
 el1_fpsimd_acc:
+el1_sve_acc:
 	VFP_ENABLE
 	B ·kernelExitToEl1(SB)  // Resume.
 el1_invalid:
@@ -631,28 +632,32 @@ TEXT ·El1_error(SB),NOSPLIT,$0
 // El0_sync is the handler for El0_sync.
 TEXT ·El0_sync(SB),NOSPLIT,$0
 	KERNEL_ENTRY_FROM_EL0
-	WORD $0xd5385219	// MRS ESR_EL1, R25
-	LSR  $ESR_ELx_EC_SHIFT, R25, R24
+	MRS ESR_EL1, R25                  // read the syndrome register
+	LSR  $ESR_ELx_EC_SHIFT, R25, R24  // exception class
 	CMP $ESR_ELx_EC_SVC64, R24
-	BEQ el0_svc
+	BEQ el0_svc                       // SVC in 64-bit state
 	CMP $ESR_ELx_EC_DABT_LOW, R24
-	BEQ el0_da
+	BEQ el0_da                        // data abort in EL0
 	CMP $ESR_ELx_EC_IABT_LOW, R24
-	BEQ el0_ia
+	BEQ el0_ia                        // instruction abort in EL0
 	CMP $ESR_ELx_EC_FP_ASIMD, R24
-	BEQ el0_fpsimd_acc
+	BEQ el0_fpsimd_acc                // FP/ASIMD access
 	CMP $ESR_ELx_EC_SVE, R24
-	BEQ el0_sve_acc
+	BEQ el0_sve_acc                   // SVE access
 	CMP $ESR_ELx_EC_FP_EXC64, R24
-	BEQ el0_fpsimd_exc
+	BEQ el0_fpsimd_exc                // FP/ASIMD exception
 	CMP $ESR_ELx_EC_SP_ALIGN, R24
-	BEQ el0_sp_pc
+	BEQ el0_sp_pc                     // stack alignment exception
 	CMP $ESR_ELx_EC_PC_ALIGN, R24
-	BEQ el0_sp_pc
+	BEQ el0_sp_pc                     // pc alignment exception
 	CMP $ESR_ELx_EC_UNKNOWN, R24
-	BEQ el0_undef
+	BEQ el0_undef                     // unknown exception in EL0
 	CMP $ESR_ELx_EC_BREAKPT_LOW, R24
-	BGE el0_dbg
+	BEQ el0_dbg                       // debug exception in EL0
+	CMP $ESR_ELx_EC_SYS64, R24
+	BEQ el0_sys                       // configurable trap
+	CMP $ESR_ELx_EC_WFx, R24
+	BEQ el0_wfx                       // WFX trap
 	B   el0_invalid
 
 el0_svc:
@@ -683,6 +688,10 @@ el0_undef:
 	EXCEPTION_EL0(El0SyncUndef)
 el0_dbg:
 	EXCEPTION_EL0(El0SyncDbg)
+el0_sys:
+	EXCEPTION_EL0(El0SyncSys)
+el0_wfx:
+	EXCEPTION_EL0(El0SyncWfx)
 el0_invalid:
 	EXCEPTION_EL0(El0SyncInv)
 
@@ -699,36 +708,11 @@ TEXT ·El0_error(SB),NOSPLIT,$0
 	CMP $ESR_ELx_SERR_NMI, R24
 	BEQ el0_nmi
 	B el0_bounce
-el0_nmi:
-        WORD $0xd538d092     //MRS   TPIDR_EL1, R18
-        WORD $0xd538601a     //MRS   FAR_EL1, R26
-
-        MOVD R26, CPU_FAULT_ADDR(RSV_REG)
-
-        MOVD $1, R3
-        MOVD R3, CPU_ERROR_TYPE(RSV_REG) // Set error type to user.
-
-        MOVD $El0ErrNMI, R3
-        MOVD R3, CPU_VECTOR_CODE(RSV_REG)
-
-        MRS ESR_EL1, R3
-        MOVD R3, CPU_ERROR_CODE(RSV_REG)
-
-        B ·kernelExitToEl1(SB)
 
+el0_nmi:
+	EXCEPTION_EL0(El0ErrNMI)
 el0_bounce:
-	WORD $0xd538d092     //MRS   TPIDR_EL1, R18
-	WORD $0xd538601a     //MRS   FAR_EL1, R26
-
-	MOVD R26, CPU_FAULT_ADDR(RSV_REG)
-
-	MOVD $1, R3
-	MOVD R3, CPU_ERROR_TYPE(RSV_REG) // Set error type to user.
-
-	MOVD $VirtualizationException, R3
-	MOVD R3, CPU_VECTOR_CODE(RSV_REG)
-
-	B ·kernelExitToEl1(SB)
+	EXCEPTION_EL0(VirtualizationException)
 
 TEXT ·El0_sync_invalid(SB),NOSPLIT,$0
 	B ·Shutdown(SB)
diff --git a/pkg/sentry/platform/ring0/offsets_arm64.go b/pkg/sentry/platform/ring0/offsets_arm64.go
index b5652deb9..164db6d5a 100644
--- a/pkg/sentry/platform/ring0/offsets_arm64.go
+++ b/pkg/sentry/platform/ring0/offsets_arm64.go
@@ -75,6 +75,7 @@ func Emit(w io.Writer) {
 	fmt.Fprintf(w, "#define El0SyncSpPc 0x%02x\n", El0SyncSpPc)
 	fmt.Fprintf(w, "#define El0SyncUndef 0x%02x\n", El0SyncUndef)
 	fmt.Fprintf(w, "#define El0SyncDbg 0x%02x\n", El0SyncDbg)
+	fmt.Fprintf(w, "#define El0SyncWfx 0x%02x\n", El0SyncWfx)
 	fmt.Fprintf(w, "#define El0SyncInv 0x%02x\n", El0SyncInv)
 
 	fmt.Fprintf(w, "#define El0ErrNMI 0x%02x\n", El0ErrNMI)