Add safecopy support for arm64 platform.

Signed-off-by: Haibo Xu <haibo.xu@arm.com>
Change-Id: I565214581eeb44045169da7f44d45a489082ac3a
PiperOrigin-RevId: 224938170

5 files changed, 386 insertions, 0 deletions

diff --git a/pkg/sentry/platform/safecopy/BUILD b/pkg/sentry/platform/safecopy/BUILD
index ee58a805e..cb8347dd8 100644
--- a/pkg/sentry/platform/safecopy/BUILD
+++ b/pkg/sentry/platform/safecopy/BUILD
@@ -6,11 +6,15 @@ go_library(
     name = "safecopy",
     srcs = [
         "atomic_amd64.s",
+        "atomic_arm64.s",
         "memclr_amd64.s",
+        "memclr_arm64.s",
         "memcpy_amd64.s",
+        "memcpy_arm64.s",
         "safecopy.go",
         "safecopy_unsafe.go",
         "sighandler_amd64.s",
+        "sighandler_arm64.s",
     ],
     importpath = "gvisor.googlesource.com/gvisor/pkg/sentry/platform/safecopy",
     visibility = ["//pkg/sentry:internal"],
diff --git a/pkg/sentry/platform/safecopy/atomic_arm64.s b/pkg/sentry/platform/safecopy/atomic_arm64.s
new file mode 100644
index 000000000..554a5c1e1
--- /dev/null
+++ b/pkg/sentry/platform/safecopy/atomic_arm64.s
@@ -0,0 +1,98 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+// handleSwapUint32Fault returns the value stored in R1. Control is transferred
+// to it when swapUint32 below receives SIGSEGV or SIGBUS, with the signal
+// number stored in R1.
+//
+// It must have the same frame configuration as swapUint32 so that it can undo
+// any potential call frame set up by the assembler.
+TEXT handleSwapUint32Fault(SB), NOSPLIT, $0-24
+	MOVW R1, sig+20(FP)
+	RET
+
+// See the corresponding doc in safecopy_unsafe.go
+//
+// The code is derived from Go source runtime/internal/atomic.Xchg.
+//
+//func swapUint32(ptr unsafe.Pointer, new uint32) (old uint32, sig int32)
+TEXT ·swapUint32(SB), NOSPLIT, $0-24
+	// Store 0 as the returned signal number. If we run to completion,
+	// this is the value the caller will see; if a signal is received,
+	// handleSwapUint32Fault will store a different value in this address.
+	MOVW $0, sig+20(FP)
+again:
+	MOVD addr+0(FP), R0
+	MOVW new+8(FP), R1
+	LDAXRW (R0), R2
+	STLXRW R1, (R0), R3
+	CBNZ R3, again
+	MOVW R2, old+16(FP)
+	RET
+
+// handleSwapUint64Fault returns the value stored in R1. Control is transferred
+// to it when swapUint64 below receives SIGSEGV or SIGBUS, with the signal
+// number stored in R1.
+//
+// It must have the same frame configuration as swapUint64 so that it can undo
+// any potential call frame set up by the assembler.
+TEXT handleSwapUint64Fault(SB), NOSPLIT, $0-28
+	MOVW R1, sig+24(FP)
+	RET
+
+// See the corresponding doc in safecopy_unsafe.go
+//
+// The code is derived from Go source runtime/internal/atomic.Xchg64.
+//
+//func swapUint64(ptr unsafe.Pointer, new uint64) (old uint64, sig int32)
+TEXT ·swapUint64(SB), NOSPLIT, $0-28
+	// Store 0 as the returned signal number. If we run to completion,
+	// this is the value the caller will see; if a signal is received,
+	// handleSwapUint64Fault will store a different value in this address.
+	MOVW $0, sig+24(FP)
+again:
+	MOVD addr+0(FP), R0
+	MOVD new+8(FP), R1
+	LDAXR (R0), R2
+	STLXR R1, (R0), R3
+	CBNZ R3, again
+	MOVD R2, old+16(FP)
+	RET
+
+// handleCompareAndSwapUint32Fault returns the value stored in R1. Control is
+// transferred to it when compareAndSwapUint32 below receives SIGSEGV or SIGBUS,
+// with the signal number stored in R1.
+//
+// It must have the same frame configuration as compareAndSwapUint32 so that it
+// can undo any potential call frame set up by the assembler.
+TEXT handleCompareAndSwapUint32Fault(SB), NOSPLIT, $0-24
+	MOVW R1, sig+20(FP)
+	RET
+
+// See the corresponding doc in safecopy_unsafe.go
+//
+// The code is derived from Go source runtime/internal/atomic.Cas.
+//
+//func compareAndSwapUint32(ptr unsafe.Pointer, old, new uint32) (prev uint32, sig int32)
+TEXT ·compareAndSwapUint32(SB), NOSPLIT, $0-24
+	// Store 0 as the returned signal number. If we run to completion, this is
+	// the value the caller will see; if a signal is received,
+	// handleCompareAndSwapUint32Fault will store a different value in this
+	// address.
+	MOVW $0, sig+20(FP)
+
+	MOVD addr+0(FP), R0
+	MOVW old+8(FP), R1
+	MOVW new+12(FP), R2
+again:
+	LDAXRW (R0), R3
+	CMPW R1, R3
+	BNE done
+	STLXRW R2, (R0), R4
+	CBNZ R4, again
+done:
+	MOVW R3, prev+16(FP)
+	RET
diff --git a/pkg/sentry/platform/safecopy/memclr_arm64.s b/pkg/sentry/platform/safecopy/memclr_arm64.s
new file mode 100644
index 000000000..7361b9067
--- /dev/null
+++ b/pkg/sentry/platform/safecopy/memclr_arm64.s
@@ -0,0 +1,74 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+// handleMemclrFault returns (the value stored in R0, the value stored in R1).
+// Control is transferred to it when memclr below receives SIGSEGV or SIGBUS,
+// with the faulting address stored in R0 and the signal number stored in R1.
+//
+// It must have the same frame configuration as memclr so that it can undo any
+// potential call frame set up by the assembler.
+TEXT handleMemclrFault(SB), NOSPLIT, $0-28
+	MOVD R0, addr+16(FP)
+	MOVW R1, sig+24(FP)
+	RET
+
+// See the corresponding doc in safecopy_unsafe.go
+//
+// The code is derived from runtime.memclrNoHeapPointers.
+//
+// func memclr(ptr unsafe.Pointer, n uintptr) (fault unsafe.Pointer, sig int32)
+TEXT ·memclr(SB), NOSPLIT, $0-28
+	// Store 0 as the returned signal number. If we run to completion,
+	// this is the value the caller will see; if a signal is received,
+	// handleMemclrFault will store a different value in this address.
+	MOVW $0, sig+24(FP)
+	MOVD ptr+0(FP), R0
+	MOVD n+8(FP), R1
+
+	// If size is less than 16 bytes, use tail_zero to zero what remains
+	CMP $16, R1
+	BLT tail_zero
+	// Get buffer offset into 16 byte aligned address for better performance
+	ANDS $15, R0, ZR
+	BNE unaligned_to_16
+aligned_to_16:
+	LSR $4, R1, R2
+zero_by_16:
+	STP.P (ZR, ZR), 16(R0) // Store pair with post index.
+	SUBS $1, R2, R2
+	BNE zero_by_16
+	ANDS $15, R1, R1
+	BEQ end
+
+	// Zero buffer with size=R1 < 16
+tail_zero:
+	TBZ $3, R1, tail_zero_4
+	MOVD.P ZR, 8(R0)
+tail_zero_4:
+	TBZ $2, R1, tail_zero_2
+	MOVW.P ZR, 4(R0)
+tail_zero_2:
+	TBZ $1, R1, tail_zero_1
+	MOVH.P ZR, 2(R0)
+tail_zero_1:
+	TBZ $0, R1, end
+	MOVB ZR, (R0)
+end:
+	RET
+
+unaligned_to_16:
+	MOVD R0, R2
+head_loop:
+	MOVBU.P ZR, 1(R0)
+	ANDS $15, R0, ZR
+	BNE head_loop
+	// Adjust length for what remains
+	SUB R2, R0, R3
+	SUB R3, R1
+	// If size is less than 16 bytes, use tail_zero to zero what remains
+	CMP $16, R1
+	BLT tail_zero
+	B aligned_to_16
diff --git a/pkg/sentry/platform/safecopy/memcpy_arm64.s b/pkg/sentry/platform/safecopy/memcpy_arm64.s
new file mode 100644
index 000000000..e7e541565
--- /dev/null
+++ b/pkg/sentry/platform/safecopy/memcpy_arm64.s
@@ -0,0 +1,78 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+// handleMemcpyFault returns (the value stored in R0, the value stored in R1).
+// Control is transferred to it when memcpy below receives SIGSEGV or SIGBUS,
+// with the faulting address stored in R0 and the signal number stored in R1.
+//
+// It must have the same frame configuration as memcpy so that it can undo any
+// potential call frame set up by the assembler.
+TEXT handleMemcpyFault(SB), NOSPLIT, $0-36
+	MOVD R0, addr+24(FP)
+	MOVW R1, sig+32(FP)
+	RET
+
+// memcpy copies data from src to dst. If a SIGSEGV or SIGBUS signal is received
+// during the copy, it returns the address that caused the fault and the number
+// of the signal that was received. Otherwise, it returns an unspecified address
+// and a signal number of 0.
+//
+// Data is copied in order, such that if a fault happens at address p, it is
+// safe to assume that all data before p-maxRegisterSize has already been
+// successfully copied.
+//
+// The code is derived from the Go source runtime.memmove.
+//
+// func memcpy(dst, src unsafe.Pointer, n uintptr) (fault unsafe.Pointer, sig int32)
+TEXT ·memcpy(SB), NOSPLIT, $-8-36
+	// Store 0 as the returned signal number. If we run to completion,
+	// this is the value the caller will see; if a signal is received,
+	// handleMemcpyFault will store a different value in this address.
+	MOVW $0, sig+32(FP)
+
+	MOVD to+0(FP), R3
+	MOVD from+8(FP), R4
+	MOVD n+16(FP), R5
+	CMP $0, R5
+	BNE check
+	RET
+
+check:
+	AND $~7, R5, R7     // R7 is N&~7.
+	SUB R7, R5, R6      // R6 is N&7.
+
+	// Copying forward proceeds by copying R7/8 words then copying R6 bytes.
+	// R3 and R4 are advanced as we copy.
+
+	// (There may be implementations of armv8 where copying by bytes until
+	// at least one of source or dest is word aligned is a worthwhile
+	// optimization, but the on the one tested so far (xgene) it did not
+	// make a significance difference.)
+
+	CMP $0, R7          // Do we need to do any word-by-word copying?
+	BEQ noforwardlarge
+	ADD R3, R7, R9      // R9 points just past where we copy by word.
+
+forwardlargeloop:
+	MOVD.P 8(R4), R8       // R8 is just a scratch register.
+	MOVD.P R8, 8(R3)
+	CMP R3, R9
+	BNE forwardlargeloop
+
+noforwardlarge:
+	CMP $0, R6          // Do we need to do any byte-by-byte copying?
+	BNE forwardtail
+	RET
+
+forwardtail:
+	ADD R3, R6, R9      // R9 points just past the destination memory.
+
+forwardtailloop:
+	MOVBU.P 1(R4), R8
+	MOVBU.P R8, 1(R3)
+	CMP R3, R9
+	BNE forwardtailloop
+	RET
diff --git a/pkg/sentry/platform/safecopy/sighandler_arm64.s b/pkg/sentry/platform/safecopy/sighandler_arm64.s
new file mode 100644
index 000000000..5e8e193e7
--- /dev/null
+++ b/pkg/sentry/platform/safecopy/sighandler_arm64.s
@@ -0,0 +1,132 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "textflag.h"
+
+// The signals handled by sigHandler.
+#define SIGBUS 7
+#define SIGSEGV 11
+
+// Offsets to the registers in context->uc_mcontext.gregs[].
+#define REG_R0 0xB8
+#define REG_R1 0xC0
+#define REG_PC 0x1B8
+
+// Offset to the si_addr field of siginfo.
+#define SI_CODE 0x08
+#define SI_ADDR 0x10
+
+// signalHandler is the signal handler for SIGSEGV and SIGBUS signals. It must
+// not be set up as a handler to any other signals.
+//
+// If the instruction causing the signal is within a safecopy-protected
+// function, the signal is handled such that execution resumes in the
+// appropriate fault handling stub with R0 containing the faulting address and
+// R1 containing the signal number. Otherwise control is transferred to the
+// previously configured signal handler (savedSigSegvHandler or
+// savedSigBusHandler).
+//
+// This function cannot be written in go because it runs whenever a signal is
+// received by the thread (preempting whatever was running), which includes when
+// garbage collector has stopped or isn't expecting any interactions (like
+// barriers).
+//
+// The arguments are the following:
+// R0 - The signal number.
+// R1 - Pointer to siginfo_t structure.
+// R2 - Pointer to ucontext structure.
+TEXT ·signalHandler(SB),NOSPLIT,$0
+	// Check if the signal is from the kernel, si_code > 0 means a kernel signal.
+	MOVD SI_CODE(R1), R7
+	CMPW $0x0, R7
+	BLE original_handler
+
+	// Check if PC is within the area we care about.
+	MOVD REG_PC(R2), R7
+	MOVD ·memcpyBegin(SB), R8
+	CMP R8, R7
+	BLO not_memcpy
+	MOVD ·memcpyEnd(SB), R8
+	CMP R8, R7
+	BHS not_memcpy
+
+	// Modify the context such that execution will resume in the fault handler.
+	MOVD $handleMemcpyFault(SB), R7
+	B handle_fault
+
+not_memcpy:
+	MOVD ·memclrBegin(SB), R8
+	CMP R8, R7
+	BLO not_memclr
+	MOVD ·memclrEnd(SB), R8
+	CMP R8, R7
+	BHS not_memclr
+
+	MOVD $handleMemclrFault(SB), R7
+	B handle_fault
+
+not_memclr:
+	MOVD ·swapUint32Begin(SB), R8
+	CMP R8, R7
+	BLO not_swapuint32
+	MOVD ·swapUint32End(SB), R8
+	CMP R8, R7
+	BHS not_swapuint32
+
+	MOVD $handleSwapUint32Fault(SB), R7
+	B handle_fault
+
+not_swapuint32:
+	MOVD ·swapUint64Begin(SB), R8
+	CMP R8, R7
+	BLO not_swapuint64
+	MOVD ·swapUint64End(SB), R8
+	CMP R8, R7
+	BHS not_swapuint64
+
+	MOVD $handleSwapUint64Fault(SB), R7
+	B handle_fault
+
+not_swapuint64:
+	MOVD ·compareAndSwapUint32Begin(SB), R8
+	CMP R8, R7
+	BLO not_casuint32
+	MOVD ·compareAndSwapUint32End(SB), R8
+	CMP R8, R7
+	BHS not_casuint32
+
+	MOVD $handleCompareAndSwapUint32Fault(SB), R7
+	B handle_fault
+
+not_casuint32:
+original_handler:
+	// Jump to the previous signal handler, which is likely the golang one.
+	MOVD ·savedSigBusHandler(SB), R7
+	MOVD ·savedSigSegVHandler(SB), R8
+	CMPW $SIGSEGV, R0
+	CSEL EQ, R8, R7, R7
+	B (R7)
+
+handle_fault:
+	// Entered with the address of the fault handler in R7; store it in PC.
+	MOVD R7, REG_PC(R2)
+
+	// Store the faulting address in R0.
+	MOVD SI_ADDR(R1), R7
+	MOVD R7, REG_R0(R2)
+
+	// Store the signal number in R1.
+	MOVW R0, REG_R1(R2)
+
+	RET