Implement MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ.

cf. 2a36ab717e8f "rseq/membarrier: Add MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ"

PiperOrigin-RevId: 336186795

4 files changed, 87 insertions, 29 deletions

diff --git a/pkg/abi/linux/membarrier.go b/pkg/abi/linux/membarrier.go
index 0f3c03e63..4f6021a1d 100644
--- a/pkg/abi/linux/membarrier.go
+++ b/pkg/abi/linux/membarrier.go
@@ -14,7 +14,7 @@
 
 package linux
 
-// membarrier(2) commands, from include/uapi/linux/membarrier.h
+// membarrier(2) commands, from include/uapi/linux/membarrier.h.
 const (
 	MEMBARRIER_CMD_QUERY                                = 0
 	MEMBARRIER_CMD_GLOBAL                               = (1 << 0)
@@ -24,4 +24,11 @@ const (
 	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED           = (1 << 4)
 	MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE          = (1 << 5)
 	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 6)
+	MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ               = (1 << 7)
+	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ      = (1 << 8)
+)
+
+// membarrier(2) flags, from include/uapi/linux/membarrier.h.
+const (
+	MEMBARRIER_CMD_FLAG_CPU = (1 << 0)
 )
diff --git a/pkg/sentry/mm/mm.go b/pkg/sentry/mm/mm.go
index 43567b92c..92cc87d84 100644
--- a/pkg/sentry/mm/mm.go
+++ b/pkg/sentry/mm/mm.go
@@ -243,6 +243,12 @@ type MemoryManager struct {
 	//
 	// membarrierPrivateEnabled is accessed using atomic memory operations.
 	membarrierPrivateEnabled uint32
+
+	// membarrierRSeqEnabled is non-zero if EnableMembarrierRSeq has previously
+	// been called.
+	//
+	// membarrierRSeqEnabled is accessed using atomic memory operations.
+	membarrierRSeqEnabled uint32
 }
 
 // vma represents a virtual memory area.
diff --git a/pkg/sentry/mm/syscalls.go b/pkg/sentry/mm/syscalls.go
index 0a66b1cdd..675efdc7c 100644
--- a/pkg/sentry/mm/syscalls.go
+++ b/pkg/sentry/mm/syscalls.go
@@ -1287,3 +1287,15 @@ func (mm *MemoryManager) EnableMembarrierPrivate() {
 func (mm *MemoryManager) IsMembarrierPrivateEnabled() bool {
 	return atomic.LoadUint32(&mm.membarrierPrivateEnabled) != 0
 }
+
+// EnableMembarrierRSeq causes future calls to IsMembarrierRSeqEnabled to
+// return true.
+func (mm *MemoryManager) EnableMembarrierRSeq() {
+	atomic.StoreUint32(&mm.membarrierRSeqEnabled, 1)
+}
+
+// IsMembarrierRSeqEnabled returns true if mm.EnableMembarrierRSeq() has
+// previously been called.
+func (mm *MemoryManager) IsMembarrierRSeqEnabled() bool {
+	return atomic.LoadUint32(&mm.membarrierRSeqEnabled) != 0
+}
diff --git a/pkg/sentry/syscalls/linux/sys_membarrier.go b/pkg/sentry/syscalls/linux/sys_membarrier.go
index 288cd8512..63ee5d435 100644
--- a/pkg/sentry/syscalls/linux/sys_membarrier.go
+++ b/pkg/sentry/syscalls/linux/sys_membarrier.go
@@ -24,47 +24,80 @@ import (
 // Membarrier implements syscall membarrier(2).
 func Membarrier(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) {
 	cmd := args[0].Int()
-	flags := args[1].Int()
-
-	p := t.Kernel().Platform
-	if !p.HaveGlobalMemoryBarrier() {
-		// Event for applications that want membarrier on a configuration that
-		// doesn't support them.
-		t.Kernel().EmitUnimplementedEvent(t)
-		return 0, nil, syserror.ENOSYS
-	}
-
-	if flags != 0 {
-		return 0, nil, syserror.EINVAL
-	}
+	flags := args[1].Uint()
 
 	switch cmd {
 	case linux.MEMBARRIER_CMD_QUERY:
-		const supportedCommands = linux.MEMBARRIER_CMD_GLOBAL |
-			linux.MEMBARRIER_CMD_GLOBAL_EXPEDITED |
-			linux.MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED |
-			linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED |
-			linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED
+		if flags != 0 {
+			return 0, nil, syserror.EINVAL
+		}
+		var supportedCommands uintptr
+		if t.Kernel().Platform.HaveGlobalMemoryBarrier() {
+			supportedCommands |= linux.MEMBARRIER_CMD_GLOBAL |
+				linux.MEMBARRIER_CMD_GLOBAL_EXPEDITED |
+				linux.MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED |
+				linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED |
+				linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED
+		}
+		if t.RSeqAvailable() {
+			supportedCommands |= linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ |
+				linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ
+		}
 		return supportedCommands, nil, nil
-	case linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED:
-		if !t.MemoryManager().IsMembarrierPrivateEnabled() {
+	case linux.MEMBARRIER_CMD_GLOBAL, linux.MEMBARRIER_CMD_GLOBAL_EXPEDITED, linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED:
+		if flags != 0 {
+			return 0, nil, syserror.EINVAL
+		}
+		if !t.Kernel().Platform.HaveGlobalMemoryBarrier() {
+			return 0, nil, syserror.EINVAL
+		}
+		if cmd == linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED && !t.MemoryManager().IsMembarrierPrivateEnabled() {
 			return 0, nil, syserror.EPERM
 		}
-		fallthrough
-	case linux.MEMBARRIER_CMD_GLOBAL, linux.MEMBARRIER_CMD_GLOBAL_EXPEDITED:
-		return 0, nil, p.GlobalMemoryBarrier()
+		return 0, nil, t.Kernel().Platform.GlobalMemoryBarrier()
 	case linux.MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
+		if flags != 0 {
+			return 0, nil, syserror.EINVAL
+		}
+		if !t.Kernel().Platform.HaveGlobalMemoryBarrier() {
+			return 0, nil, syserror.EINVAL
+		}
 		// no-op
 		return 0, nil, nil
 	case linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
+		if flags != 0 {
+			return 0, nil, syserror.EINVAL
+		}
+		if !t.Kernel().Platform.HaveGlobalMemoryBarrier() {
+			return 0, nil, syserror.EINVAL
+		}
 		t.MemoryManager().EnableMembarrierPrivate()
 		return 0, nil, nil
-	case linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE, linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE:
-		// We're aware of these, but they aren't implemented since no platform
-		// supports them yet.
-		t.Kernel().EmitUnimplementedEvent(t)
-		fallthrough
+	case linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
+		if flags&^linux.MEMBARRIER_CMD_FLAG_CPU != 0 {
+			return 0, nil, syserror.EINVAL
+		}
+		if !t.RSeqAvailable() {
+			return 0, nil, syserror.EINVAL
+		}
+		if !t.MemoryManager().IsMembarrierRSeqEnabled() {
+			return 0, nil, syserror.EPERM
+		}
+		// MEMBARRIER_CMD_FLAG_CPU and cpu_id are ignored since we don't have
+		// the ability to preempt specific CPUs.
+		return 0, nil, t.Kernel().Platform.PreemptAllCPUs()
+	case linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ:
+		if flags != 0 {
+			return 0, nil, syserror.EINVAL
+		}
+		if !t.RSeqAvailable() {
+			return 0, nil, syserror.EINVAL
+		}
+		t.MemoryManager().EnableMembarrierRSeq()
+		return 0, nil, nil
 	default:
+		// Probably a command we don't implement.
+		t.Kernel().EmitUnimplementedEvent(t)
 		return 0, nil, syserror.EINVAL
 	}
 }