diff options
| -rwxr-xr-x | pkg/atomicbitops/atomic_bitops_arm64.s | 139 | ||||
| -rwxr-xr-x[-rw-r--r--] | pkg/atomicbitops/atomicbitops.go (renamed from pkg/atomicbitops/atomic_bitops.go) | 31 | ||||
| -rwxr-xr-x[-rw-r--r--] | pkg/atomicbitops/atomicbitops_amd64.s (renamed from pkg/atomicbitops/atomic_bitops_amd64.s) | 38 | ||||
| -rwxr-xr-x[-rw-r--r--] | pkg/atomicbitops/atomicbitops_noasm.go (renamed from pkg/atomicbitops/atomic_bitops_common.go) | 42 | ||||
| -rw-r--r-- | pkg/sentry/mm/address_space.go | 23 | ||||
| -rw-r--r-- | pkg/sentry/mm/lifecycle.go | 11 |
6 files changed, 36 insertions, 248 deletions
diff --git a/pkg/atomicbitops/atomic_bitops_arm64.s b/pkg/atomicbitops/atomic_bitops_arm64.s deleted file mode 100755 index 97f8808c1..000000000 --- a/pkg/atomicbitops/atomic_bitops_arm64.s +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -#include "textflag.h" - -TEXT ·AndUint32(SB),$0-12 - MOVD ptr+0(FP), R0 - MOVW val+8(FP), R1 -again: - LDAXRW (R0), R2 - ANDW R1, R2 - STLXRW R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·OrUint32(SB),$0-12 - MOVD ptr+0(FP), R0 - MOVW val+8(FP), R1 -again: - LDAXRW (R0), R2 - ORRW R1, R2 - STLXRW R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·XorUint32(SB),$0-12 - MOVD ptr+0(FP), R0 - MOVW val+8(FP), R1 -again: - LDAXRW (R0), R2 - EORW R1, R2 - STLXRW R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·CompareAndSwapUint32(SB),$0-20 - MOVD addr+0(FP), R0 - MOVW old+8(FP), R1 - MOVW new+12(FP), R2 - -again: - LDAXRW (R0), R3 - CMPW R1, R3 - BNE done - STLXRW R2, (R0), R4 - CBNZ R4, again -done: - MOVW R3, prev+16(FP) - RET - -TEXT ·AndUint64(SB),$0-16 - MOVD ptr+0(FP), R0 - MOVD val+8(FP), R1 -again: - LDAXR (R0), R2 - AND R1, R2 - STLXR R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·OrUint64(SB),$0-16 - MOVD ptr+0(FP), R0 - MOVD val+8(FP), R1 -again: - LDAXR (R0), R2 - ORR R1, R2 - STLXR R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·XorUint64(SB),$0-16 - MOVD ptr+0(FP), R0 - MOVD val+8(FP), R1 -again: - LDAXR (R0), R2 - EOR R1, R2 - STLXR R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·CompareAndSwapUint64(SB),$0-32 - MOVD addr+0(FP), R0 - MOVD old+8(FP), R1 - MOVD new+16(FP), R2 - -again: - LDAXR (R0), R3 - CMP R1, R3 - BNE done - STLXR R2, (R0), R4 - CBNZ R4, again -done: - MOVD R3, prev+24(FP) - RET - -TEXT ·IncUnlessZeroInt32(SB),NOSPLIT,$0-9 - MOVD addr+0(FP), R0 - -again: - LDAXRW (R0), R1 - CBZ R1, fail - ADDW $1, R1 - STLXRW R1, (R0), R2 - CBNZ R2, again - MOVW $1, R2 - MOVB R2, ret+8(FP) - RET -fail: - MOVB ZR, ret+8(FP) - RET - -TEXT ·DecUnlessOneInt32(SB),NOSPLIT,$0-9 - MOVD addr+0(FP), R0 - -again: - LDAXRW (R0), R1 - SUBSW $1, R1, R1 - BEQ fail - STLXRW R1, (R0), R2 - CBNZ R2, again - MOVW $1, R2 - MOVB R2, ret+8(FP) - RET -fail: - MOVB ZR, ret+8(FP) - RET diff --git a/pkg/atomicbitops/atomic_bitops.go b/pkg/atomicbitops/atomicbitops.go index fcc41a9ea..1be081719 100644..100755 --- a/pkg/atomicbitops/atomic_bitops.go +++ b/pkg/atomicbitops/atomicbitops.go @@ -14,47 +14,34 @@ // +build amd64 arm64 -// Package atomicbitops provides basic bitwise operations in an atomic way. -// The implementation on amd64 leverages the LOCK prefix directly instead of -// relying on the generic cas primitives, and the arm64 leverages the LDAXR -// and STLXR pair primitives. +// Package atomicbitops provides extensions to the sync/atomic package. // -// WARNING: the bitwise ops provided in this package doesn't imply any memory -// ordering. Using them to construct locks must employ proper memory barriers. +// All read-modify-write operations implemented by this package have +// acquire-release memory ordering (like sync/atomic). package atomicbitops -// AndUint32 atomically applies bitwise and operation to *addr with val. +// AndUint32 atomically applies bitwise AND operation to *addr with val. func AndUint32(addr *uint32, val uint32) -// OrUint32 atomically applies bitwise or operation to *addr with val. +// OrUint32 atomically applies bitwise OR operation to *addr with val. func OrUint32(addr *uint32, val uint32) -// XorUint32 atomically applies bitwise xor operation to *addr with val. +// XorUint32 atomically applies bitwise XOR operation to *addr with val. func XorUint32(addr *uint32, val uint32) // CompareAndSwapUint32 is like sync/atomic.CompareAndSwapUint32, but returns // the value previously stored at addr. func CompareAndSwapUint32(addr *uint32, old, new uint32) uint32 -// AndUint64 atomically applies bitwise and operation to *addr with val. +// AndUint64 atomically applies bitwise AND operation to *addr with val. func AndUint64(addr *uint64, val uint64) -// OrUint64 atomically applies bitwise or operation to *addr with val. +// OrUint64 atomically applies bitwise OR operation to *addr with val. func OrUint64(addr *uint64, val uint64) -// XorUint64 atomically applies bitwise xor operation to *addr with val. +// XorUint64 atomically applies bitwise XOR operation to *addr with val. func XorUint64(addr *uint64, val uint64) // CompareAndSwapUint64 is like sync/atomic.CompareAndSwapUint64, but returns // the value previously stored at addr. func CompareAndSwapUint64(addr *uint64, old, new uint64) uint64 - -// IncUnlessZeroInt32 increments the value stored at the given address and -// returns true; unless the value stored in the pointer is zero, in which case -// it is left unmodified and false is returned. -func IncUnlessZeroInt32(addr *int32) bool - -// DecUnlessOneInt32 decrements the value stored at the given address and -// returns true; unless the value stored in the pointer is 1, in which case it -// is left unmodified and false is returned. -func DecUnlessOneInt32(addr *int32) bool diff --git a/pkg/atomicbitops/atomic_bitops_amd64.s b/pkg/atomicbitops/atomicbitops_amd64.s index db0972001..54c887ee5 100644..100755 --- a/pkg/atomicbitops/atomic_bitops_amd64.s +++ b/pkg/atomicbitops/atomicbitops_amd64.s @@ -75,41 +75,3 @@ TEXT ·CompareAndSwapUint64(SB),$0-32 CMPXCHGQ DX, 0(DI) MOVQ AX, ret+24(FP) RET - -TEXT ·IncUnlessZeroInt32(SB),NOSPLIT,$0-9 - MOVQ addr+0(FP), DI - MOVL 0(DI), AX - -retry: - TESTL AX, AX - JZ fail - LEAL 1(AX), DX - LOCK - CMPXCHGL DX, 0(DI) - JNZ retry - - SETEQ ret+8(FP) - RET - -fail: - MOVB AX, ret+8(FP) - RET - -TEXT ·DecUnlessOneInt32(SB),NOSPLIT,$0-9 - MOVQ addr+0(FP), DI - MOVL 0(DI), AX - -retry: - LEAL -1(AX), DX - TESTL DX, DX - JZ fail - LOCK - CMPXCHGL DX, 0(DI) - JNZ retry - - SETEQ ret+8(FP) - RET - -fail: - MOVB DX, ret+8(FP) - RET diff --git a/pkg/atomicbitops/atomic_bitops_common.go b/pkg/atomicbitops/atomicbitops_noasm.go index 85163ad62..3b2898256 100644..100755 --- a/pkg/atomicbitops/atomic_bitops_common.go +++ b/pkg/atomicbitops/atomicbitops_noasm.go @@ -20,7 +20,6 @@ import ( "sync/atomic" ) -// AndUint32 atomically applies bitwise and operation to *addr with val. func AndUint32(addr *uint32, val uint32) { for { o := atomic.LoadUint32(addr) @@ -31,7 +30,6 @@ func AndUint32(addr *uint32, val uint32) { } } -// OrUint32 atomically applies bitwise or operation to *addr with val. func OrUint32(addr *uint32, val uint32) { for { o := atomic.LoadUint32(addr) @@ -42,7 +40,6 @@ func OrUint32(addr *uint32, val uint32) { } } -// XorUint32 atomically applies bitwise xor operation to *addr with val. func XorUint32(addr *uint32, val uint32) { for { o := atomic.LoadUint32(addr) @@ -53,8 +50,6 @@ func XorUint32(addr *uint32, val uint32) { } } -// CompareAndSwapUint32 is like sync/atomic.CompareAndSwapUint32, but returns -// the value previously stored at addr. func CompareAndSwapUint32(addr *uint32, old, new uint32) (prev uint32) { for { prev = atomic.LoadUint32(addr) @@ -67,7 +62,6 @@ func CompareAndSwapUint32(addr *uint32, old, new uint32) (prev uint32) { } } -// AndUint64 atomically applies bitwise and operation to *addr with val. func AndUint64(addr *uint64, val uint64) { for { o := atomic.LoadUint64(addr) @@ -78,7 +72,6 @@ func AndUint64(addr *uint64, val uint64) { } } -// OrUint64 atomically applies bitwise or operation to *addr with val. func OrUint64(addr *uint64, val uint64) { for { o := atomic.LoadUint64(addr) @@ -89,7 +82,6 @@ func OrUint64(addr *uint64, val uint64) { } } -// XorUint64 atomically applies bitwise xor operation to *addr with val. func XorUint64(addr *uint64, val uint64) { for { o := atomic.LoadUint64(addr) @@ -100,8 +92,6 @@ func XorUint64(addr *uint64, val uint64) { } } -// CompareAndSwapUint64 is like sync/atomic.CompareAndSwapUint64, but returns -// the value previously stored at addr. func CompareAndSwapUint64(addr *uint64, old, new uint64) (prev uint64) { for { prev = atomic.LoadUint64(addr) @@ -113,35 +103,3 @@ func CompareAndSwapUint64(addr *uint64, old, new uint64) (prev uint64) { } } } - -// IncUnlessZeroInt32 increments the value stored at the given address and -// returns true; unless the value stored in the pointer is zero, in which case -// it is left unmodified and false is returned. -func IncUnlessZeroInt32(addr *int32) bool { - for { - v := atomic.LoadInt32(addr) - if v == 0 { - return false - } - - if atomic.CompareAndSwapInt32(addr, v, v+1) { - return true - } - } -} - -// DecUnlessOneInt32 decrements the value stored at the given address and -// returns true; unless the value stored in the pointer is 1, in which case it -// is left unmodified and false is returned. -func DecUnlessOneInt32(addr *int32) bool { - for { - v := atomic.LoadInt32(addr) - if v == 1 { - return false - } - - if atomic.CompareAndSwapInt32(addr, v, v-1) { - return true - } - } -} diff --git a/pkg/sentry/mm/address_space.go b/pkg/sentry/mm/address_space.go index e58a63deb..94d39af60 100644 --- a/pkg/sentry/mm/address_space.go +++ b/pkg/sentry/mm/address_space.go @@ -18,7 +18,6 @@ import ( "fmt" "sync/atomic" - "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/usermem" ) @@ -42,8 +41,15 @@ func (mm *MemoryManager) AddressSpace() platform.AddressSpace { func (mm *MemoryManager) Activate() error { // Fast path: the MemoryManager already has an active // platform.AddressSpace, and we just need to indicate that we need it too. - if atomicbitops.IncUnlessZeroInt32(&mm.active) { - return nil + for { + active := atomic.LoadInt32(&mm.active) + if active == 0 { + // Fall back to the slow path. + break + } + if atomic.CompareAndSwapInt32(&mm.active, active, active+1) { + return nil + } } for { @@ -118,8 +124,15 @@ func (mm *MemoryManager) Activate() error { func (mm *MemoryManager) Deactivate() { // Fast path: this is not the last goroutine to deactivate the // MemoryManager. - if atomicbitops.DecUnlessOneInt32(&mm.active) { - return + for { + active := atomic.LoadInt32(&mm.active) + if active == 1 { + // Fall back to the slow path. + break + } + if atomic.CompareAndSwapInt32(&mm.active, active, active-1) { + return + } } mm.activeMu.Lock() diff --git a/pkg/sentry/mm/lifecycle.go b/pkg/sentry/mm/lifecycle.go index 47b8fbf43..3c263ebaa 100644 --- a/pkg/sentry/mm/lifecycle.go +++ b/pkg/sentry/mm/lifecycle.go @@ -18,7 +18,6 @@ import ( "fmt" "sync/atomic" - "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/limits" @@ -229,7 +228,15 @@ func (mm *MemoryManager) Fork(ctx context.Context) (*MemoryManager, error) { // IncUsers increments mm's user count and returns true. If the user count is // already 0, IncUsers does nothing and returns false. func (mm *MemoryManager) IncUsers() bool { - return atomicbitops.IncUnlessZeroInt32(&mm.users) + for { + users := atomic.LoadInt32(&mm.users) + if users == 0 { + return false + } + if atomic.CompareAndSwapInt32(&mm.users, users, users+1) { + return true + } + } } // DecUsers decrements mm's user count. If the user count reaches 0, all |