From 906eb6295d54a05663a223f1dc379a16148de2d1 Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Tue, 18 Feb 2020 13:42:31 -0800 Subject: atomicbitops package cleanups - Redocument memory ordering from "no ordering" to "acquire-release". (No functional change: both LOCK WHATEVER on x86, and LDAXR/STLXR loops on ARM64, already have this property.) - Remove IncUnlessZeroInt32 and DecUnlessOneInt32, which were only faster than the equivalent loops using sync/atomic before the Go compiler inlined non-unsafe.Pointer atomics many releases ago. PiperOrigin-RevId: 295811743 --- pkg/atomicbitops/BUILD | 9 +- pkg/atomicbitops/atomic_bitops.go | 60 ------- pkg/atomicbitops/atomic_bitops_amd64.s | 115 -------------- pkg/atomicbitops/atomic_bitops_arm64.s | 139 ---------------- pkg/atomicbitops/atomic_bitops_common.go | 147 ----------------- pkg/atomicbitops/atomic_bitops_test.go | 262 ------------------------------- pkg/atomicbitops/atomicbitops.go | 47 ++++++ pkg/atomicbitops/atomicbitops_amd64.s | 77 +++++++++ pkg/atomicbitops/atomicbitops_arm64.s | 105 +++++++++++++ pkg/atomicbitops/atomicbitops_noasm.go | 105 +++++++++++++ pkg/atomicbitops/atomicbitops_test.go | 198 +++++++++++++++++++++++ 11 files changed, 536 insertions(+), 728 deletions(-) delete mode 100644 pkg/atomicbitops/atomic_bitops.go delete mode 100644 pkg/atomicbitops/atomic_bitops_amd64.s delete mode 100644 pkg/atomicbitops/atomic_bitops_arm64.s delete mode 100644 pkg/atomicbitops/atomic_bitops_common.go delete mode 100644 pkg/atomicbitops/atomic_bitops_test.go create mode 100644 pkg/atomicbitops/atomicbitops.go create mode 100644 pkg/atomicbitops/atomicbitops_amd64.s create mode 100644 pkg/atomicbitops/atomicbitops_arm64.s create mode 100644 pkg/atomicbitops/atomicbitops_noasm.go create mode 100644 pkg/atomicbitops/atomicbitops_test.go (limited to 'pkg/atomicbitops') diff --git a/pkg/atomicbitops/BUILD b/pkg/atomicbitops/BUILD index 3948074ba..ba8b06071 100644 --- a/pkg/atomicbitops/BUILD +++ b/pkg/atomicbitops/BUILD @@ -5,10 +5,9 @@ package(licenses = ["notice"]) go_library( name = "atomicbitops", srcs = [ - "atomic_bitops.go", - "atomic_bitops_amd64.s", - "atomic_bitops_arm64.s", - "atomic_bitops_common.go", + "atomicbitops.go", + "atomicbitops_amd64.s", + "atomicbitops_noasm.go", ], visibility = ["//:sandbox"], ) @@ -16,7 +15,7 @@ go_library( go_test( name = "atomicbitops_test", size = "small", - srcs = ["atomic_bitops_test.go"], + srcs = ["atomicbitops_test.go"], library = ":atomicbitops", deps = ["//pkg/sync"], ) diff --git a/pkg/atomicbitops/atomic_bitops.go b/pkg/atomicbitops/atomic_bitops.go deleted file mode 100644 index fcc41a9ea..000000000 --- a/pkg/atomicbitops/atomic_bitops.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 arm64 - -// Package atomicbitops provides basic bitwise operations in an atomic way. -// The implementation on amd64 leverages the LOCK prefix directly instead of -// relying on the generic cas primitives, and the arm64 leverages the LDAXR -// and STLXR pair primitives. -// -// WARNING: the bitwise ops provided in this package doesn't imply any memory -// ordering. Using them to construct locks must employ proper memory barriers. -package atomicbitops - -// AndUint32 atomically applies bitwise and operation to *addr with val. -func AndUint32(addr *uint32, val uint32) - -// OrUint32 atomically applies bitwise or operation to *addr with val. -func OrUint32(addr *uint32, val uint32) - -// XorUint32 atomically applies bitwise xor operation to *addr with val. -func XorUint32(addr *uint32, val uint32) - -// CompareAndSwapUint32 is like sync/atomic.CompareAndSwapUint32, but returns -// the value previously stored at addr. -func CompareAndSwapUint32(addr *uint32, old, new uint32) uint32 - -// AndUint64 atomically applies bitwise and operation to *addr with val. -func AndUint64(addr *uint64, val uint64) - -// OrUint64 atomically applies bitwise or operation to *addr with val. -func OrUint64(addr *uint64, val uint64) - -// XorUint64 atomically applies bitwise xor operation to *addr with val. -func XorUint64(addr *uint64, val uint64) - -// CompareAndSwapUint64 is like sync/atomic.CompareAndSwapUint64, but returns -// the value previously stored at addr. -func CompareAndSwapUint64(addr *uint64, old, new uint64) uint64 - -// IncUnlessZeroInt32 increments the value stored at the given address and -// returns true; unless the value stored in the pointer is zero, in which case -// it is left unmodified and false is returned. -func IncUnlessZeroInt32(addr *int32) bool - -// DecUnlessOneInt32 decrements the value stored at the given address and -// returns true; unless the value stored in the pointer is 1, in which case it -// is left unmodified and false is returned. -func DecUnlessOneInt32(addr *int32) bool diff --git a/pkg/atomicbitops/atomic_bitops_amd64.s b/pkg/atomicbitops/atomic_bitops_amd64.s deleted file mode 100644 index db0972001..000000000 --- a/pkg/atomicbitops/atomic_bitops_amd64.s +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -#include "textflag.h" - -TEXT ·AndUint32(SB),$0-12 - MOVQ addr+0(FP), BP - MOVL val+8(FP), AX - LOCK - ANDL AX, 0(BP) - RET - -TEXT ·OrUint32(SB),$0-12 - MOVQ addr+0(FP), BP - MOVL val+8(FP), AX - LOCK - ORL AX, 0(BP) - RET - -TEXT ·XorUint32(SB),$0-12 - MOVQ addr+0(FP), BP - MOVL val+8(FP), AX - LOCK - XORL AX, 0(BP) - RET - -TEXT ·CompareAndSwapUint32(SB),$0-20 - MOVQ addr+0(FP), DI - MOVL old+8(FP), AX - MOVL new+12(FP), DX - LOCK - CMPXCHGL DX, 0(DI) - MOVL AX, ret+16(FP) - RET - -TEXT ·AndUint64(SB),$0-16 - MOVQ addr+0(FP), BP - MOVQ val+8(FP), AX - LOCK - ANDQ AX, 0(BP) - RET - -TEXT ·OrUint64(SB),$0-16 - MOVQ addr+0(FP), BP - MOVQ val+8(FP), AX - LOCK - ORQ AX, 0(BP) - RET - -TEXT ·XorUint64(SB),$0-16 - MOVQ addr+0(FP), BP - MOVQ val+8(FP), AX - LOCK - XORQ AX, 0(BP) - RET - -TEXT ·CompareAndSwapUint64(SB),$0-32 - MOVQ addr+0(FP), DI - MOVQ old+8(FP), AX - MOVQ new+16(FP), DX - LOCK - CMPXCHGQ DX, 0(DI) - MOVQ AX, ret+24(FP) - RET - -TEXT ·IncUnlessZeroInt32(SB),NOSPLIT,$0-9 - MOVQ addr+0(FP), DI - MOVL 0(DI), AX - -retry: - TESTL AX, AX - JZ fail - LEAL 1(AX), DX - LOCK - CMPXCHGL DX, 0(DI) - JNZ retry - - SETEQ ret+8(FP) - RET - -fail: - MOVB AX, ret+8(FP) - RET - -TEXT ·DecUnlessOneInt32(SB),NOSPLIT,$0-9 - MOVQ addr+0(FP), DI - MOVL 0(DI), AX - -retry: - LEAL -1(AX), DX - TESTL DX, DX - JZ fail - LOCK - CMPXCHGL DX, 0(DI) - JNZ retry - - SETEQ ret+8(FP) - RET - -fail: - MOVB DX, ret+8(FP) - RET diff --git a/pkg/atomicbitops/atomic_bitops_arm64.s b/pkg/atomicbitops/atomic_bitops_arm64.s deleted file mode 100644 index 97f8808c1..000000000 --- a/pkg/atomicbitops/atomic_bitops_arm64.s +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build arm64 - -#include "textflag.h" - -TEXT ·AndUint32(SB),$0-12 - MOVD ptr+0(FP), R0 - MOVW val+8(FP), R1 -again: - LDAXRW (R0), R2 - ANDW R1, R2 - STLXRW R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·OrUint32(SB),$0-12 - MOVD ptr+0(FP), R0 - MOVW val+8(FP), R1 -again: - LDAXRW (R0), R2 - ORRW R1, R2 - STLXRW R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·XorUint32(SB),$0-12 - MOVD ptr+0(FP), R0 - MOVW val+8(FP), R1 -again: - LDAXRW (R0), R2 - EORW R1, R2 - STLXRW R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·CompareAndSwapUint32(SB),$0-20 - MOVD addr+0(FP), R0 - MOVW old+8(FP), R1 - MOVW new+12(FP), R2 - -again: - LDAXRW (R0), R3 - CMPW R1, R3 - BNE done - STLXRW R2, (R0), R4 - CBNZ R4, again -done: - MOVW R3, prev+16(FP) - RET - -TEXT ·AndUint64(SB),$0-16 - MOVD ptr+0(FP), R0 - MOVD val+8(FP), R1 -again: - LDAXR (R0), R2 - AND R1, R2 - STLXR R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·OrUint64(SB),$0-16 - MOVD ptr+0(FP), R0 - MOVD val+8(FP), R1 -again: - LDAXR (R0), R2 - ORR R1, R2 - STLXR R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·XorUint64(SB),$0-16 - MOVD ptr+0(FP), R0 - MOVD val+8(FP), R1 -again: - LDAXR (R0), R2 - EOR R1, R2 - STLXR R2, (R0), R3 - CBNZ R3, again - RET - -TEXT ·CompareAndSwapUint64(SB),$0-32 - MOVD addr+0(FP), R0 - MOVD old+8(FP), R1 - MOVD new+16(FP), R2 - -again: - LDAXR (R0), R3 - CMP R1, R3 - BNE done - STLXR R2, (R0), R4 - CBNZ R4, again -done: - MOVD R3, prev+24(FP) - RET - -TEXT ·IncUnlessZeroInt32(SB),NOSPLIT,$0-9 - MOVD addr+0(FP), R0 - -again: - LDAXRW (R0), R1 - CBZ R1, fail - ADDW $1, R1 - STLXRW R1, (R0), R2 - CBNZ R2, again - MOVW $1, R2 - MOVB R2, ret+8(FP) - RET -fail: - MOVB ZR, ret+8(FP) - RET - -TEXT ·DecUnlessOneInt32(SB),NOSPLIT,$0-9 - MOVD addr+0(FP), R0 - -again: - LDAXRW (R0), R1 - SUBSW $1, R1, R1 - BEQ fail - STLXRW R1, (R0), R2 - CBNZ R2, again - MOVW $1, R2 - MOVB R2, ret+8(FP) - RET -fail: - MOVB ZR, ret+8(FP) - RET diff --git a/pkg/atomicbitops/atomic_bitops_common.go b/pkg/atomicbitops/atomic_bitops_common.go deleted file mode 100644 index 85163ad62..000000000 --- a/pkg/atomicbitops/atomic_bitops_common.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !amd64,!arm64 - -package atomicbitops - -import ( - "sync/atomic" -) - -// AndUint32 atomically applies bitwise and operation to *addr with val. -func AndUint32(addr *uint32, val uint32) { - for { - o := atomic.LoadUint32(addr) - n := o & val - if atomic.CompareAndSwapUint32(addr, o, n) { - break - } - } -} - -// OrUint32 atomically applies bitwise or operation to *addr with val. -func OrUint32(addr *uint32, val uint32) { - for { - o := atomic.LoadUint32(addr) - n := o | val - if atomic.CompareAndSwapUint32(addr, o, n) { - break - } - } -} - -// XorUint32 atomically applies bitwise xor operation to *addr with val. -func XorUint32(addr *uint32, val uint32) { - for { - o := atomic.LoadUint32(addr) - n := o ^ val - if atomic.CompareAndSwapUint32(addr, o, n) { - break - } - } -} - -// CompareAndSwapUint32 is like sync/atomic.CompareAndSwapUint32, but returns -// the value previously stored at addr. -func CompareAndSwapUint32(addr *uint32, old, new uint32) (prev uint32) { - for { - prev = atomic.LoadUint32(addr) - if prev != old { - return - } - if atomic.CompareAndSwapUint32(addr, old, new) { - return - } - } -} - -// AndUint64 atomically applies bitwise and operation to *addr with val. -func AndUint64(addr *uint64, val uint64) { - for { - o := atomic.LoadUint64(addr) - n := o & val - if atomic.CompareAndSwapUint64(addr, o, n) { - break - } - } -} - -// OrUint64 atomically applies bitwise or operation to *addr with val. -func OrUint64(addr *uint64, val uint64) { - for { - o := atomic.LoadUint64(addr) - n := o | val - if atomic.CompareAndSwapUint64(addr, o, n) { - break - } - } -} - -// XorUint64 atomically applies bitwise xor operation to *addr with val. -func XorUint64(addr *uint64, val uint64) { - for { - o := atomic.LoadUint64(addr) - n := o ^ val - if atomic.CompareAndSwapUint64(addr, o, n) { - break - } - } -} - -// CompareAndSwapUint64 is like sync/atomic.CompareAndSwapUint64, but returns -// the value previously stored at addr. -func CompareAndSwapUint64(addr *uint64, old, new uint64) (prev uint64) { - for { - prev = atomic.LoadUint64(addr) - if prev != old { - return - } - if atomic.CompareAndSwapUint64(addr, old, new) { - return - } - } -} - -// IncUnlessZeroInt32 increments the value stored at the given address and -// returns true; unless the value stored in the pointer is zero, in which case -// it is left unmodified and false is returned. -func IncUnlessZeroInt32(addr *int32) bool { - for { - v := atomic.LoadInt32(addr) - if v == 0 { - return false - } - - if atomic.CompareAndSwapInt32(addr, v, v+1) { - return true - } - } -} - -// DecUnlessOneInt32 decrements the value stored at the given address and -// returns true; unless the value stored in the pointer is 1, in which case it -// is left unmodified and false is returned. -func DecUnlessOneInt32(addr *int32) bool { - for { - v := atomic.LoadInt32(addr) - if v == 1 { - return false - } - - if atomic.CompareAndSwapInt32(addr, v, v-1) { - return true - } - } -} diff --git a/pkg/atomicbitops/atomic_bitops_test.go b/pkg/atomicbitops/atomic_bitops_test.go deleted file mode 100644 index 9466d3e23..000000000 --- a/pkg/atomicbitops/atomic_bitops_test.go +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package atomicbitops - -import ( - "runtime" - "testing" - - "gvisor.dev/gvisor/pkg/sync" -) - -const iterations = 100 - -func detectRaces32(val, target uint32, fn func(*uint32, uint32)) bool { - runtime.GOMAXPROCS(100) - for n := 0; n < iterations; n++ { - x := val - var wg sync.WaitGroup - for i := uint32(0); i < 32; i++ { - wg.Add(1) - go func(a *uint32, i uint32) { - defer wg.Done() - fn(a, uint32(1< Date: Thu, 20 Feb 2020 11:14:53 -0800 Subject: Re-add atomicbitops_arm64.s to BUILD. This was inadverently dropped by cl/295811743. PiperOrigin-RevId: 296254482 --- pkg/atomicbitops/BUILD | 1 + 1 file changed, 1 insertion(+) (limited to 'pkg/atomicbitops') diff --git a/pkg/atomicbitops/BUILD b/pkg/atomicbitops/BUILD index ba8b06071..1a30f6967 100644 --- a/pkg/atomicbitops/BUILD +++ b/pkg/atomicbitops/BUILD @@ -7,6 +7,7 @@ go_library( srcs = [ "atomicbitops.go", "atomicbitops_amd64.s", + "atomicbitops_arm64.s", "atomicbitops_noasm.go", ], visibility = ["//:sandbox"], -- cgit v1.2.3