diff options
| -rw-r--r-- | pkg/sleep/BUILD | 4 | ||||
| -rw-r--r-- | pkg/sleep/commit_amd64.s | 35 | ||||
| -rw-r--r-- | pkg/sleep/commit_arm64.s | 38 | ||||
| -rw-r--r-- | pkg/sleep/commit_asm.go | 20 | ||||
| -rw-r--r-- | pkg/sleep/commit_noasm.go | 33 | ||||
| -rw-r--r-- | pkg/sleep/empty.s | 15 | ||||
| -rw-r--r-- | pkg/sleep/sleep_unsafe.go | 27 | ||||
| -rw-r--r-- | pkg/sync/BUILD | 8 | ||||
| -rw-r--r-- | pkg/sync/goyield_go113_unsafe.go (renamed from pkg/sync/spin_legacy_unsafe.go) | 7 | ||||
| -rw-r--r-- | pkg/sync/goyield_unsafe.go (renamed from pkg/sync/spin_unsafe.go) | 6 | ||||
| -rw-r--r-- | pkg/sync/memmove_unsafe.go | 28 | ||||
| -rw-r--r-- | pkg/sync/norace_unsafe.go | 11 | ||||
| -rw-r--r-- | pkg/sync/race_amd64.s (renamed from pkg/syncevent/waiter_amd64.s) | 17 | ||||
| -rw-r--r-- | pkg/sync/race_arm64.s (renamed from pkg/syncevent/waiter_arm64.s) | 17 | ||||
| -rw-r--r-- | pkg/sync/race_unsafe.go | 6 | ||||
| -rw-r--r-- | pkg/sync/runtime_unsafe.go | 76 | ||||
| -rw-r--r-- | pkg/sync/rwmutex_unsafe.go | 23 | ||||
| -rw-r--r-- | pkg/syncevent/BUILD | 4 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_asm_unsafe.go | 24 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_noasm_unsafe.go | 39 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_unsafe.go | 59 |
21 files changed, 161 insertions, 336 deletions
diff --git a/pkg/sleep/BUILD b/pkg/sleep/BUILD index ae0fe1522..48bcdd62b 100644 --- a/pkg/sleep/BUILD +++ b/pkg/sleep/BUILD @@ -5,10 +5,6 @@ package(licenses = ["notice"]) go_library( name = "sleep", srcs = [ - "commit_amd64.s", - "commit_arm64.s", - "commit_asm.go", - "commit_noasm.go", "sleep_unsafe.go", ], visibility = ["//:sandbox"], diff --git a/pkg/sleep/commit_amd64.s b/pkg/sleep/commit_amd64.s deleted file mode 100644 index bc4ac2c3c..000000000 --- a/pkg/sleep/commit_amd64.s +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "textflag.h" - -#define preparingG 1 - -// See commit_noasm.go for a description of commitSleep. -// -// func commitSleep(g uintptr, waitingG *uintptr) bool -TEXT ·commitSleep(SB),NOSPLIT,$0-24 - MOVQ waitingG+8(FP), CX - MOVQ g+0(FP), DX - - // Store the G in waitingG if it's still preparingG. If it's anything - // else it means a waker has aborted the sleep. - MOVQ $preparingG, AX - LOCK - CMPXCHGQ DX, 0(CX) - - SETEQ AX - MOVB AX, ret+16(FP) - - RET diff --git a/pkg/sleep/commit_arm64.s b/pkg/sleep/commit_arm64.s deleted file mode 100644 index d0ef15b20..000000000 --- a/pkg/sleep/commit_arm64.s +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "textflag.h" - -#define preparingG 1 - -// See commit_noasm.go for a description of commitSleep. -// -// func commitSleep(g uintptr, waitingG *uintptr) bool -TEXT ·commitSleep(SB),NOSPLIT,$0-24 - MOVD waitingG+8(FP), R0 - MOVD $preparingG, R1 - MOVD G+0(FP), R2 - - // Store the G in waitingG if it's still preparingG. If it's anything - // else it means a waker has aborted the sleep. -again: - LDAXR (R0), R3 - CMP R1, R3 - BNE ok - STLXR R2, (R0), R3 - CBNZ R3, again -ok: - CSET EQ, R0 - MOVB R0, ret+16(FP) - RET diff --git a/pkg/sleep/commit_asm.go b/pkg/sleep/commit_asm.go deleted file mode 100644 index 75728a97d..000000000 --- a/pkg/sleep/commit_asm.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 arm64 - -package sleep - -// See commit_noasm.go for a description of commitSleep. -func commitSleep(g uintptr, waitingG *uintptr) bool diff --git a/pkg/sleep/commit_noasm.go b/pkg/sleep/commit_noasm.go deleted file mode 100644 index f59061f37..000000000 --- a/pkg/sleep/commit_noasm.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !race -// +build !amd64,!arm64 - -package sleep - -import "sync/atomic" - -// commitSleep signals to wakers that the given g is now sleeping. Wakers can -// then fetch it and wake it. -// -// The commit may fail if wakers have been asserted after our last check, in -// which case they will have set s.waitingG to zero. -// -// It is written in assembly because it is called from g0, so it doesn't have -// a race context. -func commitSleep(g uintptr, waitingG *uintptr) bool { - // Try to store the G so that wakers know who to wake. - return atomic.CompareAndSwapUintptr(waitingG, preparingG, g) -} diff --git a/pkg/sleep/empty.s b/pkg/sleep/empty.s deleted file mode 100644 index fb37360ac..000000000 --- a/pkg/sleep/empty.s +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Empty assembly file so empty func definitions work. diff --git a/pkg/sleep/sleep_unsafe.go b/pkg/sleep/sleep_unsafe.go index 19bce2afb..c44206b1e 100644 --- a/pkg/sleep/sleep_unsafe.go +++ b/pkg/sleep/sleep_unsafe.go @@ -12,11 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.11 -// +build !go1.17 - -// Check go:linkname function signatures when updating Go version. - // Package sleep allows goroutines to efficiently sleep on multiple sources of // notifications (wakers). It offers O(1) complexity, which is different from // multi-channel selects which have O(n) complexity (where n is the number of @@ -91,12 +86,6 @@ var ( assertedSleeper Sleeper ) -//go:linkname gopark runtime.gopark -func gopark(unlockf func(uintptr, *uintptr) bool, wg *uintptr, reason uint8, traceEv byte, traceskip int) - -//go:linkname goready runtime.goready -func goready(g uintptr, traceskip int) - // Sleeper allows a goroutine to sleep and receive wake up notifications from // Wakers in an efficient way. // @@ -189,7 +178,7 @@ func (s *Sleeper) nextWaker(block bool) *Waker { // See:runtime2.go in the go runtime package for // the values to pass as the waitReason here. const waitReasonSelect = 9 - gopark(commitSleep, &s.waitingG, waitReasonSelect, traceEvGoBlockSelect, 0) + sync.Gopark(commitSleep, unsafe.Pointer(&s.waitingG), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0) } // Pull the shared list out and reverse it in the local @@ -212,6 +201,18 @@ func (s *Sleeper) nextWaker(block bool) *Waker { return w } +// commitSleep signals to wakers that the given g is now sleeping. Wakers can +// then fetch it and wake it. +// +// The commit may fail if wakers have been asserted after our last check, in +// which case they will have set s.waitingG to zero. +// +//go:norace +//go:nosplit +func commitSleep(g uintptr, waitingG unsafe.Pointer) bool { + return sync.RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(waitingG), preparingG, g) +} + // Fetch fetches the next wake-up notification. If a notification is immediately // available, it is returned right away. Otherwise, the behavior depends on the // value of 'block': if true, the current goroutine blocks until a notification @@ -311,7 +312,7 @@ func (s *Sleeper) enqueueAssertedWaker(w *Waker) { case 0, preparingG: default: // We managed to get a G. Wake it up. - goready(g, 0) + sync.Goready(g, 0) } } diff --git a/pkg/sync/BUILD b/pkg/sync/BUILD index 5bd4d09d1..be5bc99fc 100644 --- a/pkg/sync/BUILD +++ b/pkg/sync/BUILD @@ -33,15 +33,17 @@ go_library( "aliases.go", "checklocks_off_unsafe.go", "checklocks_on_unsafe.go", - "memmove_unsafe.go", + "goyield_go113_unsafe.go", + "goyield_unsafe.go", "mutex_unsafe.go", "nocopy.go", "norace_unsafe.go", + "race_amd64.s", + "race_arm64.s", "race_unsafe.go", + "runtime_unsafe.go", "rwmutex_unsafe.go", "seqcount.go", - "spin_legacy_unsafe.go", - "spin_unsafe.go", "sync.go", ], marshal = False, diff --git a/pkg/sync/spin_legacy_unsafe.go b/pkg/sync/goyield_go113_unsafe.go index 61fc7320e..8aee0d455 100644 --- a/pkg/sync/spin_legacy_unsafe.go +++ b/pkg/sync/goyield_go113_unsafe.go @@ -10,15 +10,8 @@ package sync import ( "runtime" - _ "unsafe" // for go:linkname ) -//go:linkname canSpin sync.runtime_canSpin -func canSpin(i int) bool - -//go:linkname doSpin sync.runtime_doSpin -func doSpin() - func goyield() { // goyield is not available until Go 1.14. runtime.Gosched() diff --git a/pkg/sync/spin_unsafe.go b/pkg/sync/goyield_unsafe.go index 18e8fc743..672ee274d 100644 --- a/pkg/sync/spin_unsafe.go +++ b/pkg/sync/goyield_unsafe.go @@ -14,11 +14,5 @@ import ( _ "unsafe" // for go:linkname ) -//go:linkname canSpin sync.runtime_canSpin -func canSpin(i int) bool - -//go:linkname doSpin sync.runtime_doSpin -func doSpin() - //go:linkname goyield runtime.goyield func goyield() diff --git a/pkg/sync/memmove_unsafe.go b/pkg/sync/memmove_unsafe.go deleted file mode 100644 index f5e630009..000000000 --- a/pkg/sync/memmove_unsafe.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build go1.12 -// +build !go1.17 - -// Check go:linkname function signatures when updating Go version. - -package sync - -import ( - "unsafe" -) - -//go:linkname memmove runtime.memmove -//go:noescape -func memmove(to, from unsafe.Pointer, n uintptr) - -// Memmove is exported for SeqAtomicLoad/SeqAtomicTryLoad<T>, which can't -// define it because go_generics can't update the go:linkname annotation. -// Furthermore, go:linkname silently doesn't work if the local name is exported -// (this is of course undocumented), which is why this indirection is -// necessary. -func Memmove(to, from unsafe.Pointer, n uintptr) { - memmove(to, from, n) -} diff --git a/pkg/sync/norace_unsafe.go b/pkg/sync/norace_unsafe.go index 006055dd6..70b5f3a5e 100644 --- a/pkg/sync/norace_unsafe.go +++ b/pkg/sync/norace_unsafe.go @@ -8,6 +8,7 @@ package sync import ( + "sync/atomic" "unsafe" ) @@ -33,3 +34,13 @@ func RaceRelease(addr unsafe.Pointer) { // RaceReleaseMerge has the same semantics as runtime.RaceReleaseMerge. func RaceReleaseMerge(addr unsafe.Pointer) { } + +// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to +// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector. +// This is necessary when implementing gopark callbacks, since no race context +// is available during their execution. +func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool { + // Use atomic.CompareAndSwapUintptr outside of race builds for + // inlinability. + return atomic.CompareAndSwapUintptr(ptr, old, new) +} diff --git a/pkg/syncevent/waiter_amd64.s b/pkg/sync/race_amd64.s index 5e216b045..57bc0ec79 100644 --- a/pkg/syncevent/waiter_amd64.s +++ b/pkg/sync/race_amd64.s @@ -12,21 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build race +// +build amd64 + #include "textflag.h" -// See waiter_noasm_unsafe.go for a description of waiterUnlock. -// -// func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool -TEXT ·waiterUnlock(SB),NOSPLIT,$0-24 +// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool +TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT,$0-25 MOVQ ptr+0(FP), DI - MOVQ wg+8(FP), SI + MOVQ old+8(FP), AX + MOVQ new+16(FP), SI - MOVQ $·preparingG(SB), AX LOCK - CMPXCHGQ DI, 0(SI) + CMPXCHGQ SI, 0(DI) SETEQ AX - MOVB AX, ret+16(FP) + MOVB AX, ret+24(FP) RET diff --git a/pkg/syncevent/waiter_arm64.s b/pkg/sync/race_arm64.s index f4c06f194..88f091fda 100644 --- a/pkg/syncevent/waiter_arm64.s +++ b/pkg/sync/race_arm64.s @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build race +// +build arm64 + #include "textflag.h" -// See waiter_noasm_unsafe.go for a description of waiterUnlock. -// -// func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool -TEXT ·waiterUnlock(SB),NOSPLIT,$0-24 - MOVD wg+8(FP), R0 - MOVD $·preparingG(SB), R1 - MOVD ptr+0(FP), R2 +// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool +TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT,$0-25 + MOVD ptr+0(FP), R0 + MOVD old+8(FP), R1 + MOVD new+16(FP), R1 again: LDAXR (R0), R3 CMP R1, R3 @@ -29,6 +30,6 @@ again: CBNZ R3, again ok: CSET EQ, R0 - MOVB R0, ret+16(FP) + MOVB R0, ret+24(FP) RET diff --git a/pkg/sync/race_unsafe.go b/pkg/sync/race_unsafe.go index 31d8fa9a6..59985c270 100644 --- a/pkg/sync/race_unsafe.go +++ b/pkg/sync/race_unsafe.go @@ -39,3 +39,9 @@ func RaceRelease(addr unsafe.Pointer) { func RaceReleaseMerge(addr unsafe.Pointer) { runtime.RaceReleaseMerge(addr) } + +// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to +// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector. +// This is necessary when implementing gopark callbacks, since no race context +// is available during their execution. +func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool diff --git a/pkg/sync/runtime_unsafe.go b/pkg/sync/runtime_unsafe.go new file mode 100644 index 000000000..7ad6a4434 --- /dev/null +++ b/pkg/sync/runtime_unsafe.go @@ -0,0 +1,76 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.13 +// +build !go1.17 + +// Check function signatures and constants when updating Go version. + +package sync + +import ( + "unsafe" +) + +// Note that go:linkname silently doesn't work if the local name is exported, +// necessitating an indirection for exported functions. + +// Memmove is runtime.memmove, exported for SeqAtomicLoad/SeqAtomicTryLoad<T>. +// +//go:nosplit +func Memmove(to, from unsafe.Pointer, n uintptr) { + memmove(to, from, n) +} + +//go:linkname memmove runtime.memmove +//go:noescape +func memmove(to, from unsafe.Pointer, n uintptr) + +// Gopark is runtime.gopark. Gopark calls unlockf(pointer to runtime.g, lock); +// if unlockf returns true, Gopark blocks until Goready(pointer to runtime.g) +// is called. unlockf and its callees must be nosplit and norace, since stack +// splitting and race context are not available where it is called. +// +//go:nosplit +func Gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceEv byte, traceskip int) { + gopark(unlockf, lock, reason, traceEv, traceskip) +} + +//go:linkname gopark runtime.gopark +func gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceEv byte, traceskip int) + +// Goready is runtime.goready. +// +//go:nosplit +func Goready(gp uintptr, traceskip int) { + goready(gp, traceskip) +} + +//go:linkname goready runtime.goready +func goready(gp uintptr, traceskip int) + +// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go. +const ( + WaitReasonSelect uint8 = 9 +) + +// Values for the traceEv argument to gopark, from Go's src/runtime/trace.go. +const ( + TraceEvGoBlockSelect byte = 24 +) + +// These functions are only used within the sync package. + +//go:linkname semacquire sync.runtime_Semacquire +func semacquire(s *uint32) + +//go:linkname semrelease sync.runtime_Semrelease +func semrelease(s *uint32, handoff bool, skipframes int) + +//go:linkname canSpin sync.runtime_canSpin +func canSpin(i int) bool + +//go:linkname doSpin sync.runtime_doSpin +func doSpin() diff --git a/pkg/sync/rwmutex_unsafe.go b/pkg/sync/rwmutex_unsafe.go index fa023f5bb..dacd7772c 100644 --- a/pkg/sync/rwmutex_unsafe.go +++ b/pkg/sync/rwmutex_unsafe.go @@ -3,11 +3,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build go1.13 -// +build !go1.17 - -// Check go:linkname function signatures when updating Go version. - // This is mostly copied from the standard library's sync/rwmutex.go. // // Happens-before relationships indicated to the race detector: @@ -23,16 +18,10 @@ import ( "unsafe" ) -//go:linkname runtimeSemacquire sync.runtime_Semacquire -func runtimeSemacquire(s *uint32) - -//go:linkname runtimeSemrelease sync.runtime_Semrelease -func runtimeSemrelease(s *uint32, handoff bool, skipframes int) - // RWMutex is identical to sync.RWMutex, but adds the DowngradeLock, // TryLock and TryRLock methods. type RWMutex struct { - // w is held if there are pending writers + // w is held by writers. // // We use CrossGoroutineMutex rather than Mutex because the lock // annotation instrumentation in Mutex will trigger false positives in @@ -78,7 +67,7 @@ func (rw *RWMutex) RLock() { } if atomic.AddInt32(&rw.readerCount, 1) < 0 { // A writer is pending, wait for it. - runtimeSemacquire(&rw.readerSem) + semacquire(&rw.readerSem) } if RaceEnabled { RaceEnable() @@ -99,7 +88,7 @@ func (rw *RWMutex) RUnlock() { // A writer is pending. if atomic.AddInt32(&rw.readerWait, -1) == 0 { // The last reader unblocks the writer. - runtimeSemrelease(&rw.writerSem, false, 0) + semrelease(&rw.writerSem, false, 0) } } if RaceEnabled { @@ -146,7 +135,7 @@ func (rw *RWMutex) Lock() { r := atomic.AddInt32(&rw.readerCount, -rwmutexMaxReaders) + rwmutexMaxReaders // Wait for active readers. if r != 0 && atomic.AddInt32(&rw.readerWait, r) != 0 { - runtimeSemacquire(&rw.writerSem) + semacquire(&rw.writerSem) } if RaceEnabled { RaceEnable() @@ -168,7 +157,7 @@ func (rw *RWMutex) Unlock() { } // Unblock blocked readers, if any. for i := 0; i < int(r); i++ { - runtimeSemrelease(&rw.readerSem, false, 0) + semrelease(&rw.readerSem, false, 0) } // Allow other writers to proceed. rw.w.Unlock() @@ -191,7 +180,7 @@ func (rw *RWMutex) DowngradeLock() { // Unblock blocked readers, if any. Note that this loop starts as 1 since r // includes this goroutine. for i := 1; i < int(r); i++ { - runtimeSemrelease(&rw.readerSem, false, 0) + semrelease(&rw.readerSem, false, 0) } // Allow other writers to proceed to rw.w.Lock(). Note that they will still // block on rw.writerSem since at least this reader exists, such that diff --git a/pkg/syncevent/BUILD b/pkg/syncevent/BUILD index 0500a22cf..42c553308 100644 --- a/pkg/syncevent/BUILD +++ b/pkg/syncevent/BUILD @@ -9,10 +9,6 @@ go_library( "receiver.go", "source.go", "syncevent.go", - "waiter_amd64.s", - "waiter_arm64.s", - "waiter_asm_unsafe.go", - "waiter_noasm_unsafe.go", "waiter_unsafe.go", ], visibility = ["//:sandbox"], diff --git a/pkg/syncevent/waiter_asm_unsafe.go b/pkg/syncevent/waiter_asm_unsafe.go deleted file mode 100644 index 19d6b0b15..000000000 --- a/pkg/syncevent/waiter_asm_unsafe.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 arm64 - -package syncevent - -import ( - "unsafe" -) - -// See waiter_noasm_unsafe.go for a description of waiterUnlock. -func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool diff --git a/pkg/syncevent/waiter_noasm_unsafe.go b/pkg/syncevent/waiter_noasm_unsafe.go deleted file mode 100644 index 0f74a689c..000000000 --- a/pkg/syncevent/waiter_noasm_unsafe.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// waiterUnlock is called from g0, so when the race detector is enabled, -// waiterUnlock must be implemented in assembly since no race context is -// available. -// -// +build !race -// +build !amd64,!arm64 - -package syncevent - -import ( - "sync/atomic" - "unsafe" -) - -// waiterUnlock is the "unlock function" passed to runtime.gopark by -// Waiter.Wait*. wg is &Waiter.g, and g is a pointer to the calling runtime.g. -// waiterUnlock returns true if Waiter.Wait should sleep and false if sleeping -// should be aborted. -// -//go:nosplit -func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool { - // The only way this CAS can fail is if a call to Waiter.NotifyPending() - // has replaced *wg with nil, in which case we should not sleep. - return atomic.CompareAndSwapPointer(wg, (unsafe.Pointer)(&preparingG), ptr) -} diff --git a/pkg/syncevent/waiter_unsafe.go b/pkg/syncevent/waiter_unsafe.go index 518f18479..b6ed2852d 100644 --- a/pkg/syncevent/waiter_unsafe.go +++ b/pkg/syncevent/waiter_unsafe.go @@ -12,11 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.11 -// +build !go1.17 - -// Check go:linkname function signatures when updating Go version. - package syncevent import ( @@ -26,17 +21,6 @@ import ( "gvisor.dev/gvisor/pkg/sync" ) -//go:linkname gopark runtime.gopark -func gopark(unlockf func(unsafe.Pointer, *unsafe.Pointer) bool, wg *unsafe.Pointer, reason uint8, traceEv byte, traceskip int) - -//go:linkname goready runtime.goready -func goready(g unsafe.Pointer, traceskip int) - -const ( - waitReasonSelect = 9 // Go: src/runtime/runtime2.go - traceEvGoBlockSelect = 24 // Go: src/runtime/trace.go -) - // Waiter allows a goroutine to block on pending events received by a Receiver. // // Waiter.Init() must be called before first use. @@ -45,20 +29,19 @@ type Waiter struct { // g is one of: // - // - nil: No goroutine is blocking in Wait. + // - 0: No goroutine is blocking in Wait. // - // - &preparingG: A goroutine is in Wait preparing to sleep, but hasn't yet + // - preparingG: A goroutine is in Wait preparing to sleep, but hasn't yet // completed waiterUnlock(). Thus the wait can only be interrupted by - // replacing the value of g with nil (the G may not be in state Gwaiting - // yet, so we can't call goready.) + // replacing the value of g with 0 (the G may not be in state Gwaiting yet, + // so we can't call goready.) // // - Otherwise: g is a pointer to the runtime.g in state Gwaiting for the // goroutine blocked in Wait, which can only be woken by calling goready. - g unsafe.Pointer `state:"zerovalue"` + g uintptr `state:"zerovalue"` } -// Sentinel object for Waiter.g. -var preparingG struct{} +const preparingG = 1 // Init must be called before first use of w. func (w *Waiter) Init() { @@ -99,21 +82,29 @@ func (w *Waiter) WaitFor(es Set) Set { } // Indicate that we're preparing to go to sleep. - atomic.StorePointer(&w.g, (unsafe.Pointer)(&preparingG)) + atomic.StoreUintptr(&w.g, preparingG) // If an event is pending, abort the sleep. if p := w.r.Pending(); p&es != NoEvents { - atomic.StorePointer(&w.g, nil) + atomic.StoreUintptr(&w.g, 0) return p } // If w.g is still preparingG (i.e. w.NotifyPending() has not been - // called or has not reached atomic.SwapPointer()), go to sleep until + // called or has not reached atomic.SwapUintptr()), go to sleep until // w.NotifyPending() => goready(). - gopark(waiterUnlock, &w.g, waitReasonSelect, traceEvGoBlockSelect, 0) + sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0) } } +//go:norace +//go:nosplit +func waiterCommit(g uintptr, wg unsafe.Pointer) bool { + // The only way this CAS can fail is if a call to Waiter.NotifyPending() + // has replaced *wg with nil, in which case we should not sleep. + return sync.RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(wg), preparingG, g) +} + // Ack marks the given events as not pending. func (w *Waiter) Ack(es Set) { w.r.Ack(es) @@ -135,20 +126,20 @@ func (w *Waiter) WaitAndAckAll() Set { for { // Indicate that we're preparing to go to sleep. - atomic.StorePointer(&w.g, (unsafe.Pointer)(&preparingG)) + atomic.StoreUintptr(&w.g, preparingG) // If an event is pending, abort the sleep. if w.r.Pending() != NoEvents { if p := w.r.PendingAndAckAll(); p != NoEvents { - atomic.StorePointer(&w.g, nil) + atomic.StoreUintptr(&w.g, 0) return p } } // If w.g is still preparingG (i.e. w.NotifyPending() has not been - // called or has not reached atomic.SwapPointer()), go to sleep until + // called or has not reached atomic.SwapUintptr()), go to sleep until // w.NotifyPending() => goready(). - gopark(waiterUnlock, &w.g, waitReasonSelect, traceEvGoBlockSelect, 0) + sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0) // Check for pending events. We call PendingAndAckAll() directly now since // we only expect to be woken after events become pending. @@ -171,14 +162,14 @@ func (w *Waiter) NotifyPending() { // goroutine. NotifyPending is called after w.r.Pending() is updated, so // concurrent and future calls to w.Wait() will observe pending events and // abort sleeping. - if atomic.LoadPointer(&w.g) == nil { + if atomic.LoadUintptr(&w.g) == 0 { return } // Wake a sleeping G, or prevent a G that is preparing to sleep from doing // so. Swap is needed here to ensure that only one call to NotifyPending // calls goready. - if g := atomic.SwapPointer(&w.g, nil); g != nil && g != (unsafe.Pointer)(&preparingG) { - goready(g, 0) + if g := atomic.SwapUintptr(&w.g, 0); g > preparingG { + sync.Goready(g, 0) } } |