diff options
Diffstat (limited to 'pkg/syncevent')
| -rw-r--r-- | pkg/syncevent/BUILD | 4 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_amd64.s | 32 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_arm64.s | 34 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_asm_unsafe.go | 24 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_noasm_unsafe.go | 39 | ||||
| -rw-r--r-- | pkg/syncevent/waiter_unsafe.go | 59 |
6 files changed, 25 insertions, 167 deletions
diff --git a/pkg/syncevent/BUILD b/pkg/syncevent/BUILD index 0500a22cf..42c553308 100644 --- a/pkg/syncevent/BUILD +++ b/pkg/syncevent/BUILD @@ -9,10 +9,6 @@ go_library( "receiver.go", "source.go", "syncevent.go", - "waiter_amd64.s", - "waiter_arm64.s", - "waiter_asm_unsafe.go", - "waiter_noasm_unsafe.go", "waiter_unsafe.go", ], visibility = ["//:sandbox"], diff --git a/pkg/syncevent/waiter_amd64.s b/pkg/syncevent/waiter_amd64.s deleted file mode 100644 index 5e216b045..000000000 --- a/pkg/syncevent/waiter_amd64.s +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "textflag.h" - -// See waiter_noasm_unsafe.go for a description of waiterUnlock. -// -// func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool -TEXT ·waiterUnlock(SB),NOSPLIT,$0-24 - MOVQ ptr+0(FP), DI - MOVQ wg+8(FP), SI - - MOVQ $·preparingG(SB), AX - LOCK - CMPXCHGQ DI, 0(SI) - - SETEQ AX - MOVB AX, ret+16(FP) - - RET - diff --git a/pkg/syncevent/waiter_arm64.s b/pkg/syncevent/waiter_arm64.s deleted file mode 100644 index f4c06f194..000000000 --- a/pkg/syncevent/waiter_arm64.s +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "textflag.h" - -// See waiter_noasm_unsafe.go for a description of waiterUnlock. -// -// func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool -TEXT ·waiterUnlock(SB),NOSPLIT,$0-24 - MOVD wg+8(FP), R0 - MOVD $·preparingG(SB), R1 - MOVD ptr+0(FP), R2 -again: - LDAXR (R0), R3 - CMP R1, R3 - BNE ok - STLXR R2, (R0), R3 - CBNZ R3, again -ok: - CSET EQ, R0 - MOVB R0, ret+16(FP) - RET - diff --git a/pkg/syncevent/waiter_asm_unsafe.go b/pkg/syncevent/waiter_asm_unsafe.go deleted file mode 100644 index 19d6b0b15..000000000 --- a/pkg/syncevent/waiter_asm_unsafe.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 arm64 - -package syncevent - -import ( - "unsafe" -) - -// See waiter_noasm_unsafe.go for a description of waiterUnlock. -func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool diff --git a/pkg/syncevent/waiter_noasm_unsafe.go b/pkg/syncevent/waiter_noasm_unsafe.go deleted file mode 100644 index 0f74a689c..000000000 --- a/pkg/syncevent/waiter_noasm_unsafe.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// waiterUnlock is called from g0, so when the race detector is enabled, -// waiterUnlock must be implemented in assembly since no race context is -// available. -// -// +build !race -// +build !amd64,!arm64 - -package syncevent - -import ( - "sync/atomic" - "unsafe" -) - -// waiterUnlock is the "unlock function" passed to runtime.gopark by -// Waiter.Wait*. wg is &Waiter.g, and g is a pointer to the calling runtime.g. -// waiterUnlock returns true if Waiter.Wait should sleep and false if sleeping -// should be aborted. -// -//go:nosplit -func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool { - // The only way this CAS can fail is if a call to Waiter.NotifyPending() - // has replaced *wg with nil, in which case we should not sleep. - return atomic.CompareAndSwapPointer(wg, (unsafe.Pointer)(&preparingG), ptr) -} diff --git a/pkg/syncevent/waiter_unsafe.go b/pkg/syncevent/waiter_unsafe.go index 518f18479..b6ed2852d 100644 --- a/pkg/syncevent/waiter_unsafe.go +++ b/pkg/syncevent/waiter_unsafe.go @@ -12,11 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.11 -// +build !go1.17 - -// Check go:linkname function signatures when updating Go version. - package syncevent import ( @@ -26,17 +21,6 @@ import ( "gvisor.dev/gvisor/pkg/sync" ) -//go:linkname gopark runtime.gopark -func gopark(unlockf func(unsafe.Pointer, *unsafe.Pointer) bool, wg *unsafe.Pointer, reason uint8, traceEv byte, traceskip int) - -//go:linkname goready runtime.goready -func goready(g unsafe.Pointer, traceskip int) - -const ( - waitReasonSelect = 9 // Go: src/runtime/runtime2.go - traceEvGoBlockSelect = 24 // Go: src/runtime/trace.go -) - // Waiter allows a goroutine to block on pending events received by a Receiver. // // Waiter.Init() must be called before first use. @@ -45,20 +29,19 @@ type Waiter struct { // g is one of: // - // - nil: No goroutine is blocking in Wait. + // - 0: No goroutine is blocking in Wait. // - // - &preparingG: A goroutine is in Wait preparing to sleep, but hasn't yet + // - preparingG: A goroutine is in Wait preparing to sleep, but hasn't yet // completed waiterUnlock(). Thus the wait can only be interrupted by - // replacing the value of g with nil (the G may not be in state Gwaiting - // yet, so we can't call goready.) + // replacing the value of g with 0 (the G may not be in state Gwaiting yet, + // so we can't call goready.) // // - Otherwise: g is a pointer to the runtime.g in state Gwaiting for the // goroutine blocked in Wait, which can only be woken by calling goready. - g unsafe.Pointer `state:"zerovalue"` + g uintptr `state:"zerovalue"` } -// Sentinel object for Waiter.g. -var preparingG struct{} +const preparingG = 1 // Init must be called before first use of w. func (w *Waiter) Init() { @@ -99,21 +82,29 @@ func (w *Waiter) WaitFor(es Set) Set { } // Indicate that we're preparing to go to sleep. - atomic.StorePointer(&w.g, (unsafe.Pointer)(&preparingG)) + atomic.StoreUintptr(&w.g, preparingG) // If an event is pending, abort the sleep. if p := w.r.Pending(); p&es != NoEvents { - atomic.StorePointer(&w.g, nil) + atomic.StoreUintptr(&w.g, 0) return p } // If w.g is still preparingG (i.e. w.NotifyPending() has not been - // called or has not reached atomic.SwapPointer()), go to sleep until + // called or has not reached atomic.SwapUintptr()), go to sleep until // w.NotifyPending() => goready(). - gopark(waiterUnlock, &w.g, waitReasonSelect, traceEvGoBlockSelect, 0) + sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0) } } +//go:norace +//go:nosplit +func waiterCommit(g uintptr, wg unsafe.Pointer) bool { + // The only way this CAS can fail is if a call to Waiter.NotifyPending() + // has replaced *wg with nil, in which case we should not sleep. + return sync.RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(wg), preparingG, g) +} + // Ack marks the given events as not pending. func (w *Waiter) Ack(es Set) { w.r.Ack(es) @@ -135,20 +126,20 @@ func (w *Waiter) WaitAndAckAll() Set { for { // Indicate that we're preparing to go to sleep. - atomic.StorePointer(&w.g, (unsafe.Pointer)(&preparingG)) + atomic.StoreUintptr(&w.g, preparingG) // If an event is pending, abort the sleep. if w.r.Pending() != NoEvents { if p := w.r.PendingAndAckAll(); p != NoEvents { - atomic.StorePointer(&w.g, nil) + atomic.StoreUintptr(&w.g, 0) return p } } // If w.g is still preparingG (i.e. w.NotifyPending() has not been - // called or has not reached atomic.SwapPointer()), go to sleep until + // called or has not reached atomic.SwapUintptr()), go to sleep until // w.NotifyPending() => goready(). - gopark(waiterUnlock, &w.g, waitReasonSelect, traceEvGoBlockSelect, 0) + sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0) // Check for pending events. We call PendingAndAckAll() directly now since // we only expect to be woken after events become pending. @@ -171,14 +162,14 @@ func (w *Waiter) NotifyPending() { // goroutine. NotifyPending is called after w.r.Pending() is updated, so // concurrent and future calls to w.Wait() will observe pending events and // abort sleeping. - if atomic.LoadPointer(&w.g) == nil { + if atomic.LoadUintptr(&w.g) == 0 { return } // Wake a sleeping G, or prevent a G that is preparing to sleep from doing // so. Swap is needed here to ensure that only one call to NotifyPending // calls goready. - if g := atomic.SwapPointer(&w.g, nil); g != nil && g != (unsafe.Pointer)(&preparingG) { - goready(g, 0) + if g := atomic.SwapUintptr(&w.g, 0); g > preparingG { + sync.Goready(g, 0) } } |