diff options
Diffstat (limited to 'pkg/sync')
| -rw-r--r-- | pkg/sync/BUILD | 8 | ||||
| -rw-r--r-- | pkg/sync/goyield_go113_unsafe.go (renamed from pkg/sync/spin_legacy_unsafe.go) | 7 | ||||
| -rw-r--r-- | pkg/sync/goyield_unsafe.go (renamed from pkg/sync/spin_unsafe.go) | 6 | ||||
| -rw-r--r-- | pkg/sync/memmove_unsafe.go | 28 | ||||
| -rw-r--r-- | pkg/sync/norace_unsafe.go | 11 | ||||
| -rw-r--r-- | pkg/sync/race_amd64.s | 33 | ||||
| -rw-r--r-- | pkg/sync/race_arm64.s | 35 | ||||
| -rw-r--r-- | pkg/sync/race_unsafe.go | 6 | ||||
| -rw-r--r-- | pkg/sync/runtime_unsafe.go | 76 | ||||
| -rw-r--r-- | pkg/sync/rwmutex_unsafe.go | 23 |
10 files changed, 172 insertions, 61 deletions
diff --git a/pkg/sync/BUILD b/pkg/sync/BUILD index 5bd4d09d1..be5bc99fc 100644 --- a/pkg/sync/BUILD +++ b/pkg/sync/BUILD @@ -33,15 +33,17 @@ go_library( "aliases.go", "checklocks_off_unsafe.go", "checklocks_on_unsafe.go", - "memmove_unsafe.go", + "goyield_go113_unsafe.go", + "goyield_unsafe.go", "mutex_unsafe.go", "nocopy.go", "norace_unsafe.go", + "race_amd64.s", + "race_arm64.s", "race_unsafe.go", + "runtime_unsafe.go", "rwmutex_unsafe.go", "seqcount.go", - "spin_legacy_unsafe.go", - "spin_unsafe.go", "sync.go", ], marshal = False, diff --git a/pkg/sync/spin_legacy_unsafe.go b/pkg/sync/goyield_go113_unsafe.go index 61fc7320e..8aee0d455 100644 --- a/pkg/sync/spin_legacy_unsafe.go +++ b/pkg/sync/goyield_go113_unsafe.go @@ -10,15 +10,8 @@ package sync import ( "runtime" - _ "unsafe" // for go:linkname ) -//go:linkname canSpin sync.runtime_canSpin -func canSpin(i int) bool - -//go:linkname doSpin sync.runtime_doSpin -func doSpin() - func goyield() { // goyield is not available until Go 1.14. runtime.Gosched() diff --git a/pkg/sync/spin_unsafe.go b/pkg/sync/goyield_unsafe.go index 18e8fc743..672ee274d 100644 --- a/pkg/sync/spin_unsafe.go +++ b/pkg/sync/goyield_unsafe.go @@ -14,11 +14,5 @@ import ( _ "unsafe" // for go:linkname ) -//go:linkname canSpin sync.runtime_canSpin -func canSpin(i int) bool - -//go:linkname doSpin sync.runtime_doSpin -func doSpin() - //go:linkname goyield runtime.goyield func goyield() diff --git a/pkg/sync/memmove_unsafe.go b/pkg/sync/memmove_unsafe.go deleted file mode 100644 index f5e630009..000000000 --- a/pkg/sync/memmove_unsafe.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build go1.12 -// +build !go1.17 - -// Check go:linkname function signatures when updating Go version. - -package sync - -import ( - "unsafe" -) - -//go:linkname memmove runtime.memmove -//go:noescape -func memmove(to, from unsafe.Pointer, n uintptr) - -// Memmove is exported for SeqAtomicLoad/SeqAtomicTryLoad<T>, which can't -// define it because go_generics can't update the go:linkname annotation. -// Furthermore, go:linkname silently doesn't work if the local name is exported -// (this is of course undocumented), which is why this indirection is -// necessary. -func Memmove(to, from unsafe.Pointer, n uintptr) { - memmove(to, from, n) -} diff --git a/pkg/sync/norace_unsafe.go b/pkg/sync/norace_unsafe.go index 006055dd6..70b5f3a5e 100644 --- a/pkg/sync/norace_unsafe.go +++ b/pkg/sync/norace_unsafe.go @@ -8,6 +8,7 @@ package sync import ( + "sync/atomic" "unsafe" ) @@ -33,3 +34,13 @@ func RaceRelease(addr unsafe.Pointer) { // RaceReleaseMerge has the same semantics as runtime.RaceReleaseMerge. func RaceReleaseMerge(addr unsafe.Pointer) { } + +// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to +// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector. +// This is necessary when implementing gopark callbacks, since no race context +// is available during their execution. +func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool { + // Use atomic.CompareAndSwapUintptr outside of race builds for + // inlinability. + return atomic.CompareAndSwapUintptr(ptr, old, new) +} diff --git a/pkg/sync/race_amd64.s b/pkg/sync/race_amd64.s new file mode 100644 index 000000000..57bc0ec79 --- /dev/null +++ b/pkg/sync/race_amd64.s @@ -0,0 +1,33 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build race +// +build amd64 + +#include "textflag.h" + +// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool +TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT,$0-25 + MOVQ ptr+0(FP), DI + MOVQ old+8(FP), AX + MOVQ new+16(FP), SI + + LOCK + CMPXCHGQ SI, 0(DI) + + SETEQ AX + MOVB AX, ret+24(FP) + + RET + diff --git a/pkg/sync/race_arm64.s b/pkg/sync/race_arm64.s new file mode 100644 index 000000000..88f091fda --- /dev/null +++ b/pkg/sync/race_arm64.s @@ -0,0 +1,35 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build race +// +build arm64 + +#include "textflag.h" + +// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool +TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT,$0-25 + MOVD ptr+0(FP), R0 + MOVD old+8(FP), R1 + MOVD new+16(FP), R1 +again: + LDAXR (R0), R3 + CMP R1, R3 + BNE ok + STLXR R2, (R0), R3 + CBNZ R3, again +ok: + CSET EQ, R0 + MOVB R0, ret+24(FP) + RET + diff --git a/pkg/sync/race_unsafe.go b/pkg/sync/race_unsafe.go index 31d8fa9a6..59985c270 100644 --- a/pkg/sync/race_unsafe.go +++ b/pkg/sync/race_unsafe.go @@ -39,3 +39,9 @@ func RaceRelease(addr unsafe.Pointer) { func RaceReleaseMerge(addr unsafe.Pointer) { runtime.RaceReleaseMerge(addr) } + +// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to +// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector. +// This is necessary when implementing gopark callbacks, since no race context +// is available during their execution. +func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool diff --git a/pkg/sync/runtime_unsafe.go b/pkg/sync/runtime_unsafe.go new file mode 100644 index 000000000..7ad6a4434 --- /dev/null +++ b/pkg/sync/runtime_unsafe.go @@ -0,0 +1,76 @@ +// Copyright 2020 The gVisor Authors. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.13 +// +build !go1.17 + +// Check function signatures and constants when updating Go version. + +package sync + +import ( + "unsafe" +) + +// Note that go:linkname silently doesn't work if the local name is exported, +// necessitating an indirection for exported functions. + +// Memmove is runtime.memmove, exported for SeqAtomicLoad/SeqAtomicTryLoad<T>. +// +//go:nosplit +func Memmove(to, from unsafe.Pointer, n uintptr) { + memmove(to, from, n) +} + +//go:linkname memmove runtime.memmove +//go:noescape +func memmove(to, from unsafe.Pointer, n uintptr) + +// Gopark is runtime.gopark. Gopark calls unlockf(pointer to runtime.g, lock); +// if unlockf returns true, Gopark blocks until Goready(pointer to runtime.g) +// is called. unlockf and its callees must be nosplit and norace, since stack +// splitting and race context are not available where it is called. +// +//go:nosplit +func Gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceEv byte, traceskip int) { + gopark(unlockf, lock, reason, traceEv, traceskip) +} + +//go:linkname gopark runtime.gopark +func gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceEv byte, traceskip int) + +// Goready is runtime.goready. +// +//go:nosplit +func Goready(gp uintptr, traceskip int) { + goready(gp, traceskip) +} + +//go:linkname goready runtime.goready +func goready(gp uintptr, traceskip int) + +// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go. +const ( + WaitReasonSelect uint8 = 9 +) + +// Values for the traceEv argument to gopark, from Go's src/runtime/trace.go. +const ( + TraceEvGoBlockSelect byte = 24 +) + +// These functions are only used within the sync package. + +//go:linkname semacquire sync.runtime_Semacquire +func semacquire(s *uint32) + +//go:linkname semrelease sync.runtime_Semrelease +func semrelease(s *uint32, handoff bool, skipframes int) + +//go:linkname canSpin sync.runtime_canSpin +func canSpin(i int) bool + +//go:linkname doSpin sync.runtime_doSpin +func doSpin() diff --git a/pkg/sync/rwmutex_unsafe.go b/pkg/sync/rwmutex_unsafe.go index fa023f5bb..dacd7772c 100644 --- a/pkg/sync/rwmutex_unsafe.go +++ b/pkg/sync/rwmutex_unsafe.go @@ -3,11 +3,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build go1.13 -// +build !go1.17 - -// Check go:linkname function signatures when updating Go version. - // This is mostly copied from the standard library's sync/rwmutex.go. // // Happens-before relationships indicated to the race detector: @@ -23,16 +18,10 @@ import ( "unsafe" ) -//go:linkname runtimeSemacquire sync.runtime_Semacquire -func runtimeSemacquire(s *uint32) - -//go:linkname runtimeSemrelease sync.runtime_Semrelease -func runtimeSemrelease(s *uint32, handoff bool, skipframes int) - // RWMutex is identical to sync.RWMutex, but adds the DowngradeLock, // TryLock and TryRLock methods. type RWMutex struct { - // w is held if there are pending writers + // w is held by writers. // // We use CrossGoroutineMutex rather than Mutex because the lock // annotation instrumentation in Mutex will trigger false positives in @@ -78,7 +67,7 @@ func (rw *RWMutex) RLock() { } if atomic.AddInt32(&rw.readerCount, 1) < 0 { // A writer is pending, wait for it. - runtimeSemacquire(&rw.readerSem) + semacquire(&rw.readerSem) } if RaceEnabled { RaceEnable() @@ -99,7 +88,7 @@ func (rw *RWMutex) RUnlock() { // A writer is pending. if atomic.AddInt32(&rw.readerWait, -1) == 0 { // The last reader unblocks the writer. - runtimeSemrelease(&rw.writerSem, false, 0) + semrelease(&rw.writerSem, false, 0) } } if RaceEnabled { @@ -146,7 +135,7 @@ func (rw *RWMutex) Lock() { r := atomic.AddInt32(&rw.readerCount, -rwmutexMaxReaders) + rwmutexMaxReaders // Wait for active readers. if r != 0 && atomic.AddInt32(&rw.readerWait, r) != 0 { - runtimeSemacquire(&rw.writerSem) + semacquire(&rw.writerSem) } if RaceEnabled { RaceEnable() @@ -168,7 +157,7 @@ func (rw *RWMutex) Unlock() { } // Unblock blocked readers, if any. for i := 0; i < int(r); i++ { - runtimeSemrelease(&rw.readerSem, false, 0) + semrelease(&rw.readerSem, false, 0) } // Allow other writers to proceed. rw.w.Unlock() @@ -191,7 +180,7 @@ func (rw *RWMutex) DowngradeLock() { // Unblock blocked readers, if any. Note that this loop starts as 1 since r // includes this goroutine. for i := 1; i < int(r); i++ { - runtimeSemrelease(&rw.readerSem, false, 0) + semrelease(&rw.readerSem, false, 0) } // Allow other writers to proceed to rw.w.Lock(). Note that they will still // block on rw.writerSem since at least this reader exists, such that |