1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
package mm
import (
"fmt"
"sync/atomic"
"gvisor.dev/gvisor/pkg/refsvfs2"
)
// ownerType is used to customize logging. Note that we use a pointer to T so
// that we do not copy the entire object when passed as a format parameter.
var aioMappableownerType *aioMappable
// Refs implements refs.RefCounter. It keeps a reference count using atomic
// operations and calls the destructor when the count reaches zero.
//
// Note that the number of references is actually refCount + 1 so that a default
// zero-value Refs object contains one reference.
//
// +stateify savable
type aioMappableRefs struct {
// refCount is composed of two fields:
//
// [32-bit speculative references]:[32-bit real references]
//
// Speculative references are used for TryIncRef, to avoid a CompareAndSwap
// loop. See IncRef, DecRef and TryIncRef for details of how these fields are
// used.
refCount int64
}
// EnableLeakCheck enables reference leak checking on r.
func (r *aioMappableRefs) EnableLeakCheck() {
if refsvfs2.LeakCheckEnabled() {
refsvfs2.Register(r, fmt.Sprintf("%T", aioMappableownerType))
}
}
// LeakMessage implements refsvfs2.CheckedObject.LeakMessage.
func (r *aioMappableRefs) LeakMessage() string {
return fmt.Sprintf("%T %p: reference count of %d instead of 0", aioMappableownerType, r, r.ReadRefs())
}
// ReadRefs returns the current number of references. The returned count is
// inherently racy and is unsafe to use without external synchronization.
func (r *aioMappableRefs) ReadRefs() int64 {
return atomic.LoadInt64(&r.refCount) + 1
}
// IncRef implements refs.RefCounter.IncRef.
//
//go:nosplit
func (r *aioMappableRefs) IncRef() {
if v := atomic.AddInt64(&r.refCount, 1); v <= 0 {
panic(fmt.Sprintf("Incrementing non-positive count %p on %T", r, aioMappableownerType))
}
}
// TryIncRef implements refs.RefCounter.TryIncRef.
//
// To do this safely without a loop, a speculative reference is first acquired
// on the object. This allows multiple concurrent TryIncRef calls to distinguish
// other TryIncRef calls from genuine references held.
//
//go:nosplit
func (r *aioMappableRefs) TryIncRef() bool {
const speculativeRef = 1 << 32
v := atomic.AddInt64(&r.refCount, speculativeRef)
if int32(v) < 0 {
atomic.AddInt64(&r.refCount, -speculativeRef)
return false
}
atomic.AddInt64(&r.refCount, -speculativeRef+1)
return true
}
// DecRef implements refs.RefCounter.DecRef.
//
// Note that speculative references are counted here. Since they were added
// prior to real references reaching zero, they will successfully convert to
// real references. In other words, we see speculative references only in the
// following case:
//
// A: TryIncRef [speculative increase => sees non-negative references]
// B: DecRef [real decrease]
// A: TryIncRef [transform speculative to real]
//
//go:nosplit
func (r *aioMappableRefs) DecRef(destroy func()) {
switch v := atomic.AddInt64(&r.refCount, -1); {
case v < -1:
panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %T", r, aioMappableownerType))
case v == -1:
if refsvfs2.LeakCheckEnabled() {
refsvfs2.Unregister(r, fmt.Sprintf("%T", aioMappableownerType))
}
if destroy != nil {
destroy()
}
}
}
func (r *aioMappableRefs) afterLoad() {
if refsvfs2.LeakCheckEnabled() && r.ReadRefs() > 0 {
r.EnableLeakCheck()
}
}
|
