4 files changed, 593 insertions, 118 deletions
diff --git a/pkg/sentry/fsimpl/devpts/BUILD b/pkg/sentry/fsimpl/devpts/BUILD
deleted file mode 100644
index 84baaac66..000000000
--- a/pkg/sentry/fsimpl/devpts/BUILD
+++ /dev/null
@@ -1,62 +0,0 @@
-load("//tools:defs.bzl", "go_library", "go_test")
-load("//tools/go_generics:defs.bzl", "go_template_instance")
-
-licenses(["notice"])
-
-go_template_instance(
-    name = "root_inode_refs",
-    out = "root_inode_refs.go",
-    package = "devpts",
-    prefix = "rootInode",
-    template = "//pkg/refs_vfs2:refs_template",
-    types = {
-        "T": "rootInode",
-    },
-)
-
-go_library(
-    name = "devpts",
-    srcs = [
-        "devpts.go",
-        "line_discipline.go",
-        "master.go",
-        "queue.go",
-        "replica.go",
-        "root_inode_refs.go",
-        "terminal.go",
-    ],
-    visibility = ["//pkg/sentry:internal"],
-    deps = [
-        "//pkg/abi/linux",
-        "//pkg/context",
-        "//pkg/log",
-        "//pkg/marshal",
-        "//pkg/marshal/primitive",
-        "//pkg/refs",
-        "//pkg/safemem",
-        "//pkg/sentry/arch",
-        "//pkg/sentry/fs",
-        "//pkg/sentry/fs/lock",
-        "//pkg/sentry/fsimpl/kernfs",
-        "//pkg/sentry/kernel",
-        "//pkg/sentry/kernel/auth",
-        "//pkg/sentry/unimpl",
-        "//pkg/sentry/vfs",
-        "//pkg/sync",
-        "//pkg/syserror",
-        "//pkg/usermem",
-        "//pkg/waiter",
-    ],
-)
-
-go_test(
-    name = "devpts_test",
-    size = "small",
-    srcs = ["devpts_test.go"],
-    library = ":devpts",
-    deps = [
-        "//pkg/abi/linux",
-        "//pkg/sentry/contexttest",
-        "//pkg/usermem",
-    ],
-)
diff --git a/pkg/sentry/fsimpl/devpts/devpts_state_autogen.go b/pkg/sentry/fsimpl/devpts/devpts_state_autogen.go
new file mode 100644
index 000000000..dd9e03e42
--- /dev/null
+++ b/pkg/sentry/fsimpl/devpts/devpts_state_autogen.go
@@ -0,0 +1,475 @@
+// automatically generated by stateify.
+
+package devpts
+
+import (
+	"gvisor.dev/gvisor/pkg/state"
+)
+
+func (fstype *FilesystemType) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.FilesystemType"
+}
+
+func (fstype *FilesystemType) StateFields() []string {
+	return []string{
+		"initErr",
+		"fs",
+		"root",
+	}
+}
+
+func (fstype *FilesystemType) beforeSave() {}
+
+func (fstype *FilesystemType) StateSave(stateSinkObject state.Sink) {
+	fstype.beforeSave()
+	stateSinkObject.Save(0, &fstype.initErr)
+	stateSinkObject.Save(1, &fstype.fs)
+	stateSinkObject.Save(2, &fstype.root)
+}
+
+func (fstype *FilesystemType) afterLoad() {}
+
+func (fstype *FilesystemType) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &fstype.initErr)
+	stateSourceObject.Load(1, &fstype.fs)
+	stateSourceObject.Load(2, &fstype.root)
+}
+
+func (fs *filesystem) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.filesystem"
+}
+
+func (fs *filesystem) StateFields() []string {
+	return []string{
+		"Filesystem",
+		"devMinor",
+	}
+}
+
+func (fs *filesystem) beforeSave() {}
+
+func (fs *filesystem) StateSave(stateSinkObject state.Sink) {
+	fs.beforeSave()
+	stateSinkObject.Save(0, &fs.Filesystem)
+	stateSinkObject.Save(1, &fs.devMinor)
+}
+
+func (fs *filesystem) afterLoad() {}
+
+func (fs *filesystem) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &fs.Filesystem)
+	stateSourceObject.Load(1, &fs.devMinor)
+}
+
+func (i *rootInode) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.rootInode"
+}
+
+func (i *rootInode) StateFields() []string {
+	return []string{
+		"implStatFS",
+		"InodeAlwaysValid",
+		"InodeAttrs",
+		"InodeDirectoryNoNewChildren",
+		"InodeNotSymlink",
+		"InodeTemporary",
+		"OrderedChildren",
+		"rootInodeRefs",
+		"locks",
+		"master",
+		"replicas",
+		"nextIdx",
+	}
+}
+
+func (i *rootInode) beforeSave() {}
+
+func (i *rootInode) StateSave(stateSinkObject state.Sink) {
+	i.beforeSave()
+	stateSinkObject.Save(0, &i.implStatFS)
+	stateSinkObject.Save(1, &i.InodeAlwaysValid)
+	stateSinkObject.Save(2, &i.InodeAttrs)
+	stateSinkObject.Save(3, &i.InodeDirectoryNoNewChildren)
+	stateSinkObject.Save(4, &i.InodeNotSymlink)
+	stateSinkObject.Save(5, &i.InodeTemporary)
+	stateSinkObject.Save(6, &i.OrderedChildren)
+	stateSinkObject.Save(7, &i.rootInodeRefs)
+	stateSinkObject.Save(8, &i.locks)
+	stateSinkObject.Save(9, &i.master)
+	stateSinkObject.Save(10, &i.replicas)
+	stateSinkObject.Save(11, &i.nextIdx)
+}
+
+func (i *rootInode) afterLoad() {}
+
+func (i *rootInode) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &i.implStatFS)
+	stateSourceObject.Load(1, &i.InodeAlwaysValid)
+	stateSourceObject.Load(2, &i.InodeAttrs)
+	stateSourceObject.Load(3, &i.InodeDirectoryNoNewChildren)
+	stateSourceObject.Load(4, &i.InodeNotSymlink)
+	stateSourceObject.Load(5, &i.InodeTemporary)
+	stateSourceObject.Load(6, &i.OrderedChildren)
+	stateSourceObject.Load(7, &i.rootInodeRefs)
+	stateSourceObject.Load(8, &i.locks)
+	stateSourceObject.Load(9, &i.master)
+	stateSourceObject.Load(10, &i.replicas)
+	stateSourceObject.Load(11, &i.nextIdx)
+}
+
+func (i *implStatFS) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.implStatFS"
+}
+
+func (i *implStatFS) StateFields() []string {
+	return []string{}
+}
+
+func (i *implStatFS) beforeSave() {}
+
+func (i *implStatFS) StateSave(stateSinkObject state.Sink) {
+	i.beforeSave()
+}
+
+func (i *implStatFS) afterLoad() {}
+
+func (i *implStatFS) StateLoad(stateSourceObject state.Source) {
+}
+
+func (l *lineDiscipline) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.lineDiscipline"
+}
+
+func (l *lineDiscipline) StateFields() []string {
+	return []string{
+		"size",
+		"inQueue",
+		"outQueue",
+		"termios",
+		"column",
+	}
+}
+
+func (l *lineDiscipline) beforeSave() {}
+
+func (l *lineDiscipline) StateSave(stateSinkObject state.Sink) {
+	l.beforeSave()
+	if !state.IsZeroValue(&l.masterWaiter) {
+		state.Failf("masterWaiter is %#v, expected zero", &l.masterWaiter)
+	}
+	if !state.IsZeroValue(&l.replicaWaiter) {
+		state.Failf("replicaWaiter is %#v, expected zero", &l.replicaWaiter)
+	}
+	stateSinkObject.Save(0, &l.size)
+	stateSinkObject.Save(1, &l.inQueue)
+	stateSinkObject.Save(2, &l.outQueue)
+	stateSinkObject.Save(3, &l.termios)
+	stateSinkObject.Save(4, &l.column)
+}
+
+func (l *lineDiscipline) afterLoad() {}
+
+func (l *lineDiscipline) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &l.size)
+	stateSourceObject.Load(1, &l.inQueue)
+	stateSourceObject.Load(2, &l.outQueue)
+	stateSourceObject.Load(3, &l.termios)
+	stateSourceObject.Load(4, &l.column)
+}
+
+func (o *outputQueueTransformer) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.outputQueueTransformer"
+}
+
+func (o *outputQueueTransformer) StateFields() []string {
+	return []string{}
+}
+
+func (o *outputQueueTransformer) beforeSave() {}
+
+func (o *outputQueueTransformer) StateSave(stateSinkObject state.Sink) {
+	o.beforeSave()
+}
+
+func (o *outputQueueTransformer) afterLoad() {}
+
+func (o *outputQueueTransformer) StateLoad(stateSourceObject state.Source) {
+}
+
+func (i *inputQueueTransformer) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.inputQueueTransformer"
+}
+
+func (i *inputQueueTransformer) StateFields() []string {
+	return []string{}
+}
+
+func (i *inputQueueTransformer) beforeSave() {}
+
+func (i *inputQueueTransformer) StateSave(stateSinkObject state.Sink) {
+	i.beforeSave()
+}
+
+func (i *inputQueueTransformer) afterLoad() {}
+
+func (i *inputQueueTransformer) StateLoad(stateSourceObject state.Source) {
+}
+
+func (mi *masterInode) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.masterInode"
+}
+
+func (mi *masterInode) StateFields() []string {
+	return []string{
+		"implStatFS",
+		"InodeAttrs",
+		"InodeNoopRefCount",
+		"InodeNotDirectory",
+		"InodeNotSymlink",
+		"locks",
+		"root",
+	}
+}
+
+func (mi *masterInode) beforeSave() {}
+
+func (mi *masterInode) StateSave(stateSinkObject state.Sink) {
+	mi.beforeSave()
+	stateSinkObject.Save(0, &mi.implStatFS)
+	stateSinkObject.Save(1, &mi.InodeAttrs)
+	stateSinkObject.Save(2, &mi.InodeNoopRefCount)
+	stateSinkObject.Save(3, &mi.InodeNotDirectory)
+	stateSinkObject.Save(4, &mi.InodeNotSymlink)
+	stateSinkObject.Save(5, &mi.locks)
+	stateSinkObject.Save(6, &mi.root)
+}
+
+func (mi *masterInode) afterLoad() {}
+
+func (mi *masterInode) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &mi.implStatFS)
+	stateSourceObject.Load(1, &mi.InodeAttrs)
+	stateSourceObject.Load(2, &mi.InodeNoopRefCount)
+	stateSourceObject.Load(3, &mi.InodeNotDirectory)
+	stateSourceObject.Load(4, &mi.InodeNotSymlink)
+	stateSourceObject.Load(5, &mi.locks)
+	stateSourceObject.Load(6, &mi.root)
+}
+
+func (mfd *masterFileDescription) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.masterFileDescription"
+}
+
+func (mfd *masterFileDescription) StateFields() []string {
+	return []string{
+		"vfsfd",
+		"FileDescriptionDefaultImpl",
+		"LockFD",
+		"inode",
+		"t",
+	}
+}
+
+func (mfd *masterFileDescription) beforeSave() {}
+
+func (mfd *masterFileDescription) StateSave(stateSinkObject state.Sink) {
+	mfd.beforeSave()
+	stateSinkObject.Save(0, &mfd.vfsfd)
+	stateSinkObject.Save(1, &mfd.FileDescriptionDefaultImpl)
+	stateSinkObject.Save(2, &mfd.LockFD)
+	stateSinkObject.Save(3, &mfd.inode)
+	stateSinkObject.Save(4, &mfd.t)
+}
+
+func (mfd *masterFileDescription) afterLoad() {}
+
+func (mfd *masterFileDescription) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &mfd.vfsfd)
+	stateSourceObject.Load(1, &mfd.FileDescriptionDefaultImpl)
+	stateSourceObject.Load(2, &mfd.LockFD)
+	stateSourceObject.Load(3, &mfd.inode)
+	stateSourceObject.Load(4, &mfd.t)
+}
+
+func (q *queue) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.queue"
+}
+
+func (q *queue) StateFields() []string {
+	return []string{
+		"readBuf",
+		"waitBuf",
+		"waitBufLen",
+		"readable",
+		"transformer",
+	}
+}
+
+func (q *queue) beforeSave() {}
+
+func (q *queue) StateSave(stateSinkObject state.Sink) {
+	q.beforeSave()
+	stateSinkObject.Save(0, &q.readBuf)
+	stateSinkObject.Save(1, &q.waitBuf)
+	stateSinkObject.Save(2, &q.waitBufLen)
+	stateSinkObject.Save(3, &q.readable)
+	stateSinkObject.Save(4, &q.transformer)
+}
+
+func (q *queue) afterLoad() {}
+
+func (q *queue) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &q.readBuf)
+	stateSourceObject.Load(1, &q.waitBuf)
+	stateSourceObject.Load(2, &q.waitBufLen)
+	stateSourceObject.Load(3, &q.readable)
+	stateSourceObject.Load(4, &q.transformer)
+}
+
+func (ri *replicaInode) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.replicaInode"
+}
+
+func (ri *replicaInode) StateFields() []string {
+	return []string{
+		"implStatFS",
+		"InodeAttrs",
+		"InodeNoopRefCount",
+		"InodeNotDirectory",
+		"InodeNotSymlink",
+		"locks",
+		"root",
+		"t",
+	}
+}
+
+func (ri *replicaInode) beforeSave() {}
+
+func (ri *replicaInode) StateSave(stateSinkObject state.Sink) {
+	ri.beforeSave()
+	stateSinkObject.Save(0, &ri.implStatFS)
+	stateSinkObject.Save(1, &ri.InodeAttrs)
+	stateSinkObject.Save(2, &ri.InodeNoopRefCount)
+	stateSinkObject.Save(3, &ri.InodeNotDirectory)
+	stateSinkObject.Save(4, &ri.InodeNotSymlink)
+	stateSinkObject.Save(5, &ri.locks)
+	stateSinkObject.Save(6, &ri.root)
+	stateSinkObject.Save(7, &ri.t)
+}
+
+func (ri *replicaInode) afterLoad() {}
+
+func (ri *replicaInode) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &ri.implStatFS)
+	stateSourceObject.Load(1, &ri.InodeAttrs)
+	stateSourceObject.Load(2, &ri.InodeNoopRefCount)
+	stateSourceObject.Load(3, &ri.InodeNotDirectory)
+	stateSourceObject.Load(4, &ri.InodeNotSymlink)
+	stateSourceObject.Load(5, &ri.locks)
+	stateSourceObject.Load(6, &ri.root)
+	stateSourceObject.Load(7, &ri.t)
+}
+
+func (rfd *replicaFileDescription) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.replicaFileDescription"
+}
+
+func (rfd *replicaFileDescription) StateFields() []string {
+	return []string{
+		"vfsfd",
+		"FileDescriptionDefaultImpl",
+		"LockFD",
+		"inode",
+	}
+}
+
+func (rfd *replicaFileDescription) beforeSave() {}
+
+func (rfd *replicaFileDescription) StateSave(stateSinkObject state.Sink) {
+	rfd.beforeSave()
+	stateSinkObject.Save(0, &rfd.vfsfd)
+	stateSinkObject.Save(1, &rfd.FileDescriptionDefaultImpl)
+	stateSinkObject.Save(2, &rfd.LockFD)
+	stateSinkObject.Save(3, &rfd.inode)
+}
+
+func (rfd *replicaFileDescription) afterLoad() {}
+
+func (rfd *replicaFileDescription) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &rfd.vfsfd)
+	stateSourceObject.Load(1, &rfd.FileDescriptionDefaultImpl)
+	stateSourceObject.Load(2, &rfd.LockFD)
+	stateSourceObject.Load(3, &rfd.inode)
+}
+
+func (r *rootInodeRefs) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.rootInodeRefs"
+}
+
+func (r *rootInodeRefs) StateFields() []string {
+	return []string{
+		"refCount",
+	}
+}
+
+func (r *rootInodeRefs) beforeSave() {}
+
+func (r *rootInodeRefs) StateSave(stateSinkObject state.Sink) {
+	r.beforeSave()
+	stateSinkObject.Save(0, &r.refCount)
+}
+
+func (r *rootInodeRefs) afterLoad() {}
+
+func (r *rootInodeRefs) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &r.refCount)
+}
+
+func (tm *Terminal) StateTypeName() string {
+	return "pkg/sentry/fsimpl/devpts.Terminal"
+}
+
+func (tm *Terminal) StateFields() []string {
+	return []string{
+		"n",
+		"ld",
+		"masterKTTY",
+		"replicaKTTY",
+	}
+}
+
+func (tm *Terminal) beforeSave() {}
+
+func (tm *Terminal) StateSave(stateSinkObject state.Sink) {
+	tm.beforeSave()
+	stateSinkObject.Save(0, &tm.n)
+	stateSinkObject.Save(1, &tm.ld)
+	stateSinkObject.Save(2, &tm.masterKTTY)
+	stateSinkObject.Save(3, &tm.replicaKTTY)
+}
+
+func (tm *Terminal) afterLoad() {}
+
+func (tm *Terminal) StateLoad(stateSourceObject state.Source) {
+	stateSourceObject.Load(0, &tm.n)
+	stateSourceObject.Load(1, &tm.ld)
+	stateSourceObject.Load(2, &tm.masterKTTY)
+	stateSourceObject.Load(3, &tm.replicaKTTY)
+}
+
+func init() {
+	state.Register((*FilesystemType)(nil))
+	state.Register((*filesystem)(nil))
+	state.Register((*rootInode)(nil))
+	state.Register((*implStatFS)(nil))
+	state.Register((*lineDiscipline)(nil))
+	state.Register((*outputQueueTransformer)(nil))
+	state.Register((*inputQueueTransformer)(nil))
+	state.Register((*masterInode)(nil))
+	state.Register((*masterFileDescription)(nil))
+	state.Register((*queue)(nil))
+	state.Register((*replicaInode)(nil))
+	state.Register((*replicaFileDescription)(nil))
+	state.Register((*rootInodeRefs)(nil))
+	state.Register((*Terminal)(nil))
+}
diff --git a/pkg/sentry/fsimpl/devpts/devpts_test.go b/pkg/sentry/fsimpl/devpts/devpts_test.go
deleted file mode 100644
index 448390cfe..000000000
--- a/pkg/sentry/fsimpl/devpts/devpts_test.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2018 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package devpts
-
-import (
-	"testing"
-
-	"gvisor.dev/gvisor/pkg/abi/linux"
-	"gvisor.dev/gvisor/pkg/sentry/contexttest"
-	"gvisor.dev/gvisor/pkg/usermem"
-)
-
-func TestSimpleMasterToReplica(t *testing.T) {
-	ld := newLineDiscipline(linux.DefaultReplicaTermios)
-	ctx := contexttest.Context(t)
-	inBytes := []byte("hello, tty\n")
-	src := usermem.BytesIOSequence(inBytes)
-	outBytes := make([]byte, 32)
-	dst := usermem.BytesIOSequence(outBytes)
-
-	// Write to the input queue.
-	nw, err := ld.inputQueueWrite(ctx, src)
-	if err != nil {
-		t.Fatalf("error writing to input queue: %v", err)
-	}
-	if nw != int64(len(inBytes)) {
-		t.Fatalf("wrote wrong length: got %d, want %d", nw, len(inBytes))
-	}
-
-	// Read from the input queue.
-	nr, err := ld.inputQueueRead(ctx, dst)
-	if err != nil {
-		t.Fatalf("error reading from input queue: %v", err)
-	}
-	if nr != int64(len(inBytes)) {
-		t.Fatalf("read wrong length: got %d, want %d", nr, len(inBytes))
-	}
-
-	outStr := string(outBytes[:nr])
-	inStr := string(inBytes)
-	if outStr != inStr {
-		t.Fatalf("written and read strings do not match: got %q, want %q", outStr, inStr)
-	}
-}
diff --git a/pkg/sentry/fsimpl/devpts/root_inode_refs.go b/pkg/sentry/fsimpl/devpts/root_inode_refs.go
new file mode 100644
index 000000000..051801202
--- /dev/null
+++ b/pkg/sentry/fsimpl/devpts/root_inode_refs.go
@@ -0,0 +1,118 @@
+package devpts
+
+import (
+	"fmt"
+	"runtime"
+	"sync/atomic"
+
+	"gvisor.dev/gvisor/pkg/log"
+	refs_vfs1 "gvisor.dev/gvisor/pkg/refs"
+)
+
+// ownerType is used to customize logging. Note that we use a pointer to T so
+// that we do not copy the entire object when passed as a format parameter.
+var rootInodeownerType *rootInode
+
+// Refs implements refs.RefCounter. It keeps a reference count using atomic
+// operations and calls the destructor when the count reaches zero.
+//
+// Note that the number of references is actually refCount + 1 so that a default
+// zero-value Refs object contains one reference.
+//
+// TODO(gvisor.dev/issue/1486): Store stack traces when leak check is enabled in
+// a map with 16-bit hashes, and store the hash in the top 16 bits of refCount.
+// This will allow us to add stack trace information to the leak messages
+// without growing the size of Refs.
+//
+// +stateify savable
+type rootInodeRefs struct {
+	// refCount is composed of two fields:
+	//
+	//	[32-bit speculative references]:[32-bit real references]
+	//
+	// Speculative references are used for TryIncRef, to avoid a CompareAndSwap
+	// loop. See IncRef, DecRef and TryIncRef for details of how these fields are
+	// used.
+	refCount int64
+}
+
+func (r *rootInodeRefs) finalize() {
+	var note string
+	switch refs_vfs1.GetLeakMode() {
+	case refs_vfs1.NoLeakChecking:
+		return
+	case refs_vfs1.UninitializedLeakChecking:
+		note = "(Leak checker uninitialized): "
+	}
+	if n := r.ReadRefs(); n != 0 {
+		log.Warningf("%sRefs %p owned by %T garbage collected with ref count of %d (want 0)", note, r, rootInodeownerType, n)
+	}
+}
+
+// EnableLeakCheck checks for reference leaks when Refs gets garbage collected.
+func (r *rootInodeRefs) EnableLeakCheck() {
+	if refs_vfs1.GetLeakMode() != refs_vfs1.NoLeakChecking {
+		runtime.SetFinalizer(r, (*rootInodeRefs).finalize)
+	}
+}
+
+// ReadRefs returns the current number of references. The returned count is
+// inherently racy and is unsafe to use without external synchronization.
+func (r *rootInodeRefs) ReadRefs() int64 {
+
+	return atomic.LoadInt64(&r.refCount) + 1
+}
+
+// IncRef implements refs.RefCounter.IncRef.
+//
+//go:nosplit
+func (r *rootInodeRefs) IncRef() {
+	if v := atomic.AddInt64(&r.refCount, 1); v <= 0 {
+		panic(fmt.Sprintf("Incrementing non-positive ref count %p owned by %T", r, rootInodeownerType))
+	}
+}
+
+// TryIncRef implements refs.RefCounter.TryIncRef.
+//
+// To do this safely without a loop, a speculative reference is first acquired
+// on the object. This allows multiple concurrent TryIncRef calls to distinguish
+// other TryIncRef calls from genuine references held.
+//
+//go:nosplit
+func (r *rootInodeRefs) TryIncRef() bool {
+	const speculativeRef = 1 << 32
+	v := atomic.AddInt64(&r.refCount, speculativeRef)
+	if int32(v) < 0 {
+
+		atomic.AddInt64(&r.refCount, -speculativeRef)
+		return false
+	}
+
+	atomic.AddInt64(&r.refCount, -speculativeRef+1)
+	return true
+}
+
+// DecRef implements refs.RefCounter.DecRef.
+//
+// Note that speculative references are counted here. Since they were added
+// prior to real references reaching zero, they will successfully convert to
+// real references. In other words, we see speculative references only in the
+// following case:
+//
+//	A: TryIncRef [speculative increase => sees non-negative references]
+//	B: DecRef [real decrease]
+//	A: TryIncRef [transform speculative to real]
+//
+//go:nosplit
+func (r *rootInodeRefs) DecRef(destroy func()) {
+	switch v := atomic.AddInt64(&r.refCount, -1); {
+	case v < -1:
+		panic(fmt.Sprintf("Decrementing non-positive ref count %p, owned by %T", r, rootInodeownerType))
+
+	case v == -1:
+
+		if destroy != nil {
+			destroy()
+		}
+	}
+}