diff options
Diffstat (limited to 'pkg/sentry/fsimpl')
36 files changed, 293 insertions, 194 deletions
diff --git a/pkg/sentry/fsimpl/devpts/devpts.go b/pkg/sentry/fsimpl/devpts/devpts.go index 0eaff9087..57580f4d4 100644 --- a/pkg/sentry/fsimpl/devpts/devpts.go +++ b/pkg/sentry/fsimpl/devpts/devpts.go @@ -111,12 +111,13 @@ func (fs *filesystem) Release(ctx context.Context) { // rootInode is the root directory inode for the devpts mounts. type rootInode struct { - rootInodeRefs + implStatFS kernfs.AlwaysValid kernfs.InodeAttrs kernfs.InodeDirectoryNoNewChildren kernfs.InodeNotSymlink kernfs.OrderedChildren + rootInodeRefs locks vfs.FileLocks @@ -240,3 +241,10 @@ func (i *rootInode) IterDirents(ctx context.Context, cb vfs.IterDirentsCallback, func (i *rootInode) DecRef(context.Context) { i.rootInodeRefs.DecRef(i.Destroy) } + +type implStatFS struct{} + +// StatFS implements kernfs.Inode.StatFS. +func (*implStatFS) StatFS(context.Context, *vfs.Filesystem) (linux.Statfs, error) { + return vfs.GenericStatFS(linux.DEVPTS_SUPER_MAGIC), nil +} diff --git a/pkg/sentry/fsimpl/devpts/master.go b/pkg/sentry/fsimpl/devpts/master.go index 3bb397f71..60feb1993 100644 --- a/pkg/sentry/fsimpl/devpts/master.go +++ b/pkg/sentry/fsimpl/devpts/master.go @@ -30,6 +30,7 @@ import ( // masterInode is the inode for the master end of the Terminal. type masterInode struct { + implStatFS kernfs.InodeAttrs kernfs.InodeNoopRefCount kernfs.InodeNotDirectory diff --git a/pkg/sentry/fsimpl/devpts/root_inode_refs.go b/pkg/sentry/fsimpl/devpts/root_inode_refs.go index 1b7090229..051801202 100644 --- a/pkg/sentry/fsimpl/devpts/root_inode_refs.go +++ b/pkg/sentry/fsimpl/devpts/root_inode_refs.go @@ -2,11 +2,11 @@ package devpts import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/devpts/slave.go b/pkg/sentry/fsimpl/devpts/slave.go index 32e4e1908..a9da7af64 100644 --- a/pkg/sentry/fsimpl/devpts/slave.go +++ b/pkg/sentry/fsimpl/devpts/slave.go @@ -29,6 +29,7 @@ import ( // slaveInode is the inode for the slave end of the Terminal. type slaveInode struct { + implStatFS kernfs.InodeAttrs kernfs.InodeNoopRefCount kernfs.InodeNotDirectory diff --git a/pkg/sentry/fsimpl/fuse/fusefs.go b/pkg/sentry/fsimpl/fuse/fusefs.go index 9717c0e15..810819ae4 100644 --- a/pkg/sentry/fsimpl/fuse/fusefs.go +++ b/pkg/sentry/fsimpl/fuse/fusefs.go @@ -200,9 +200,9 @@ func (fs *filesystem) Release(ctx context.Context) { type inode struct { inodeRefs kernfs.InodeAttrs + kernfs.InodeDirectoryNoNewChildren kernfs.InodeNoDynamicLookup kernfs.InodeNotSymlink - kernfs.InodeDirectoryNoNewChildren kernfs.OrderedChildren locks vfs.FileLocks @@ -331,3 +331,9 @@ func (i *inode) Stat(ctx context.Context, fs *vfs.Filesystem, opts vfs.StatOptio func (i *inode) DecRef(context.Context) { i.inodeRefs.DecRef(i.Destroy) } + +// StatFS implements kernfs.Inode.StatFS. +func (i *inode) StatFS(ctx context.Context, fs *vfs.Filesystem) (linux.Statfs, error) { + // TODO(gvisor.dev/issues/3413): Complete the implementation of statfs. + return vfs.GenericStatFS(linux.FUSE_SUPER_MAGIC), nil +} diff --git a/pkg/sentry/fsimpl/fuse/inode_refs.go b/pkg/sentry/fsimpl/fuse/inode_refs.go index 12e7d6e6c..6b9456e1d 100644 --- a/pkg/sentry/fsimpl/fuse/inode_refs.go +++ b/pkg/sentry/fsimpl/fuse/inode_refs.go @@ -2,11 +2,11 @@ package fuse import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/host/connected_endpoint_refs.go b/pkg/sentry/fsimpl/host/connected_endpoint_refs.go index 7fa5a516d..babb3f664 100644 --- a/pkg/sentry/fsimpl/host/connected_endpoint_refs.go +++ b/pkg/sentry/fsimpl/host/connected_endpoint_refs.go @@ -2,11 +2,11 @@ package host import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/host/host.go b/pkg/sentry/fsimpl/host/host.go index 2d3821f33..7561f821c 100644 --- a/pkg/sentry/fsimpl/host/host.go +++ b/pkg/sentry/fsimpl/host/host.go @@ -186,6 +186,7 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe // inode implements kernfs.Inode. type inode struct { + kernfs.InodeNoStatFS kernfs.InodeNotDirectory kernfs.InodeNotSymlink diff --git a/pkg/sentry/fsimpl/host/inode_refs.go b/pkg/sentry/fsimpl/host/inode_refs.go index c294b8b80..17f90ce4a 100644 --- a/pkg/sentry/fsimpl/host/inode_refs.go +++ b/pkg/sentry/fsimpl/host/inode_refs.go @@ -2,11 +2,11 @@ package host import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/kernfs/dentry_refs.go b/pkg/sentry/fsimpl/kernfs/dentry_refs.go index dd5325635..79863b3bc 100644 --- a/pkg/sentry/fsimpl/kernfs/dentry_refs.go +++ b/pkg/sentry/fsimpl/kernfs/dentry_refs.go @@ -2,11 +2,11 @@ package kernfs import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/kernfs/dynamic_bytes_file.go b/pkg/sentry/fsimpl/kernfs/dynamic_bytes_file.go index 12adf727a..1ee089620 100644 --- a/pkg/sentry/fsimpl/kernfs/dynamic_bytes_file.go +++ b/pkg/sentry/fsimpl/kernfs/dynamic_bytes_file.go @@ -35,6 +35,7 @@ import ( // +stateify savable type DynamicBytesFile struct { InodeAttrs + InodeNoStatFS InodeNoopRefCount InodeNotDirectory InodeNotSymlink diff --git a/pkg/sentry/fsimpl/kernfs/filesystem.go b/pkg/sentry/fsimpl/kernfs/filesystem.go index e5d6b5c35..0e3011689 100644 --- a/pkg/sentry/fsimpl/kernfs/filesystem.go +++ b/pkg/sentry/fsimpl/kernfs/filesystem.go @@ -721,14 +721,13 @@ func (fs *Filesystem) StatAt(ctx context.Context, rp *vfs.ResolvingPath, opts vf // StatFSAt implements vfs.FilesystemImpl.StatFSAt. func (fs *Filesystem) StatFSAt(ctx context.Context, rp *vfs.ResolvingPath) (linux.Statfs, error) { fs.mu.RLock() - _, _, err := fs.walkExistingLocked(ctx, rp) + _, inode, err := fs.walkExistingLocked(ctx, rp) fs.mu.RUnlock() fs.processDeferredDecRefs(ctx) if err != nil { return linux.Statfs{}, err } - // TODO(gvisor.dev/issue/1193): actually implement statfs. - return linux.Statfs{}, syserror.ENOSYS + return inode.StatFS(ctx, fs.VFSFilesystem()) } // SymlinkAt implements vfs.FilesystemImpl.SymlinkAt. diff --git a/pkg/sentry/fsimpl/kernfs/inode_impl_util.go b/pkg/sentry/fsimpl/kernfs/inode_impl_util.go index f442a5606..c0b863ba4 100644 --- a/pkg/sentry/fsimpl/kernfs/inode_impl_util.go +++ b/pkg/sentry/fsimpl/kernfs/inode_impl_util.go @@ -546,12 +546,13 @@ func (InodeSymlink) Open(ctx context.Context, rp *vfs.ResolvingPath, vfsd *vfs.D // // +stateify savable type StaticDirectory struct { - StaticDirectoryRefs - InodeNotSymlink - InodeDirectoryNoNewChildren InodeAttrs + InodeDirectoryNoNewChildren InodeNoDynamicLookup + InodeNoStatFS + InodeNotSymlink OrderedChildren + StaticDirectoryRefs locks vfs.FileLocks fdOpts GenericDirectoryFDOptions @@ -609,3 +610,12 @@ type AlwaysValid struct{} func (*AlwaysValid) Valid(context.Context) bool { return true } + +// InodeNoStatFS partially implements the Inode interface, where the client +// filesystem doesn't support statfs(2). +type InodeNoStatFS struct{} + +// StatFS implements Inode.StatFS. +func (*InodeNoStatFS) StatFS(context.Context, *vfs.Filesystem) (linux.Statfs, error) { + return linux.Statfs{}, syserror.ENOSYS +} diff --git a/pkg/sentry/fsimpl/kernfs/kernfs.go b/pkg/sentry/fsimpl/kernfs/kernfs.go index ca3685800..88fcd54aa 100644 --- a/pkg/sentry/fsimpl/kernfs/kernfs.go +++ b/pkg/sentry/fsimpl/kernfs/kernfs.go @@ -320,6 +320,11 @@ type Inode interface { // Precondition: rp.Done(). vfsd.Impl() must be the kernfs Dentry containing // the inode on which Open() is being called. Open(ctx context.Context, rp *vfs.ResolvingPath, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) + + // StatFS returns filesystem statistics for the client filesystem. This + // corresponds to vfs.FilesystemImpl.StatFSAt. If the client filesystem + // doesn't support statfs(2), this should return ENOSYS. + StatFS(ctx context.Context, fs *vfs.Filesystem) (linux.Statfs, error) } type inodeRefs interface { diff --git a/pkg/sentry/fsimpl/kernfs/kernfs_state_autogen.go b/pkg/sentry/fsimpl/kernfs/kernfs_state_autogen.go index 12aaf797f..cf480327f 100644 --- a/pkg/sentry/fsimpl/kernfs/kernfs_state_autogen.go +++ b/pkg/sentry/fsimpl/kernfs/kernfs_state_autogen.go @@ -36,6 +36,7 @@ func (x *DynamicBytesFile) StateTypeName() string { func (x *DynamicBytesFile) StateFields() []string { return []string{ "InodeAttrs", + "InodeNoStatFS", "InodeNoopRefCount", "InodeNotDirectory", "InodeNotSymlink", @@ -49,22 +50,24 @@ func (x *DynamicBytesFile) beforeSave() {} func (x *DynamicBytesFile) StateSave(m state.Sink) { x.beforeSave() m.Save(0, &x.InodeAttrs) - m.Save(1, &x.InodeNoopRefCount) - m.Save(2, &x.InodeNotDirectory) - m.Save(3, &x.InodeNotSymlink) - m.Save(4, &x.locks) - m.Save(5, &x.data) + m.Save(1, &x.InodeNoStatFS) + m.Save(2, &x.InodeNoopRefCount) + m.Save(3, &x.InodeNotDirectory) + m.Save(4, &x.InodeNotSymlink) + m.Save(5, &x.locks) + m.Save(6, &x.data) } func (x *DynamicBytesFile) afterLoad() {} func (x *DynamicBytesFile) StateLoad(m state.Source) { m.Load(0, &x.InodeAttrs) - m.Load(1, &x.InodeNoopRefCount) - m.Load(2, &x.InodeNotDirectory) - m.Load(3, &x.InodeNotSymlink) - m.Load(4, &x.locks) - m.Load(5, &x.data) + m.Load(1, &x.InodeNoStatFS) + m.Load(2, &x.InodeNoopRefCount) + m.Load(3, &x.InodeNotDirectory) + m.Load(4, &x.InodeNotSymlink) + m.Load(5, &x.locks) + m.Load(6, &x.data) } func (x *DynamicBytesFD) StateTypeName() string { @@ -108,12 +111,13 @@ func (x *StaticDirectory) StateTypeName() string { func (x *StaticDirectory) StateFields() []string { return []string{ - "StaticDirectoryRefs", - "InodeNotSymlink", - "InodeDirectoryNoNewChildren", "InodeAttrs", + "InodeDirectoryNoNewChildren", "InodeNoDynamicLookup", + "InodeNoStatFS", + "InodeNotSymlink", "OrderedChildren", + "StaticDirectoryRefs", "locks", "fdOpts", } @@ -123,27 +127,29 @@ func (x *StaticDirectory) beforeSave() {} func (x *StaticDirectory) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.StaticDirectoryRefs) - m.Save(1, &x.InodeNotSymlink) - m.Save(2, &x.InodeDirectoryNoNewChildren) - m.Save(3, &x.InodeAttrs) - m.Save(4, &x.InodeNoDynamicLookup) + m.Save(0, &x.InodeAttrs) + m.Save(1, &x.InodeDirectoryNoNewChildren) + m.Save(2, &x.InodeNoDynamicLookup) + m.Save(3, &x.InodeNoStatFS) + m.Save(4, &x.InodeNotSymlink) m.Save(5, &x.OrderedChildren) - m.Save(6, &x.locks) - m.Save(7, &x.fdOpts) + m.Save(6, &x.StaticDirectoryRefs) + m.Save(7, &x.locks) + m.Save(8, &x.fdOpts) } func (x *StaticDirectory) afterLoad() {} func (x *StaticDirectory) StateLoad(m state.Source) { - m.Load(0, &x.StaticDirectoryRefs) - m.Load(1, &x.InodeNotSymlink) - m.Load(2, &x.InodeDirectoryNoNewChildren) - m.Load(3, &x.InodeAttrs) - m.Load(4, &x.InodeNoDynamicLookup) + m.Load(0, &x.InodeAttrs) + m.Load(1, &x.InodeDirectoryNoNewChildren) + m.Load(2, &x.InodeNoDynamicLookup) + m.Load(3, &x.InodeNoStatFS) + m.Load(4, &x.InodeNotSymlink) m.Load(5, &x.OrderedChildren) - m.Load(6, &x.locks) - m.Load(7, &x.fdOpts) + m.Load(6, &x.StaticDirectoryRefs) + m.Load(7, &x.locks) + m.Load(8, &x.fdOpts) } func (x *slotList) StateTypeName() string { diff --git a/pkg/sentry/fsimpl/kernfs/static_directory_refs.go b/pkg/sentry/fsimpl/kernfs/static_directory_refs.go index 80513f6aa..478b04bdd 100644 --- a/pkg/sentry/fsimpl/kernfs/static_directory_refs.go +++ b/pkg/sentry/fsimpl/kernfs/static_directory_refs.go @@ -2,11 +2,11 @@ package kernfs import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/kernfs/symlink.go b/pkg/sentry/fsimpl/kernfs/symlink.go index 2ab3f53fd..64731a3e4 100644 --- a/pkg/sentry/fsimpl/kernfs/symlink.go +++ b/pkg/sentry/fsimpl/kernfs/symlink.go @@ -28,6 +28,7 @@ type StaticSymlink struct { InodeAttrs InodeNoopRefCount InodeSymlink + InodeNoStatFS target string } diff --git a/pkg/sentry/fsimpl/pipefs/pipefs.go b/pkg/sentry/fsimpl/pipefs/pipefs.go index 2ca793db9..7053ad6db 100644 --- a/pkg/sentry/fsimpl/pipefs/pipefs.go +++ b/pkg/sentry/fsimpl/pipefs/pipefs.go @@ -143,14 +143,16 @@ func (i *inode) SetStat(ctx context.Context, vfsfs *vfs.Filesystem, creds *auth. return syserror.EPERM } -// TODO(gvisor.dev/issue/1193): kernfs does not provide a way to implement -// statfs, from which we should indicate PIPEFS_MAGIC. - // Open implements kernfs.Inode.Open. func (i *inode) Open(ctx context.Context, rp *vfs.ResolvingPath, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) { return i.pipe.Open(ctx, rp.Mount(), vfsd, opts.Flags, &i.locks) } +// StatFS implements kernfs.Inode.StatFS. +func (i *inode) StatFS(ctx context.Context, fs *vfs.Filesystem) (linux.Statfs, error) { + return vfs.GenericStatFS(linux.PIPEFS_MAGIC), nil +} + // NewConnectedPipeFDs returns a pair of FileDescriptions representing the read // and write ends of a newly-created pipe, as for pipe(2) and pipe2(2). // diff --git a/pkg/sentry/fsimpl/proc/fd_dir_inode_refs.go b/pkg/sentry/fsimpl/proc/fd_dir_inode_refs.go index 8ed286c46..9431c1506 100644 --- a/pkg/sentry/fsimpl/proc/fd_dir_inode_refs.go +++ b/pkg/sentry/fsimpl/proc/fd_dir_inode_refs.go @@ -2,11 +2,11 @@ package proc import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/proc/fd_info_dir_inode_refs.go b/pkg/sentry/fsimpl/proc/fd_info_dir_inode_refs.go index 957c6a6dd..872b20eb0 100644 --- a/pkg/sentry/fsimpl/proc/fd_info_dir_inode_refs.go +++ b/pkg/sentry/fsimpl/proc/fd_info_dir_inode_refs.go @@ -2,11 +2,11 @@ package proc import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/proc/filesystem.go b/pkg/sentry/fsimpl/proc/filesystem.go index c350ec127..03b5941b9 100644 --- a/pkg/sentry/fsimpl/proc/filesystem.go +++ b/pkg/sentry/fsimpl/proc/filesystem.go @@ -121,3 +121,10 @@ func newStaticDir(creds *auth.Credentials, devMajor, devMinor uint32, ino uint64 type InternalData struct { Cgroups map[string]string } + +type implStatFS struct{} + +// StatFS implements kernfs.Inode.StatFS. +func (*implStatFS) StatFS(context.Context, *vfs.Filesystem) (linux.Statfs, error) { + return vfs.GenericStatFS(linux.PROC_SUPER_MAGIC), nil +} diff --git a/pkg/sentry/fsimpl/proc/proc_state_autogen.go b/pkg/sentry/fsimpl/proc/proc_state_autogen.go index 907ef38e0..1a13f066a 100644 --- a/pkg/sentry/fsimpl/proc/proc_state_autogen.go +++ b/pkg/sentry/fsimpl/proc/proc_state_autogen.go @@ -77,12 +77,13 @@ func (x *subtasksInode) StateTypeName() string { func (x *subtasksInode) StateFields() []string { return []string{ - "subtasksInodeRefs", - "InodeNotSymlink", - "InodeDirectoryNoNewChildren", + "implStatFS", + "AlwaysValid", "InodeAttrs", + "InodeDirectoryNoNewChildren", + "InodeNotSymlink", "OrderedChildren", - "AlwaysValid", + "subtasksInodeRefs", "locks", "fs", "task", @@ -95,33 +96,35 @@ func (x *subtasksInode) beforeSave() {} func (x *subtasksInode) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.subtasksInodeRefs) - m.Save(1, &x.InodeNotSymlink) - m.Save(2, &x.InodeDirectoryNoNewChildren) - m.Save(3, &x.InodeAttrs) - m.Save(4, &x.OrderedChildren) - m.Save(5, &x.AlwaysValid) - m.Save(6, &x.locks) - m.Save(7, &x.fs) - m.Save(8, &x.task) - m.Save(9, &x.pidns) - m.Save(10, &x.cgroupControllers) + m.Save(0, &x.implStatFS) + m.Save(1, &x.AlwaysValid) + m.Save(2, &x.InodeAttrs) + m.Save(3, &x.InodeDirectoryNoNewChildren) + m.Save(4, &x.InodeNotSymlink) + m.Save(5, &x.OrderedChildren) + m.Save(6, &x.subtasksInodeRefs) + m.Save(7, &x.locks) + m.Save(8, &x.fs) + m.Save(9, &x.task) + m.Save(10, &x.pidns) + m.Save(11, &x.cgroupControllers) } func (x *subtasksInode) afterLoad() {} func (x *subtasksInode) StateLoad(m state.Source) { - m.Load(0, &x.subtasksInodeRefs) - m.Load(1, &x.InodeNotSymlink) - m.Load(2, &x.InodeDirectoryNoNewChildren) - m.Load(3, &x.InodeAttrs) - m.Load(4, &x.OrderedChildren) - m.Load(5, &x.AlwaysValid) - m.Load(6, &x.locks) - m.Load(7, &x.fs) - m.Load(8, &x.task) - m.Load(9, &x.pidns) - m.Load(10, &x.cgroupControllers) + m.Load(0, &x.implStatFS) + m.Load(1, &x.AlwaysValid) + m.Load(2, &x.InodeAttrs) + m.Load(3, &x.InodeDirectoryNoNewChildren) + m.Load(4, &x.InodeNotSymlink) + m.Load(5, &x.OrderedChildren) + m.Load(6, &x.subtasksInodeRefs) + m.Load(7, &x.locks) + m.Load(8, &x.fs) + m.Load(9, &x.task) + m.Load(10, &x.pidns) + m.Load(11, &x.cgroupControllers) } func (x *subtasksInodeRefs) StateTypeName() string { @@ -153,12 +156,13 @@ func (x *taskInode) StateTypeName() string { func (x *taskInode) StateFields() []string { return []string{ - "taskInodeRefs", - "InodeNotSymlink", + "implStatFS", + "InodeAttrs", "InodeDirectoryNoNewChildren", "InodeNoDynamicLookup", - "InodeAttrs", + "InodeNotSymlink", "OrderedChildren", + "taskInodeRefs", "locks", "task", } @@ -168,27 +172,29 @@ func (x *taskInode) beforeSave() {} func (x *taskInode) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.taskInodeRefs) - m.Save(1, &x.InodeNotSymlink) + m.Save(0, &x.implStatFS) + m.Save(1, &x.InodeAttrs) m.Save(2, &x.InodeDirectoryNoNewChildren) m.Save(3, &x.InodeNoDynamicLookup) - m.Save(4, &x.InodeAttrs) + m.Save(4, &x.InodeNotSymlink) m.Save(5, &x.OrderedChildren) - m.Save(6, &x.locks) - m.Save(7, &x.task) + m.Save(6, &x.taskInodeRefs) + m.Save(7, &x.locks) + m.Save(8, &x.task) } func (x *taskInode) afterLoad() {} func (x *taskInode) StateLoad(m state.Source) { - m.Load(0, &x.taskInodeRefs) - m.Load(1, &x.InodeNotSymlink) + m.Load(0, &x.implStatFS) + m.Load(1, &x.InodeAttrs) m.Load(2, &x.InodeDirectoryNoNewChildren) m.Load(3, &x.InodeNoDynamicLookup) - m.Load(4, &x.InodeAttrs) + m.Load(4, &x.InodeNotSymlink) m.Load(5, &x.OrderedChildren) - m.Load(6, &x.locks) - m.Load(7, &x.task) + m.Load(6, &x.taskInodeRefs) + m.Load(7, &x.locks) + m.Load(8, &x.task) } func (x *fdDirInode) StateTypeName() string { @@ -197,13 +203,14 @@ func (x *fdDirInode) StateTypeName() string { func (x *fdDirInode) StateFields() []string { return []string{ + "fdDir", "fdDirInodeRefs", - "InodeNotSymlink", - "InodeDirectoryNoNewChildren", + "implStatFS", + "AlwaysValid", "InodeAttrs", + "InodeDirectoryNoNewChildren", + "InodeNotSymlink", "OrderedChildren", - "AlwaysValid", - "fdDir", } } @@ -211,25 +218,27 @@ func (x *fdDirInode) beforeSave() {} func (x *fdDirInode) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.fdDirInodeRefs) - m.Save(1, &x.InodeNotSymlink) - m.Save(2, &x.InodeDirectoryNoNewChildren) - m.Save(3, &x.InodeAttrs) - m.Save(4, &x.OrderedChildren) - m.Save(5, &x.AlwaysValid) - m.Save(6, &x.fdDir) + m.Save(0, &x.fdDir) + m.Save(1, &x.fdDirInodeRefs) + m.Save(2, &x.implStatFS) + m.Save(3, &x.AlwaysValid) + m.Save(4, &x.InodeAttrs) + m.Save(5, &x.InodeDirectoryNoNewChildren) + m.Save(6, &x.InodeNotSymlink) + m.Save(7, &x.OrderedChildren) } func (x *fdDirInode) afterLoad() {} func (x *fdDirInode) StateLoad(m state.Source) { - m.Load(0, &x.fdDirInodeRefs) - m.Load(1, &x.InodeNotSymlink) - m.Load(2, &x.InodeDirectoryNoNewChildren) - m.Load(3, &x.InodeAttrs) - m.Load(4, &x.OrderedChildren) - m.Load(5, &x.AlwaysValid) - m.Load(6, &x.fdDir) + m.Load(0, &x.fdDir) + m.Load(1, &x.fdDirInodeRefs) + m.Load(2, &x.implStatFS) + m.Load(3, &x.AlwaysValid) + m.Load(4, &x.InodeAttrs) + m.Load(5, &x.InodeDirectoryNoNewChildren) + m.Load(6, &x.InodeNotSymlink) + m.Load(7, &x.OrderedChildren) } func (x *fdSymlink) StateTypeName() string { @@ -238,6 +247,7 @@ func (x *fdSymlink) StateTypeName() string { func (x *fdSymlink) StateFields() []string { return []string{ + "implStatFS", "InodeAttrs", "InodeNoopRefCount", "InodeSymlink", @@ -250,21 +260,23 @@ func (x *fdSymlink) beforeSave() {} func (x *fdSymlink) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.InodeAttrs) - m.Save(1, &x.InodeNoopRefCount) - m.Save(2, &x.InodeSymlink) - m.Save(3, &x.task) - m.Save(4, &x.fd) + m.Save(0, &x.implStatFS) + m.Save(1, &x.InodeAttrs) + m.Save(2, &x.InodeNoopRefCount) + m.Save(3, &x.InodeSymlink) + m.Save(4, &x.task) + m.Save(5, &x.fd) } func (x *fdSymlink) afterLoad() {} func (x *fdSymlink) StateLoad(m state.Source) { - m.Load(0, &x.InodeAttrs) - m.Load(1, &x.InodeNoopRefCount) - m.Load(2, &x.InodeSymlink) - m.Load(3, &x.task) - m.Load(4, &x.fd) + m.Load(0, &x.implStatFS) + m.Load(1, &x.InodeAttrs) + m.Load(2, &x.InodeNoopRefCount) + m.Load(3, &x.InodeSymlink) + m.Load(4, &x.task) + m.Load(5, &x.fd) } func (x *fdInfoDirInode) StateTypeName() string { @@ -273,13 +285,14 @@ func (x *fdInfoDirInode) StateTypeName() string { func (x *fdInfoDirInode) StateFields() []string { return []string{ + "fdDir", "fdInfoDirInodeRefs", - "InodeNotSymlink", - "InodeDirectoryNoNewChildren", + "implStatFS", + "AlwaysValid", "InodeAttrs", + "InodeDirectoryNoNewChildren", + "InodeNotSymlink", "OrderedChildren", - "AlwaysValid", - "fdDir", } } @@ -287,25 +300,27 @@ func (x *fdInfoDirInode) beforeSave() {} func (x *fdInfoDirInode) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.fdInfoDirInodeRefs) - m.Save(1, &x.InodeNotSymlink) - m.Save(2, &x.InodeDirectoryNoNewChildren) - m.Save(3, &x.InodeAttrs) - m.Save(4, &x.OrderedChildren) - m.Save(5, &x.AlwaysValid) - m.Save(6, &x.fdDir) + m.Save(0, &x.fdDir) + m.Save(1, &x.fdInfoDirInodeRefs) + m.Save(2, &x.implStatFS) + m.Save(3, &x.AlwaysValid) + m.Save(4, &x.InodeAttrs) + m.Save(5, &x.InodeDirectoryNoNewChildren) + m.Save(6, &x.InodeNotSymlink) + m.Save(7, &x.OrderedChildren) } func (x *fdInfoDirInode) afterLoad() {} func (x *fdInfoDirInode) StateLoad(m state.Source) { - m.Load(0, &x.fdInfoDirInodeRefs) - m.Load(1, &x.InodeNotSymlink) - m.Load(2, &x.InodeDirectoryNoNewChildren) - m.Load(3, &x.InodeAttrs) - m.Load(4, &x.OrderedChildren) - m.Load(5, &x.AlwaysValid) - m.Load(6, &x.fdDir) + m.Load(0, &x.fdDir) + m.Load(1, &x.fdInfoDirInodeRefs) + m.Load(2, &x.implStatFS) + m.Load(3, &x.AlwaysValid) + m.Load(4, &x.InodeAttrs) + m.Load(5, &x.InodeDirectoryNoNewChildren) + m.Load(6, &x.InodeNotSymlink) + m.Load(7, &x.OrderedChildren) } func (x *fdInfoData) StateTypeName() string { @@ -670,6 +685,7 @@ func (x *exeSymlink) StateTypeName() string { func (x *exeSymlink) StateFields() []string { return []string{ + "implStatFS", "InodeAttrs", "InodeNoopRefCount", "InodeSymlink", @@ -681,19 +697,21 @@ func (x *exeSymlink) beforeSave() {} func (x *exeSymlink) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.InodeAttrs) - m.Save(1, &x.InodeNoopRefCount) - m.Save(2, &x.InodeSymlink) - m.Save(3, &x.task) + m.Save(0, &x.implStatFS) + m.Save(1, &x.InodeAttrs) + m.Save(2, &x.InodeNoopRefCount) + m.Save(3, &x.InodeSymlink) + m.Save(4, &x.task) } func (x *exeSymlink) afterLoad() {} func (x *exeSymlink) StateLoad(m state.Source) { - m.Load(0, &x.InodeAttrs) - m.Load(1, &x.InodeNoopRefCount) - m.Load(2, &x.InodeSymlink) - m.Load(3, &x.task) + m.Load(0, &x.implStatFS) + m.Load(1, &x.InodeAttrs) + m.Load(2, &x.InodeNoopRefCount) + m.Load(3, &x.InodeSymlink) + m.Load(4, &x.task) } func (x *mountInfoData) StateTypeName() string { @@ -1011,12 +1029,13 @@ func (x *tasksInode) StateTypeName() string { func (x *tasksInode) StateFields() []string { return []string{ - "tasksInodeRefs", - "InodeNotSymlink", - "InodeDirectoryNoNewChildren", + "implStatFS", + "AlwaysValid", "InodeAttrs", + "InodeDirectoryNoNewChildren", + "InodeNotSymlink", "OrderedChildren", - "AlwaysValid", + "tasksInodeRefs", "locks", "fs", "pidns", @@ -1030,35 +1049,37 @@ func (x *tasksInode) beforeSave() {} func (x *tasksInode) StateSave(m state.Sink) { x.beforeSave() - m.Save(0, &x.tasksInodeRefs) - m.Save(1, &x.InodeNotSymlink) - m.Save(2, &x.InodeDirectoryNoNewChildren) - m.Save(3, &x.InodeAttrs) - m.Save(4, &x.OrderedChildren) - m.Save(5, &x.AlwaysValid) - m.Save(6, &x.locks) - m.Save(7, &x.fs) - m.Save(8, &x.pidns) - m.Save(9, &x.selfSymlink) - m.Save(10, &x.threadSelfSymlink) - m.Save(11, &x.cgroupControllers) + m.Save(0, &x.implStatFS) + m.Save(1, &x.AlwaysValid) + m.Save(2, &x.InodeAttrs) + m.Save(3, &x.InodeDirectoryNoNewChildren) + m.Save(4, &x.InodeNotSymlink) + m.Save(5, &x.OrderedChildren) + m.Save(6, &x.tasksInodeRefs) + m.Save(7, &x.locks) + m.Save(8, &x.fs) + m.Save(9, &x.pidns) + m.Save(10, &x.selfSymlink) + m.Save(11, &x.threadSelfSymlink) + m.Save(12, &x.cgroupControllers) } func (x *tasksInode) afterLoad() {} func (x *tasksInode) StateLoad(m state.Source) { - m.Load(0, &x.tasksInodeRefs) - m.Load(1, &x.InodeNotSymlink) - m.Load(2, &x.InodeDirectoryNoNewChildren) - m.Load(3, &x.InodeAttrs) - m.Load(4, &x.OrderedChildren) - m.Load(5, &x.AlwaysValid) - m.Load(6, &x.locks) - m.Load(7, &x.fs) - m.Load(8, &x.pidns) - m.Load(9, &x.selfSymlink) - m.Load(10, &x.threadSelfSymlink) - m.Load(11, &x.cgroupControllers) + m.Load(0, &x.implStatFS) + m.Load(1, &x.AlwaysValid) + m.Load(2, &x.InodeAttrs) + m.Load(3, &x.InodeDirectoryNoNewChildren) + m.Load(4, &x.InodeNotSymlink) + m.Load(5, &x.OrderedChildren) + m.Load(6, &x.tasksInodeRefs) + m.Load(7, &x.locks) + m.Load(8, &x.fs) + m.Load(9, &x.pidns) + m.Load(10, &x.selfSymlink) + m.Load(11, &x.threadSelfSymlink) + m.Load(12, &x.cgroupControllers) } func (x *statData) StateTypeName() string { diff --git a/pkg/sentry/fsimpl/proc/subtasks.go b/pkg/sentry/fsimpl/proc/subtasks.go index 01c0efb3a..d57d94dbc 100644 --- a/pkg/sentry/fsimpl/proc/subtasks.go +++ b/pkg/sentry/fsimpl/proc/subtasks.go @@ -31,12 +31,13 @@ import ( // // +stateify savable type subtasksInode struct { - subtasksInodeRefs - kernfs.InodeNotSymlink - kernfs.InodeDirectoryNoNewChildren + implStatFS + kernfs.AlwaysValid kernfs.InodeAttrs + kernfs.InodeDirectoryNoNewChildren + kernfs.InodeNotSymlink kernfs.OrderedChildren - kernfs.AlwaysValid + subtasksInodeRefs locks vfs.FileLocks diff --git a/pkg/sentry/fsimpl/proc/subtasks_inode_refs.go b/pkg/sentry/fsimpl/proc/subtasks_inode_refs.go index a80ec9e0a..c6d9b3522 100644 --- a/pkg/sentry/fsimpl/proc/subtasks_inode_refs.go +++ b/pkg/sentry/fsimpl/proc/subtasks_inode_refs.go @@ -2,11 +2,11 @@ package proc import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/proc/task.go b/pkg/sentry/fsimpl/proc/task.go index 66b557abd..dbdb5d929 100644 --- a/pkg/sentry/fsimpl/proc/task.go +++ b/pkg/sentry/fsimpl/proc/task.go @@ -32,12 +32,13 @@ import ( // // +stateify savable type taskInode struct { - taskInodeRefs - kernfs.InodeNotSymlink + implStatFS + kernfs.InodeAttrs kernfs.InodeDirectoryNoNewChildren kernfs.InodeNoDynamicLookup - kernfs.InodeAttrs + kernfs.InodeNotSymlink kernfs.OrderedChildren + taskInodeRefs locks vfs.FileLocks diff --git a/pkg/sentry/fsimpl/proc/task_fds.go b/pkg/sentry/fsimpl/proc/task_fds.go index 0527b2de8..3f0d78461 100644 --- a/pkg/sentry/fsimpl/proc/task_fds.go +++ b/pkg/sentry/fsimpl/proc/task_fds.go @@ -100,13 +100,14 @@ func (i *fdDir) IterDirents(ctx context.Context, cb vfs.IterDirentsCallback, off // // +stateify savable type fdDirInode struct { + fdDir fdDirInodeRefs - kernfs.InodeNotSymlink - kernfs.InodeDirectoryNoNewChildren + implStatFS + kernfs.AlwaysValid kernfs.InodeAttrs + kernfs.InodeDirectoryNoNewChildren + kernfs.InodeNotSymlink kernfs.OrderedChildren - kernfs.AlwaysValid - fdDir } var _ kernfs.Inode = (*fdDirInode)(nil) @@ -185,6 +186,7 @@ func (i *fdDirInode) DecRef(context.Context) { // // +stateify savable type fdSymlink struct { + implStatFS kernfs.InodeAttrs kernfs.InodeNoopRefCount kernfs.InodeSymlink @@ -233,13 +235,14 @@ func (s *fdSymlink) Getlink(ctx context.Context, mnt *vfs.Mount) (vfs.VirtualDen // // +stateify savable type fdInfoDirInode struct { + fdDir fdInfoDirInodeRefs - kernfs.InodeNotSymlink - kernfs.InodeDirectoryNoNewChildren + implStatFS + kernfs.AlwaysValid kernfs.InodeAttrs + kernfs.InodeDirectoryNoNewChildren + kernfs.InodeNotSymlink kernfs.OrderedChildren - kernfs.AlwaysValid - fdDir } var _ kernfs.Inode = (*fdInfoDirInode)(nil) diff --git a/pkg/sentry/fsimpl/proc/task_files.go b/pkg/sentry/fsimpl/proc/task_files.go index 830b78949..356036b9b 100644 --- a/pkg/sentry/fsimpl/proc/task_files.go +++ b/pkg/sentry/fsimpl/proc/task_files.go @@ -648,6 +648,7 @@ func (o *oomScoreAdj) Write(ctx context.Context, src usermem.IOSequence, offset // // +stateify savable type exeSymlink struct { + implStatFS kernfs.InodeAttrs kernfs.InodeNoopRefCount kernfs.InodeSymlink @@ -832,6 +833,7 @@ func (s *namespaceSymlink) Getlink(ctx context.Context, mnt *vfs.Mount) (vfs.Vir // namespaceInode is a synthetic inode created to represent a namespace in // /proc/[pid]/ns/*. type namespaceInode struct { + implStatFS kernfs.InodeAttrs kernfs.InodeNoopRefCount kernfs.InodeNotDirectory diff --git a/pkg/sentry/fsimpl/proc/task_inode_refs.go b/pkg/sentry/fsimpl/proc/task_inode_refs.go index c4835cbca..714488450 100644 --- a/pkg/sentry/fsimpl/proc/task_inode_refs.go +++ b/pkg/sentry/fsimpl/proc/task_inode_refs.go @@ -2,11 +2,11 @@ package proc import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/proc/tasks.go b/pkg/sentry/fsimpl/proc/tasks.go index 863c4467e..3ea00ab87 100644 --- a/pkg/sentry/fsimpl/proc/tasks.go +++ b/pkg/sentry/fsimpl/proc/tasks.go @@ -37,12 +37,13 @@ const ( // // +stateify savable type tasksInode struct { - tasksInodeRefs - kernfs.InodeNotSymlink - kernfs.InodeDirectoryNoNewChildren + implStatFS + kernfs.AlwaysValid kernfs.InodeAttrs + kernfs.InodeDirectoryNoNewChildren + kernfs.InodeNotSymlink kernfs.OrderedChildren - kernfs.AlwaysValid + tasksInodeRefs locks vfs.FileLocks diff --git a/pkg/sentry/fsimpl/proc/tasks_files.go b/pkg/sentry/fsimpl/proc/tasks_files.go index 7d8983aa5..8c41729e4 100644 --- a/pkg/sentry/fsimpl/proc/tasks_files.go +++ b/pkg/sentry/fsimpl/proc/tasks_files.go @@ -32,6 +32,7 @@ import ( ) type selfSymlink struct { + implStatFS kernfs.InodeAttrs kernfs.InodeNoopRefCount kernfs.InodeSymlink @@ -74,6 +75,7 @@ func (*selfSymlink) SetStat(context.Context, *vfs.Filesystem, *auth.Credentials, } type threadSelfSymlink struct { + implStatFS kernfs.InodeAttrs kernfs.InodeNoopRefCount kernfs.InodeSymlink diff --git a/pkg/sentry/fsimpl/proc/tasks_inode_refs.go b/pkg/sentry/fsimpl/proc/tasks_inode_refs.go index 5dfb34238..22d9cc488 100644 --- a/pkg/sentry/fsimpl/proc/tasks_inode_refs.go +++ b/pkg/sentry/fsimpl/proc/tasks_inode_refs.go @@ -2,11 +2,11 @@ package proc import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/sockfs/sockfs.go b/pkg/sentry/fsimpl/sockfs/sockfs.go index c61818ff6..94a998568 100644 --- a/pkg/sentry/fsimpl/sockfs/sockfs.go +++ b/pkg/sentry/fsimpl/sockfs/sockfs.go @@ -81,10 +81,10 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe // inode implements kernfs.Inode. type inode struct { - kernfs.InodeNotDirectory - kernfs.InodeNotSymlink kernfs.InodeAttrs kernfs.InodeNoopRefCount + kernfs.InodeNotDirectory + kernfs.InodeNotSymlink } // Open implements kernfs.Inode.Open. @@ -92,6 +92,11 @@ func (i *inode) Open(ctx context.Context, rp *vfs.ResolvingPath, vfsd *vfs.Dentr return nil, syserror.ENXIO } +// StatFS implements kernfs.Inode.StatFS. +func (i *inode) StatFS(ctx context.Context, fs *vfs.Filesystem) (linux.Statfs, error) { + return vfs.GenericStatFS(linux.SOCKFS_MAGIC), nil +} + // NewDentry constructs and returns a sockfs dentry. // // Preconditions: mnt.Filesystem() must have been returned by NewFilesystem(). diff --git a/pkg/sentry/fsimpl/sys/dir_refs.go b/pkg/sentry/fsimpl/sys/dir_refs.go index c05154e2b..89609b198 100644 --- a/pkg/sentry/fsimpl/sys/dir_refs.go +++ b/pkg/sentry/fsimpl/sys/dir_refs.go @@ -2,11 +2,11 @@ package sys import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so diff --git a/pkg/sentry/fsimpl/sys/kcov.go b/pkg/sentry/fsimpl/sys/kcov.go index 92710d877..73f3d3309 100644 --- a/pkg/sentry/fsimpl/sys/kcov.go +++ b/pkg/sentry/fsimpl/sys/kcov.go @@ -39,8 +39,9 @@ func (fs *filesystem) newKcovFile(ctx context.Context, creds *auth.Credentials) type kcovInode struct { kernfs.InodeAttrs kernfs.InodeNoopRefCount - kernfs.InodeNotSymlink kernfs.InodeNotDirectory + kernfs.InodeNotSymlink + implStatFS } func (i *kcovInode) Open(ctx context.Context, rp *vfs.ResolvingPath, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) { diff --git a/pkg/sentry/fsimpl/sys/sys.go b/pkg/sentry/fsimpl/sys/sys.go index ea30a4ec2..39952d2d0 100644 --- a/pkg/sentry/fsimpl/sys/sys.go +++ b/pkg/sentry/fsimpl/sys/sys.go @@ -163,9 +163,16 @@ func (d *dir) DecRef(context.Context) { d.dirRefs.DecRef(d.Destroy) } +// StatFS implements kernfs.Inode.StatFS. +func (d *dir) StatFS(ctx context.Context, fs *vfs.Filesystem) (linux.Statfs, error) { + return vfs.GenericStatFS(linux.SYSFS_MAGIC), nil +} + // cpuFile implements kernfs.Inode. type cpuFile struct { + implStatFS kernfs.DynamicBytesFile + maxCores uint } @@ -182,3 +189,10 @@ func (fs *filesystem) newCPUFile(creds *auth.Credentials, maxCores uint, mode li d.Init(c) return d } + +type implStatFS struct{} + +// StatFS implements kernfs.Inode.StatFS. +func (*implStatFS) StatFS(context.Context, *vfs.Filesystem) (linux.Statfs, error) { + return vfs.GenericStatFS(linux.SYSFS_MAGIC), nil +} diff --git a/pkg/sentry/fsimpl/tmpfs/inode_refs.go b/pkg/sentry/fsimpl/tmpfs/inode_refs.go index 38cc30981..dbf0b2766 100644 --- a/pkg/sentry/fsimpl/tmpfs/inode_refs.go +++ b/pkg/sentry/fsimpl/tmpfs/inode_refs.go @@ -2,11 +2,11 @@ package tmpfs import ( "fmt" - refs_vfs1 "gvisor.dev/gvisor/pkg/refs" "runtime" "sync/atomic" "gvisor.dev/gvisor/pkg/log" + refs_vfs1 "gvisor.dev/gvisor/pkg/refs" ) // ownerType is used to customize logging. Note that we use a pointer to T so |