diff options
39 files changed, 1708 insertions, 1386 deletions
diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index cb272aef6..aa2fd1f47 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -183,9 +183,13 @@ steps: - <<: *benchmarks label: ":metal: FFMPEG benchmarks" command: make benchmark-platforms BENCHMARKS_SUITE=ffmpeg BENCHMARKS_TARGETS=test/benchmarks/media:ffmpeg_test + # For fio, running with --test.benchtime=Xs scales the written/read + # bytes to several GB. This is not a problem for root/bind/volume mounts, + # but for tmpfs mounts, the size can grow to more memory than the machine + # has availabe. Fix the runs to 10GB written/read for the benchmark. - <<: *benchmarks label: ":floppy_disk: FIO benchmarks" - command: make benchmark-platforms BENCHMARKS_SUITE=fio BENCHMARKS_TARGETS=test/benchmarks/fs:fio_test + command: make benchmark-platforms BENCHMARKS_SUITE=fio BENCHMARKS_TARGETS=test/benchmarks/fs:fio_test BENCHMARKS_OPTIONS=--test.benchtime=10000x - <<: *benchmarks label: ":globe_with_meridians: HTTPD benchmarks" command: make benchmark-platforms BENCHMARKS_FILTER="Continuous" BENCHMARKS_SUITE=httpd BENCHMARKS_TARGETS=test/benchmarks/network:httpd_test diff --git a/pkg/abi/linux/ptrace_amd64.go b/pkg/abi/linux/ptrace_amd64.go index ed3881e27..50e22fe7e 100644 --- a/pkg/abi/linux/ptrace_amd64.go +++ b/pkg/abi/linux/ptrace_amd64.go @@ -50,3 +50,14 @@ type PtraceRegs struct { Fs uint64 Gs uint64 } + +// InstructionPointer returns the address of the next instruction to +// be executed. +func (p *PtraceRegs) InstructionPointer() uint64 { + return p.Rip +} + +// StackPointer returns the address of the Stack pointer. +func (p *PtraceRegs) StackPointer() uint64 { + return p.Rsp +} diff --git a/pkg/abi/linux/ptrace_arm64.go b/pkg/abi/linux/ptrace_arm64.go index 6147738b3..da36811d2 100644 --- a/pkg/abi/linux/ptrace_arm64.go +++ b/pkg/abi/linux/ptrace_arm64.go @@ -27,3 +27,14 @@ type PtraceRegs struct { Pc uint64 Pstate uint64 } + +// InstructionPointer returns the address of the next instruction to be +// executed. +func (p *PtraceRegs) InstructionPointer() uint64 { + return p.Pc +} + +// StackPointer returns the address of the Stack pointer. +func (p *PtraceRegs) StackPointer() uint64 { + return p.Sp +} diff --git a/pkg/sentry/fsimpl/devpts/devpts.go b/pkg/sentry/fsimpl/devpts/devpts.go index d8c237753..e75954105 100644 --- a/pkg/sentry/fsimpl/devpts/devpts.go +++ b/pkg/sentry/fsimpl/devpts/devpts.go @@ -137,6 +137,11 @@ func (fs *filesystem) Release(ctx context.Context) { fs.Filesystem.Release(ctx) } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return "" +} + // rootInode is the root directory inode for the devpts mounts. // // +stateify savable diff --git a/pkg/sentry/fsimpl/ext/filesystem.go b/pkg/sentry/fsimpl/ext/filesystem.go index 917f1873d..d4fc484a2 100644 --- a/pkg/sentry/fsimpl/ext/filesystem.go +++ b/pkg/sentry/fsimpl/ext/filesystem.go @@ -548,3 +548,8 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe defer fs.mu.RUnlock() return genericPrependPath(vfsroot, vd.Mount(), vd.Dentry().Impl().(*dentry), b) } + +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return "" +} diff --git a/pkg/sentry/fsimpl/fuse/fusefs.go b/pkg/sentry/fsimpl/fuse/fusefs.go index 204d8d143..fb0ba2c6d 100644 --- a/pkg/sentry/fsimpl/fuse/fusefs.go +++ b/pkg/sentry/fsimpl/fuse/fusefs.go @@ -47,6 +47,9 @@ type FilesystemType struct{} // +stateify savable type filesystemOptions struct { + // mopts contains the raw, unparsed mount options passed to this filesystem. + mopts string + // userID specifies the numeric uid of the mount owner. // This option should not be specified by the filesystem owner. // It is set by libfuse (or, if libfuse is not used, must be set @@ -108,7 +111,7 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt return nil, nil, err } - var fsopts filesystemOptions + fsopts := filesystemOptions{mopts: opts.Data} mopts := vfs.GenericParseMountOptions(opts.Data) deviceDescriptorStr, ok := mopts["fd"] if !ok { @@ -260,6 +263,11 @@ func (fs *filesystem) Release(ctx context.Context) { fs.Filesystem.Release(ctx) } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return fs.opts.mopts +} + // inode implements kernfs.Inode. // // +stateify savable diff --git a/pkg/sentry/fsimpl/gofer/filesystem.go b/pkg/sentry/fsimpl/gofer/filesystem.go index 8f95473b6..c34451269 100644 --- a/pkg/sentry/fsimpl/gofer/filesystem.go +++ b/pkg/sentry/fsimpl/gofer/filesystem.go @@ -15,7 +15,9 @@ package gofer import ( + "fmt" "math" + "strings" "sync" "sync/atomic" @@ -1608,3 +1610,58 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe defer fs.renameMu.RUnlock() return genericPrependPath(vfsroot, vd.Mount(), vd.Dentry().Impl().(*dentry), b) } + +type mopt struct { + key string + value interface{} +} + +func (m mopt) String() string { + if m.value == nil { + return fmt.Sprintf("%s", m.key) + } + return fmt.Sprintf("%s=%v", m.key, m.value) +} + +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + optsKV := []mopt{ + {moptTransport, transportModeFD}, // Only valid value, currently. + {moptReadFD, fs.opts.fd}, // Currently, read and write FD are the same. + {moptWriteFD, fs.opts.fd}, // Currently, read and write FD are the same. + {moptAname, fs.opts.aname}, + {moptDfltUID, fs.opts.dfltuid}, + {moptDfltGID, fs.opts.dfltgid}, + {moptMsize, fs.opts.msize}, + {moptVersion, fs.opts.version}, + {moptDentryCacheLimit, fs.opts.maxCachedDentries}, + } + + switch fs.opts.interop { + case InteropModeExclusive: + optsKV = append(optsKV, mopt{moptCache, cacheFSCache}) + case InteropModeWritethrough: + optsKV = append(optsKV, mopt{moptCache, cacheFSCacheWritethrough}) + case InteropModeShared: + if fs.opts.regularFilesUseSpecialFileFD { + optsKV = append(optsKV, mopt{moptCache, cacheNone}) + } else { + optsKV = append(optsKV, mopt{moptCache, cacheRemoteRevalidating}) + } + } + if fs.opts.forcePageCache { + optsKV = append(optsKV, mopt{moptForcePageCache, nil}) + } + if fs.opts.limitHostFDTranslation { + optsKV = append(optsKV, mopt{moptLimitHostFDTranslation, nil}) + } + if fs.opts.overlayfsStaleRead { + optsKV = append(optsKV, mopt{moptOverlayfsStaleRead, nil}) + } + + opts := make([]string, 0, len(optsKV)) + for _, opt := range optsKV { + opts = append(opts, opt.String()) + } + return strings.Join(opts, ",") +} diff --git a/pkg/sentry/fsimpl/gofer/gofer.go b/pkg/sentry/fsimpl/gofer/gofer.go index 1508cbdf1..71569dc65 100644 --- a/pkg/sentry/fsimpl/gofer/gofer.go +++ b/pkg/sentry/fsimpl/gofer/gofer.go @@ -66,6 +66,34 @@ import ( // Name is the default filesystem name. const Name = "9p" +// Mount option names for goferfs. +const ( + moptTransport = "trans" + moptReadFD = "rfdno" + moptWriteFD = "wfdno" + moptAname = "aname" + moptDfltUID = "dfltuid" + moptDfltGID = "dfltgid" + moptMsize = "msize" + moptVersion = "version" + moptDentryCacheLimit = "dentry_cache_limit" + moptCache = "cache" + moptForcePageCache = "force_page_cache" + moptLimitHostFDTranslation = "limit_host_fd_translation" + moptOverlayfsStaleRead = "overlayfs_stale_read" +) + +// Valid values for the "cache" mount option. +const ( + cacheNone = "none" + cacheFSCache = "fscache" + cacheFSCacheWritethrough = "fscache_writethrough" + cacheRemoteRevalidating = "remote_revalidating" +) + +// Valid values for "trans" mount option. +const transportModeFD = "fd" + // FilesystemType implements vfs.FilesystemType. // // +stateify savable @@ -301,39 +329,39 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt // Get the attach name. fsopts.aname = "/" - if aname, ok := mopts["aname"]; ok { - delete(mopts, "aname") + if aname, ok := mopts[moptAname]; ok { + delete(mopts, moptAname) fsopts.aname = aname } // Parse the cache policy. For historical reasons, this defaults to the // least generally-applicable option, InteropModeExclusive. fsopts.interop = InteropModeExclusive - if cache, ok := mopts["cache"]; ok { - delete(mopts, "cache") + if cache, ok := mopts[moptCache]; ok { + delete(mopts, moptCache) switch cache { - case "fscache": + case cacheFSCache: fsopts.interop = InteropModeExclusive - case "fscache_writethrough": + case cacheFSCacheWritethrough: fsopts.interop = InteropModeWritethrough - case "none": + case cacheNone: fsopts.regularFilesUseSpecialFileFD = true fallthrough - case "remote_revalidating": + case cacheRemoteRevalidating: fsopts.interop = InteropModeShared default: - ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid cache policy: cache=%s", cache) + ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid cache policy: %s=%s", moptCache, cache) return nil, nil, syserror.EINVAL } } // Parse the default UID and GID. fsopts.dfltuid = _V9FS_DEFUID - if dfltuidstr, ok := mopts["dfltuid"]; ok { - delete(mopts, "dfltuid") + if dfltuidstr, ok := mopts[moptDfltUID]; ok { + delete(mopts, moptDfltUID) dfltuid, err := strconv.ParseUint(dfltuidstr, 10, 32) if err != nil { - ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid default UID: dfltuid=%s", dfltuidstr) + ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid default UID: %s=%s", moptDfltUID, dfltuidstr) return nil, nil, syserror.EINVAL } // In Linux, dfltuid is interpreted as a UID and is converted to a KUID @@ -342,11 +370,11 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt fsopts.dfltuid = auth.KUID(dfltuid) } fsopts.dfltgid = _V9FS_DEFGID - if dfltgidstr, ok := mopts["dfltgid"]; ok { - delete(mopts, "dfltgid") + if dfltgidstr, ok := mopts[moptDfltGID]; ok { + delete(mopts, moptDfltGID) dfltgid, err := strconv.ParseUint(dfltgidstr, 10, 32) if err != nil { - ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid default UID: dfltgid=%s", dfltgidstr) + ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid default UID: %s=%s", moptDfltGID, dfltgidstr) return nil, nil, syserror.EINVAL } fsopts.dfltgid = auth.KGID(dfltgid) @@ -354,11 +382,11 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt // Parse the 9P message size. fsopts.msize = 1024 * 1024 // 1M, tested to give good enough performance up to 64M - if msizestr, ok := mopts["msize"]; ok { - delete(mopts, "msize") + if msizestr, ok := mopts[moptMsize]; ok { + delete(mopts, moptMsize) msize, err := strconv.ParseUint(msizestr, 10, 32) if err != nil { - ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid message size: msize=%s", msizestr) + ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid message size: %s=%s", moptMsize, msizestr) return nil, nil, syserror.EINVAL } fsopts.msize = uint32(msize) @@ -366,34 +394,34 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt // Parse the 9P protocol version. fsopts.version = p9.HighestVersionString() - if version, ok := mopts["version"]; ok { - delete(mopts, "version") + if version, ok := mopts[moptVersion]; ok { + delete(mopts, moptVersion) fsopts.version = version } // Parse the dentry cache limit. fsopts.maxCachedDentries = 1000 - if str, ok := mopts["dentry_cache_limit"]; ok { - delete(mopts, "dentry_cache_limit") + if str, ok := mopts[moptDentryCacheLimit]; ok { + delete(mopts, moptDentryCacheLimit) maxCachedDentries, err := strconv.ParseUint(str, 10, 64) if err != nil { - ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid dentry cache limit: dentry_cache_limit=%s", str) + ctx.Warningf("gofer.FilesystemType.GetFilesystem: invalid dentry cache limit: %s=%s", moptDentryCacheLimit, str) return nil, nil, syserror.EINVAL } fsopts.maxCachedDentries = maxCachedDentries } // Handle simple flags. - if _, ok := mopts["force_page_cache"]; ok { - delete(mopts, "force_page_cache") + if _, ok := mopts[moptForcePageCache]; ok { + delete(mopts, moptForcePageCache) fsopts.forcePageCache = true } - if _, ok := mopts["limit_host_fd_translation"]; ok { - delete(mopts, "limit_host_fd_translation") + if _, ok := mopts[moptLimitHostFDTranslation]; ok { + delete(mopts, moptLimitHostFDTranslation) fsopts.limitHostFDTranslation = true } - if _, ok := mopts["overlayfs_stale_read"]; ok { - delete(mopts, "overlayfs_stale_read") + if _, ok := mopts[moptOverlayfsStaleRead]; ok { + delete(mopts, moptOverlayfsStaleRead) fsopts.overlayfsStaleRead = true } // fsopts.regularFilesUseSpecialFileFD can only be enabled by specifying @@ -469,34 +497,34 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt func getFDFromMountOptionsMap(ctx context.Context, mopts map[string]string) (int, error) { // Check that the transport is "fd". - trans, ok := mopts["trans"] - if !ok || trans != "fd" { - ctx.Warningf("gofer.getFDFromMountOptionsMap: transport must be specified as 'trans=fd'") + trans, ok := mopts[moptTransport] + if !ok || trans != transportModeFD { + ctx.Warningf("gofer.getFDFromMountOptionsMap: transport must be specified as '%s=%s'", moptTransport, transportModeFD) return -1, syserror.EINVAL } - delete(mopts, "trans") + delete(mopts, moptTransport) // Check that read and write FDs are provided and identical. - rfdstr, ok := mopts["rfdno"] + rfdstr, ok := mopts[moptReadFD] if !ok { - ctx.Warningf("gofer.getFDFromMountOptionsMap: read FD must be specified as 'rfdno=<file descriptor>'") + ctx.Warningf("gofer.getFDFromMountOptionsMap: read FD must be specified as '%s=<file descriptor>'", moptReadFD) return -1, syserror.EINVAL } - delete(mopts, "rfdno") + delete(mopts, moptReadFD) rfd, err := strconv.Atoi(rfdstr) if err != nil { - ctx.Warningf("gofer.getFDFromMountOptionsMap: invalid read FD: rfdno=%s", rfdstr) + ctx.Warningf("gofer.getFDFromMountOptionsMap: invalid read FD: %s=%s", moptReadFD, rfdstr) return -1, syserror.EINVAL } - wfdstr, ok := mopts["wfdno"] + wfdstr, ok := mopts[moptWriteFD] if !ok { - ctx.Warningf("gofer.getFDFromMountOptionsMap: write FD must be specified as 'wfdno=<file descriptor>'") + ctx.Warningf("gofer.getFDFromMountOptionsMap: write FD must be specified as '%s=<file descriptor>'", moptWriteFD) return -1, syserror.EINVAL } - delete(mopts, "wfdno") + delete(mopts, moptWriteFD) wfd, err := strconv.Atoi(wfdstr) if err != nil { - ctx.Warningf("gofer.getFDFromMountOptionsMap: invalid write FD: wfdno=%s", wfdstr) + ctx.Warningf("gofer.getFDFromMountOptionsMap: invalid write FD: %s=%s", moptWriteFD, wfdstr) return -1, syserror.EINVAL } if rfd != wfd { diff --git a/pkg/sentry/fsimpl/host/host.go b/pkg/sentry/fsimpl/host/host.go index ad5de80dc..b9cce4181 100644 --- a/pkg/sentry/fsimpl/host/host.go +++ b/pkg/sentry/fsimpl/host/host.go @@ -260,6 +260,11 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe return vfs.PrependPathSyntheticError{} } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return "" +} + // CheckPermissions implements kernfs.Inode.CheckPermissions. func (i *inode) CheckPermissions(ctx context.Context, creds *auth.Credentials, ats vfs.AccessTypes) error { var s unix.Stat_t diff --git a/pkg/sentry/fsimpl/kernfs/kernfs_test.go b/pkg/sentry/fsimpl/kernfs/kernfs_test.go index e63588e33..1cd3137e6 100644 --- a/pkg/sentry/fsimpl/kernfs/kernfs_test.go +++ b/pkg/sentry/fsimpl/kernfs/kernfs_test.go @@ -67,6 +67,11 @@ type filesystem struct { kernfs.Filesystem } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return "" +} + type file struct { kernfs.DynamicBytesFile content string diff --git a/pkg/sentry/fsimpl/overlay/filesystem.go b/pkg/sentry/fsimpl/overlay/filesystem.go index 917709d75..84e37f793 100644 --- a/pkg/sentry/fsimpl/overlay/filesystem.go +++ b/pkg/sentry/fsimpl/overlay/filesystem.go @@ -1764,3 +1764,15 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe defer fs.renameMu.RUnlock() return genericPrependPath(vfsroot, vd.Mount(), vd.Dentry().Impl().(*dentry), b) } + +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + // Return the mount options from the topmost layer. + var vd vfs.VirtualDentry + if fs.opts.UpperRoot.Ok() { + vd = fs.opts.UpperRoot + } else { + vd = fs.opts.LowerRoots[0] + } + return vd.Mount().Filesystem().Impl().MountOptions() +} diff --git a/pkg/sentry/fsimpl/pipefs/pipefs.go b/pkg/sentry/fsimpl/pipefs/pipefs.go index 429733c10..3f05e444e 100644 --- a/pkg/sentry/fsimpl/pipefs/pipefs.go +++ b/pkg/sentry/fsimpl/pipefs/pipefs.go @@ -80,6 +80,11 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe return vfs.PrependPathSyntheticError{} } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return "" +} + // inode implements kernfs.Inode. // // +stateify savable diff --git a/pkg/sentry/fsimpl/proc/filesystem.go b/pkg/sentry/fsimpl/proc/filesystem.go index 8716d0a3c..254a8b062 100644 --- a/pkg/sentry/fsimpl/proc/filesystem.go +++ b/pkg/sentry/fsimpl/proc/filesystem.go @@ -104,6 +104,11 @@ func (fs *filesystem) Release(ctx context.Context) { fs.Filesystem.Release(ctx) } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return fmt.Sprintf("dentry_cache_limit=%d", fs.MaxCachedDentries) +} + // dynamicInode is an overfitted interface for common Inodes with // dynamicByteSource types used in procfs. // diff --git a/pkg/sentry/fsimpl/sockfs/sockfs.go b/pkg/sentry/fsimpl/sockfs/sockfs.go index fda1fa942..735756280 100644 --- a/pkg/sentry/fsimpl/sockfs/sockfs.go +++ b/pkg/sentry/fsimpl/sockfs/sockfs.go @@ -85,6 +85,11 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe return vfs.PrependPathSyntheticError{} } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return "" +} + // inode implements kernfs.Inode. // // +stateify savable diff --git a/pkg/sentry/fsimpl/sys/sys.go b/pkg/sentry/fsimpl/sys/sys.go index dbd9ebdda..1d9280dae 100644 --- a/pkg/sentry/fsimpl/sys/sys.go +++ b/pkg/sentry/fsimpl/sys/sys.go @@ -143,6 +143,11 @@ func (fs *filesystem) Release(ctx context.Context) { fs.Filesystem.Release(ctx) } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return fmt.Sprintf("dentry_cache_limit=%d", fs.MaxCachedDentries) +} + // dir implements kernfs.Inode. // // +stateify savable diff --git a/pkg/sentry/fsimpl/tmpfs/filesystem.go b/pkg/sentry/fsimpl/tmpfs/filesystem.go index 4f675c21e..5fdca1d46 100644 --- a/pkg/sentry/fsimpl/tmpfs/filesystem.go +++ b/pkg/sentry/fsimpl/tmpfs/filesystem.go @@ -898,3 +898,8 @@ func (fs *filesystem) PrependPath(ctx context.Context, vfsroot, vd vfs.VirtualDe d = d.parent } } + +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return fs.mopts +} diff --git a/pkg/sentry/fsimpl/tmpfs/tmpfs.go b/pkg/sentry/fsimpl/tmpfs/tmpfs.go index a01e413e0..8df81f589 100644 --- a/pkg/sentry/fsimpl/tmpfs/tmpfs.go +++ b/pkg/sentry/fsimpl/tmpfs/tmpfs.go @@ -70,6 +70,10 @@ type filesystem struct { // devMinor is the filesystem's minor device number. devMinor is immutable. devMinor uint32 + // mopts contains the tmpfs-specific mount options passed to this + // filesystem. Immutable. + mopts string + // mu serializes changes to the Dentry tree. mu sync.RWMutex `state:"nosave"` @@ -184,6 +188,7 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt mfp: mfp, clock: clock, devMinor: devMinor, + mopts: opts.Data, } fs.vfsfs.Init(vfsObj, newFSType, &fs) diff --git a/pkg/sentry/fsimpl/verity/filesystem.go b/pkg/sentry/fsimpl/verity/filesystem.go index 9057d2b4e..6cb1a23e0 100644 --- a/pkg/sentry/fsimpl/verity/filesystem.go +++ b/pkg/sentry/fsimpl/verity/filesystem.go @@ -590,6 +590,23 @@ func (fs *filesystem) lookupAndVerifyLocked(ctx context.Context, parent *dentry, return nil, err } + // Clear the Merkle tree file if they are to be generated at runtime. + // TODO(b/182315468): Optimize the Merkle tree generate process to + // allow only updating certain files/directories. + if fs.allowRuntimeEnable { + childMerkleFD, err := vfsObj.OpenAt(ctx, fs.creds, &vfs.PathOperation{ + Root: childMerkleVD, + Start: childMerkleVD, + }, &vfs.OpenOptions{ + Flags: linux.O_RDWR | linux.O_TRUNC, + Mode: 0644, + }) + if err != nil { + return nil, err + } + childMerkleFD.DecRef(ctx) + } + // The dentry needs to be cleaned up if any error occurs. IncRef will be // called if a verity child dentry is successfully created. defer childMerkleVD.DecRef(ctx) diff --git a/pkg/sentry/fsimpl/verity/verity.go b/pkg/sentry/fsimpl/verity/verity.go index ec64015cd..0d9b0ee2c 100644 --- a/pkg/sentry/fsimpl/verity/verity.go +++ b/pkg/sentry/fsimpl/verity/verity.go @@ -311,6 +311,24 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt d.DecRef(ctx) return nil, nil, alertIntegrityViolation("Failed to find root Merkle file") } + + // Clear the Merkle tree file if they are to be generated at runtime. + // TODO(b/182315468): Optimize the Merkle tree generate process to + // allow only updating certain files/directories. + if fs.allowRuntimeEnable { + lowerMerkleFD, err := vfsObj.OpenAt(ctx, fs.creds, &vfs.PathOperation{ + Root: lowerMerkleVD, + Start: lowerMerkleVD, + }, &vfs.OpenOptions{ + Flags: linux.O_RDWR | linux.O_TRUNC, + Mode: 0644, + }) + if err != nil { + return nil, nil, err + } + lowerMerkleFD.DecRef(ctx) + } + d.lowerMerkleVD = lowerMerkleVD // Get metadata from the underlying file system. @@ -419,6 +437,11 @@ func (fs *filesystem) Release(ctx context.Context) { fs.lowerMount.DecRef(ctx) } +// MountOptions implements vfs.FilesystemImpl.MountOptions. +func (fs *filesystem) MountOptions() string { + return "" +} + // dentry implements vfs.DentryImpl. // // +stateify savable diff --git a/pkg/sentry/vfs/anonfs.go b/pkg/sentry/vfs/anonfs.go index 7ad0eaf86..3caf417ca 100644 --- a/pkg/sentry/vfs/anonfs.go +++ b/pkg/sentry/vfs/anonfs.go @@ -291,6 +291,11 @@ func (fs *anonFilesystem) PrependPath(ctx context.Context, vfsroot, vd VirtualDe return PrependPathSyntheticError{} } +// MountOptions implements FilesystemImpl.MountOptions. +func (fs *anonFilesystem) MountOptions() string { + return "" +} + // IncRef implements DentryImpl.IncRef. func (d *anonDentry) IncRef() { // no-op diff --git a/pkg/sentry/vfs/filesystem.go b/pkg/sentry/vfs/filesystem.go index 2c4b81e78..059939010 100644 --- a/pkg/sentry/vfs/filesystem.go +++ b/pkg/sentry/vfs/filesystem.go @@ -502,6 +502,15 @@ type FilesystemImpl interface { // // Preconditions: vd.Mount().Filesystem().Impl() == this FilesystemImpl. PrependPath(ctx context.Context, vfsroot, vd VirtualDentry, b *fspath.Builder) error + + // MountOptions returns mount options for the current filesystem. This + // should only return options specific to the filesystem (i.e. don't return + // "ro", "rw", etc). Options should be returned as a comma-separated string, + // similar to the input to the 5th argument to mount. + // + // If the implementation has no filesystem-specific options, it should + // return the empty string. + MountOptions() string } // PrependPathAtVFSRootError is returned by implementations of diff --git a/pkg/sentry/vfs/mount.go b/pkg/sentry/vfs/mount.go index bac9eb905..922f9e697 100644 --- a/pkg/sentry/vfs/mount.go +++ b/pkg/sentry/vfs/mount.go @@ -959,13 +959,17 @@ func manglePath(p string) string { // superBlockOpts returns the super block options string for the the mount at // the given path. func superBlockOpts(mountPath string, mnt *Mount) string { - // gVisor doesn't (yet) have a concept of super block options, so we - // use the ro/rw bit from the mount flag. + // Compose super block options by combining global mount flags with + // FS-specific mount options. opts := "rw" if mnt.ReadOnly() { opts = "ro" } + if mopts := mnt.fs.Impl().MountOptions(); mopts != "" { + opts += "," + mopts + } + // NOTE(b/147673608): If the mount is a cgroup, we also need to include // the cgroup name in the options. For now we just read that from the // path. diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD index e3e289da3..2c3b4058b 100644 --- a/runsc/cmd/BUILD +++ b/runsc/cmd/BUILD @@ -77,6 +77,7 @@ go_test( "delete_test.go", "exec_test.go", "gofer_test.go", + "mitigate_test.go", ], data = [ "//runsc", @@ -91,6 +92,8 @@ go_test( "//pkg/urpc", "//runsc/config", "//runsc/container", + "//runsc/mitigate", + "//runsc/mitigate/mock", "//runsc/specutils", "@com_github_google_go_cmp//cmp:go_default_library", "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", diff --git a/runsc/cmd/mitigate.go b/runsc/cmd/mitigate.go index 822af1917..fddf0e0dd 100644 --- a/runsc/cmd/mitigate.go +++ b/runsc/cmd/mitigate.go @@ -16,6 +16,8 @@ package cmd import ( "context" + "fmt" + "io/ioutil" "github.com/google/subcommands" "gvisor.dev/gvisor/pkg/log" @@ -23,9 +25,23 @@ import ( "gvisor.dev/gvisor/runsc/mitigate" ) +const ( + // cpuInfo is the path used to parse CPU info. + cpuInfo = "/proc/cpuinfo" + // allPossibleCPUs is the path used to enable CPUs. + allPossibleCPUs = "/sys/devices/system/cpu/possible" +) + // Mitigate implements subcommands.Command for the "mitigate" command. type Mitigate struct { - mitigate mitigate.Mitigate + // Run the command without changing the underlying system. + dryRun bool + // Reverse mitigate by turning on all CPU cores. + reverse bool + // Path to file to read to create CPUSet. + path string + // Callback to check if a given thread is vulnerable. + vulnerable func(other mitigate.Thread) bool } // Name implements subcommands.command.name. @@ -38,14 +54,19 @@ func (*Mitigate) Synopsis() string { return "mitigate mitigates the underlying system against side channel attacks" } -// Usage implements subcommands.Command.Usage. -func (m *Mitigate) Usage() string { - return m.mitigate.Usage() +// Usage implments Usage for cmd.Mitigate. +func (m Mitigate) Usage() string { + return `mitigate [flags] + +mitigate mitigates a system to the "MDS" vulnerability by implementing a manual shutdown of SMT. The command checks /proc/cpuinfo for cpus having the MDS vulnerability, and if found, shutdown all but one CPU per hyperthread pair via /sys/devices/system/cpu/cpu{N}/online. CPUs can be restored by writing "2" to each file in /sys/devices/system/cpu/cpu{N}/online or performing a system reboot. + +The command can be reversed with --reverse, which reads the total CPUs from /sys/devices/system/cpu/possible and enables all with /sys/devices/system/cpu/cpu{N}/online.` } -// SetFlags implements subcommands.Command.SetFlags. +// SetFlags sets flags for the command Mitigate. func (m *Mitigate) SetFlags(f *flag.FlagSet) { - m.mitigate.SetFlags(f) + f.BoolVar(&m.dryRun, "dryrun", false, "run the command without changing system") + f.BoolVar(&m.reverse, "reverse", false, "reverse mitigate by enabling all CPUs") } // Execute implements subcommands.Command.Execute. @@ -55,10 +76,97 @@ func (m *Mitigate) Execute(_ context.Context, f *flag.FlagSet, args ...interface return subcommands.ExitUsageError } - if err := m.mitigate.Execute(); err != nil { + m.path = cpuInfo + if m.reverse { + m.path = allPossibleCPUs + } + + m.vulnerable = func(other mitigate.Thread) bool { + return other.IsVulnerable() + } + + if _, err := m.doExecute(); err != nil { log.Warningf("Execute failed: %v", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } + +// Execute executes the Mitigate command. +func (m *Mitigate) doExecute() (mitigate.CPUSet, error) { + if m.dryRun { + log.Infof("Running with DryRun. No cpu settings will be changed.") + } + if m.reverse { + data, err := ioutil.ReadFile(m.path) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %v", m.path, err) + } + + set, err := m.doReverse(data) + if err != nil { + return nil, fmt.Errorf("reverse operation failed: %v", err) + } + return set, nil + } + + data, err := ioutil.ReadFile(m.path) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %v", m.path, err) + } + set, err := m.doMitigate(data) + if err != nil { + return nil, fmt.Errorf("mitigate operation failed: %v", err) + } + return set, nil +} + +func (m *Mitigate) doMitigate(data []byte) (mitigate.CPUSet, error) { + set, err := mitigate.NewCPUSet(data, m.vulnerable) + if err != nil { + return nil, err + } + + log.Infof("Mitigate found the following CPUs...") + log.Infof("%s", set) + + disableList := set.GetShutdownList() + log.Infof("Disabling threads on thread pairs.") + for _, t := range disableList { + log.Infof("Disable thread: %s", t) + if m.dryRun { + continue + } + if err := t.Disable(); err != nil { + return nil, fmt.Errorf("error disabling thread: %s err: %v", t, err) + } + } + log.Infof("Shutdown successful.") + return set, nil +} + +func (m *Mitigate) doReverse(data []byte) (mitigate.CPUSet, error) { + set, err := mitigate.NewCPUSetFromPossible(data) + if err != nil { + return nil, err + } + + log.Infof("Reverse mitigate found the following CPUs...") + log.Infof("%s", set) + + enableList := set.GetRemainingList() + + log.Infof("Enabling all CPUs...") + for _, t := range enableList { + log.Infof("Enabling thread: %s", t) + if m.dryRun { + continue + } + if err := t.Enable(); err != nil { + return nil, fmt.Errorf("error enabling thread: %s err: %v", t, err) + } + } + log.Infof("Enable successful.") + return set, nil +} diff --git a/runsc/cmd/mitigate_test.go b/runsc/cmd/mitigate_test.go new file mode 100644 index 000000000..163fece42 --- /dev/null +++ b/runsc/cmd/mitigate_test.go @@ -0,0 +1,169 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + + "gvisor.dev/gvisor/runsc/mitigate" + "gvisor.dev/gvisor/runsc/mitigate/mock" +) + +type executeTestCase struct { + name string + mitigateData string + mitigateError error + mitigateCPU int + reverseData string + reverseError error + reverseCPU int +} + +func TestExecute(t *testing.T) { + + partial := `processor : 1 +vendor_id : AuthenticAMD +cpu family : 23 +model : 49 +model name : AMD EPYC 7B12 +physical id : 0 +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass +power management: +` + + for _, tc := range []executeTestCase{ + { + name: "CascadeLake4", + mitigateData: mock.CascadeLake4.MakeCPUString(), + mitigateCPU: 2, + reverseData: mock.CascadeLake4.MakeSysPossibleString(), + reverseCPU: 4, + }, + { + name: "Empty", + mitigateData: "", + mitigateError: fmt.Errorf(`mitigate operation failed: no cpus found for: ""`), + reverseData: "", + reverseError: fmt.Errorf(`reverse operation failed: mismatch regex from possible: ""`), + }, + { + name: "Partial", + mitigateData: `processor : 0 +vendor_id : AuthenticAMD +cpu family : 23 +model : 49 +model name : AMD EPYC 7B12 +physical id : 0 +core id : 0 +cpu cores : 1 +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass +power management::84 + +` + partial, + mitigateError: fmt.Errorf(`mitigate operation failed: failed to match key "core id": %q`, partial), + reverseData: "1-", + reverseError: fmt.Errorf(`reverse operation failed: mismatch regex from possible: %q`, "1-"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + m := &Mitigate{ + dryRun: true, + vulnerable: func(other mitigate.Thread) bool { + return other.IsVulnerable() + }, + } + m.doExecuteTest(t, "Mitigate", tc.mitigateData, tc.mitigateCPU, tc.mitigateError) + + m.reverse = true + m.doExecuteTest(t, "Reverse", tc.reverseData, tc.reverseCPU, tc.reverseError) + }) + } +} + +func TestExecuteSmoke(t *testing.T) { + smokeMitigate, err := ioutil.ReadFile(cpuInfo) + if err != nil { + t.Fatalf("Failed to read %s: %v", cpuInfo, err) + } + + m := &Mitigate{ + dryRun: true, + vulnerable: func(other mitigate.Thread) bool { + return other.IsVulnerable() + }, + } + + m.doExecuteTest(t, "Mitigate", string(smokeMitigate), 0, nil) + + smokeReverse, err := ioutil.ReadFile(allPossibleCPUs) + if err != nil { + t.Fatalf("Failed to read %s: %v", allPossibleCPUs, err) + } + + m.reverse = true + m.doExecuteTest(t, "Reverse", string(smokeReverse), 0, nil) +} + +// doExecuteTest runs Execute with the mitigate operation and reverse operation. +func (m *Mitigate) doExecuteTest(t *testing.T, name, data string, want int, wantErr error) { + t.Run(name, func(t *testing.T) { + file, err := ioutil.TempFile("", "outfile.txt") + if err != nil { + t.Fatalf("Failed to create tmpfile: %v", err) + } + defer os.Remove(file.Name()) + + if _, err := file.WriteString(data); err != nil { + t.Fatalf("Failed to write to file: %v", err) + } + + // Set fields for mitigate and dryrun to keep test hermetic. + m.path = file.Name() + + set, err := m.doExecute() + if err = checkErr(wantErr, err); err != nil { + t.Fatalf("Mitigate error mismatch: %v", err) + } + + // case where test should end in error or we don't care + // about how many cpus are returned. + if wantErr != nil || want < 1 { + return + } + got := len(set.GetRemainingList()) + if want != got { + t.Fatalf("Failed wrong number of remaining CPUs: want %d, got %d", want, got) + } + + }) +} + +// checkErr checks error for equality. +func checkErr(want, got error) error { + switch { + case want == nil && got == nil: + case want != nil && got == nil: + fallthrough + case want == nil && got != nil: + fallthrough + case want.Error() != strings.Trim(got.Error(), " "): + return fmt.Errorf("got: %v want: %v", got, want) + } + return nil +} diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index a15de02c7..5a0c468a4 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -1601,12 +1601,12 @@ func TestReadonlyRoot(t *testing.T) { } // Read mounts to check that root is readonly. - out, err := executeCombinedOutput(c, "/bin/sh", "-c", "mount | grep ' / '") + out, err := executeCombinedOutput(c, "/bin/sh", "-c", "mount | grep ' / ' | grep -o -e '(.*)'") if err != nil { t.Fatalf("exec failed: %v", err) } - t.Logf("root mount: %q", out) - if !strings.Contains(string(out), "(ro)") { + t.Logf("root mount options: %q", out) + if !strings.Contains(string(out), "ro") { t.Errorf("root not mounted readonly: %q", out) } @@ -1659,13 +1659,13 @@ func TestReadonlyMount(t *testing.T) { } // Read mounts to check that volume is readonly. - cmd := fmt.Sprintf("mount | grep ' %s '", dir) + cmd := fmt.Sprintf("mount | grep ' %s ' | grep -o -e '(.*)'", dir) out, err := executeCombinedOutput(c, "/bin/sh", "-c", cmd) if err != nil { t.Fatalf("exec failed, err: %v", err) } - t.Logf("mount: %q", out) - if !strings.Contains(string(out), "(ro)") { + t.Logf("mount options: %q", out) + if !strings.Contains(string(out), "ro") { t.Errorf("volume not mounted readonly: %q", out) } diff --git a/runsc/mitigate/BUILD b/runsc/mitigate/BUILD index 561854e66..1238890fc 100644 --- a/runsc/mitigate/BUILD +++ b/runsc/mitigate/BUILD @@ -4,28 +4,20 @@ package(licenses = ["notice"]) go_library( name = "mitigate", - srcs = [ - "cpu.go", - "mitigate.go", - "mitigate_conf.go", - ], + srcs = ["mitigate.go"], visibility = [ "//runsc:__subpackages__", ], - deps = [ - "//pkg/log", - "//runsc/flag", - "@in_gopkg_yaml_v2//:go_default_library", - ], + deps = ["@in_gopkg_yaml_v2//:go_default_library"], ) go_test( name = "mitigate_test", size = "small", - srcs = [ - "cpu_test.go", - "mitigate_test.go", - ], + srcs = ["mitigate_test.go"], library = ":mitigate", - deps = ["@com_github_google_go_cmp//cmp:go_default_library"], + deps = [ + "//runsc/mitigate/mock", + "@com_github_google_go_cmp//cmp:go_default_library", + ], ) diff --git a/runsc/mitigate/cpu.go b/runsc/mitigate/cpu.go deleted file mode 100644 index 4b2aa351f..000000000 --- a/runsc/mitigate/cpu.go +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mitigate - -import ( - "fmt" - "io/ioutil" - "regexp" - "strconv" - "strings" -) - -const ( - // mds is the only bug we care about. - mds = "mds" - - // Constants for parsing /proc/cpuinfo. - processorKey = "processor" - vendorIDKey = "vendor_id" - cpuFamilyKey = "cpu family" - modelKey = "model" - physicalIDKey = "physical id" - coreIDKey = "core id" - bugsKey = "bugs" - - // Path to shutdown a CPU. - cpuOnlineTemplate = "/sys/devices/system/cpu/cpu%d/online" -) - -// cpuSet contains a map of all CPUs on the system, mapped -// by Physical ID and CoreIDs. threads with the same -// Core and Physical ID are Hyperthread pairs. -type cpuSet map[cpuID]*threadGroup - -// newCPUSet creates a CPUSet from data read from /proc/cpuinfo. -func newCPUSet(data []byte, vulnerable func(thread) bool) (cpuSet, error) { - processors, err := getThreads(string(data)) - if err != nil { - return nil, err - } - - set := make(cpuSet) - for _, p := range processors { - // Each ID is of the form physicalID:coreID. Hyperthread pairs - // have identical physical and core IDs. We need to match - // Hyperthread pairs so that we can shutdown all but one per - // pair. - core, ok := set[p.id] - if !ok { - core = &threadGroup{} - set[p.id] = core - } - core.isVulnerable = core.isVulnerable || vulnerable(p) - core.threads = append(core.threads, p) - } - return set, nil -} - -// newCPUSetFromPossible makes a cpuSet data read from -// /sys/devices/system/cpu/possible. This is used in enable operations -// where the caller simply wants to enable all CPUS. -func newCPUSetFromPossible(data []byte) (cpuSet, error) { - threads, err := getThreadsFromPossible(data) - if err != nil { - return nil, err - } - - // We don't care if a CPU is vulnerable or not, we just - // want to return a list of all CPUs on the host. - set := cpuSet{ - threads[0].id: &threadGroup{ - threads: threads, - isVulnerable: false, - }, - } - return set, nil -} - -// String implements the String method for CPUSet. -func (c cpuSet) String() string { - ret := "" - for _, tg := range c { - ret += fmt.Sprintf("%s\n", tg) - } - return ret -} - -// getRemainingList returns the list of threads that will remain active -// after mitigation. -func (c cpuSet) getRemainingList() []thread { - threads := make([]thread, 0, len(c)) - for _, core := range c { - // If we're vulnerable, take only one thread from the pair. - if core.isVulnerable { - threads = append(threads, core.threads[0]) - continue - } - // Otherwise don't shutdown anything. - threads = append(threads, core.threads...) - } - return threads -} - -// getShutdownList returns the list of threads that will be shutdown on -// mitigation. -func (c cpuSet) getShutdownList() []thread { - threads := make([]thread, 0) - for _, core := range c { - // Only if we're vulnerable do shutdown anything. In this case, - // shutdown all but the first entry. - if core.isVulnerable && len(core.threads) > 1 { - threads = append(threads, core.threads[1:]...) - } - } - return threads -} - -// threadGroup represents Hyperthread pairs on the same physical/core ID. -type threadGroup struct { - threads []thread - isVulnerable bool -} - -// String implements the String method for threadGroup. -func (c threadGroup) String() string { - ret := fmt.Sprintf("ThreadGroup:\nIsVulnerable: %t\n", c.isVulnerable) - for _, processor := range c.threads { - ret += fmt.Sprintf("%s\n", processor) - } - return ret -} - -// getThreads returns threads structs from reading /proc/cpuinfo. -func getThreads(data string) ([]thread, error) { - // Each processor entry should start with the - // processor key. Find the beginings of each. - r := buildRegex(processorKey, `\d+`) - indices := r.FindAllStringIndex(data, -1) - if len(indices) < 1 { - return nil, fmt.Errorf("no cpus found for: %q", data) - } - - // Add the ending index for last entry. - indices = append(indices, []int{len(data), -1}) - - // Valid cpus are now defined by strings in between - // indexes (e.g. data[index[i], index[i+1]]). - // There should be len(indicies) - 1 CPUs - // since the last index is the end of the string. - cpus := make([]thread, 0, len(indices)) - // Find each string that represents a CPU. These begin "processor". - for i := 1; i < len(indices); i++ { - start := indices[i-1][0] - end := indices[i][0] - // Parse the CPU entry, which should be between start/end. - c, err := newThread(data[start:end]) - if err != nil { - return nil, err - } - cpus = append(cpus, c) - } - return cpus, nil -} - -// getThreadsFromPossible makes threads from data read from /sys/devices/system/cpu/possible. -func getThreadsFromPossible(data []byte) ([]thread, error) { - possibleRegex := regexp.MustCompile(`(?m)^(\d+)(-(\d+))?$`) - matches := possibleRegex.FindStringSubmatch(string(data)) - if len(matches) != 4 { - return nil, fmt.Errorf("mismatch regex from %s: %q", allPossibleCPUs, string(data)) - } - - // If matches[3] is empty, we only have one cpu entry. - if matches[3] == "" { - matches[3] = matches[1] - } - - begin, err := strconv.ParseInt(matches[1], 10, 64) - if err != nil { - return nil, fmt.Errorf("failed to parse begin: %v", err) - } - end, err := strconv.ParseInt(matches[3], 10, 64) - if err != nil { - return nil, fmt.Errorf("failed to parse end: %v", err) - } - if begin > end || begin < 0 || end < 0 { - return nil, fmt.Errorf("invalid cpu bounds from possible: begin: %d end: %d", begin, end) - } - - ret := make([]thread, 0, end-begin) - for i := begin; i <= end; i++ { - ret = append(ret, thread{ - processorNumber: i, - id: cpuID{ - physicalID: 0, // we don't care about id for enable ops. - coreID: 0, - }, - }) - } - - return ret, nil -} - -// cpuID for each thread is defined by the physical and -// core IDs. If equal, two threads are Hyperthread pairs. -type cpuID struct { - physicalID int64 - coreID int64 -} - -// type cpu represents pertinent info about a cpu. -type thread struct { - processorNumber int64 // the processor number of this CPU. - vendorID string // the vendorID of CPU (e.g. AuthenticAMD). - cpuFamily int64 // CPU family number (e.g. 6 for CascadeLake/Skylake). - model int64 // CPU model number (e.g. 85 for CascadeLake/Skylake). - id cpuID // id for this thread - bugs map[string]struct{} // map of vulnerabilities parsed from the 'bugs' field. -} - -// newThread parses a CPU from a single cpu entry from /proc/cpuinfo. -func newThread(data string) (thread, error) { - empty := thread{} - processor, err := parseProcessor(data) - if err != nil { - return empty, err - } - - vendorID, err := parseVendorID(data) - if err != nil { - return empty, err - } - - cpuFamily, err := parseCPUFamily(data) - if err != nil { - return empty, err - } - - model, err := parseModel(data) - if err != nil { - return empty, err - } - - physicalID, err := parsePhysicalID(data) - if err != nil { - return empty, err - } - - coreID, err := parseCoreID(data) - if err != nil { - return empty, err - } - - bugs, err := parseBugs(data) - if err != nil { - return empty, err - } - - return thread{ - processorNumber: processor, - vendorID: vendorID, - cpuFamily: cpuFamily, - model: model, - id: cpuID{ - physicalID: physicalID, - coreID: coreID, - }, - bugs: bugs, - }, nil -} - -// String implements the String method for thread. -func (t thread) String() string { - template := `CPU: %d -CPU ID: %+v -Vendor: %s -Family/Model: %d/%d -Bugs: %s -` - bugs := make([]string, 0) - for bug := range t.bugs { - bugs = append(bugs, bug) - } - - return fmt.Sprintf(template, t.processorNumber, t.id, t.vendorID, t.cpuFamily, t.model, strings.Join(bugs, ",")) -} - -// enable turns on the CPU by writing 1 to /sys/devices/cpu/cpu{N}/online. -func (t thread) enable() error { - cpuPath := fmt.Sprintf(cpuOnlineTemplate, t.processorNumber) - return ioutil.WriteFile(cpuPath, []byte{'1'}, 0644) -} - -// disable turns off the CPU by writing 0 to /sys/devices/cpu/cpu{N}/online. -func (t thread) disable() error { - cpuPath := fmt.Sprintf(cpuOnlineTemplate, t.processorNumber) - return ioutil.WriteFile(cpuPath, []byte{'0'}, 0644) -} - -// isVulnerable checks if a CPU is vulnerable to mds. -func (t thread) isVulnerable() bool { - _, ok := t.bugs[mds] - return ok -} - -// isActive checks if a CPU is active from /sys/devices/system/cpu/cpu{N}/online -// If the file does not exist (ioutil returns in error), we assume the CPU is on. -func (t thread) isActive() bool { - cpuPath := fmt.Sprintf(cpuOnlineTemplate, t.processorNumber) - data, err := ioutil.ReadFile(cpuPath) - if err != nil { - return true - } - return len(data) > 0 && data[0] != '0' -} - -// similarTo checks family/model/bugs fields for equality of two -// processors. -func (t thread) similarTo(other thread) bool { - if t.vendorID != other.vendorID { - return false - } - - if other.cpuFamily != t.cpuFamily { - return false - } - - if other.model != t.model { - return false - } - - if len(other.bugs) != len(t.bugs) { - return false - } - - for bug := range t.bugs { - if _, ok := other.bugs[bug]; !ok { - return false - } - } - return true -} - -// parseProcessor grabs the processor field from /proc/cpuinfo output. -func parseProcessor(data string) (int64, error) { - return parseIntegerResult(data, processorKey) -} - -// parseVendorID grabs the vendor_id field from /proc/cpuinfo output. -func parseVendorID(data string) (string, error) { - return parseRegex(data, vendorIDKey, `[\w\d]+`) -} - -// parseCPUFamily grabs the cpu family field from /proc/cpuinfo output. -func parseCPUFamily(data string) (int64, error) { - return parseIntegerResult(data, cpuFamilyKey) -} - -// parseModel grabs the model field from /proc/cpuinfo output. -func parseModel(data string) (int64, error) { - return parseIntegerResult(data, modelKey) -} - -// parsePhysicalID parses the physical id field. -func parsePhysicalID(data string) (int64, error) { - return parseIntegerResult(data, physicalIDKey) -} - -// parseCoreID parses the core id field. -func parseCoreID(data string) (int64, error) { - return parseIntegerResult(data, coreIDKey) -} - -// parseBugs grabs the bugs field from /proc/cpuinfo output. -func parseBugs(data string) (map[string]struct{}, error) { - result, err := parseRegex(data, bugsKey, `[\d\w\s]*`) - if err != nil { - return nil, err - } - bugs := strings.Split(result, " ") - ret := make(map[string]struct{}, len(bugs)) - for _, bug := range bugs { - ret[bug] = struct{}{} - } - return ret, nil -} - -// parseIntegerResult parses fields expecting an integer. -func parseIntegerResult(data, key string) (int64, error) { - result, err := parseRegex(data, key, `\d+`) - if err != nil { - return 0, err - } - return strconv.ParseInt(result, 0, 64) -} - -// buildRegex builds a regex for parsing each CPU field. -func buildRegex(key, match string) *regexp.Regexp { - reg := fmt.Sprintf(`(?m)^%s\s*:\s*(.*)$`, key) - return regexp.MustCompile(reg) -} - -// parseRegex parses data with key inserted into a standard regex template. -func parseRegex(data, key, match string) (string, error) { - r := buildRegex(key, match) - matches := r.FindStringSubmatch(data) - if len(matches) < 2 { - return "", fmt.Errorf("failed to match key %q: %q", key, data) - } - return matches[1], nil -} diff --git a/runsc/mitigate/cpu_test.go b/runsc/mitigate/cpu_test.go deleted file mode 100644 index 374333465..000000000 --- a/runsc/mitigate/cpu_test.go +++ /dev/null @@ -1,605 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mitigate - -import ( - "fmt" - "io/ioutil" - "strings" - "testing" -) - -// mockCPU represents data from CPUs that will be mitigated. -type mockCPU struct { - name string - vendorID string - family int - model int - modelName string - bugs string - physicalCores int - cores int - threadsPerCore int -} - -var cascadeLake4 = mockCPU{ - name: "CascadeLake", - vendorID: "GenuineIntel", - family: 6, - model: 85, - modelName: "Intel(R) Xeon(R) CPU", - bugs: "spectre_v1 spectre_v2 spec_store_bypass mds swapgs taa", - physicalCores: 1, - cores: 2, - threadsPerCore: 2, -} - -var haswell2 = mockCPU{ - name: "Haswell", - vendorID: "GenuineIntel", - family: 6, - model: 63, - modelName: "Intel(R) Xeon(R) CPU", - bugs: "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs", - physicalCores: 1, - cores: 1, - threadsPerCore: 2, -} - -var haswell2core = mockCPU{ - name: "Haswell2Physical", - vendorID: "GenuineIntel", - family: 6, - model: 63, - modelName: "Intel(R) Xeon(R) CPU", - bugs: "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs", - physicalCores: 2, - cores: 1, - threadsPerCore: 1, -} - -var amd8 = mockCPU{ - name: "AMD", - vendorID: "AuthenticAMD", - family: 23, - model: 49, - modelName: "AMD EPYC 7B12", - bugs: "sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass", - physicalCores: 4, - cores: 1, - threadsPerCore: 2, -} - -// makeCPUString makes a string formated like /proc/cpuinfo for each cpuTestCase -func (tc mockCPU) makeCPUString() string { - template := `processor : %d -vendor_id : %s -cpu family : %d -model : %d -model name : %s -physical id : %d -core id : %d -cpu cores : %d -bugs : %s -` - ret := `` - for i := 0; i < tc.physicalCores; i++ { - for j := 0; j < tc.cores; j++ { - for k := 0; k < tc.threadsPerCore; k++ { - processorNum := (i*tc.cores+j)*tc.threadsPerCore + k - ret += fmt.Sprintf(template, - processorNum, /*processor*/ - tc.vendorID, /*vendor_id*/ - tc.family, /*cpu family*/ - tc.model, /*model*/ - tc.modelName, /*model name*/ - i, /*physical id*/ - j, /*core id*/ - tc.cores*tc.physicalCores, /*cpu cores*/ - tc.bugs /*bugs*/) - } - } - } - return ret -} - -func (tc mockCPU) makeSysPossibleString() string { - max := tc.physicalCores * tc.cores * tc.threadsPerCore - if max == 1 { - return "0" - } - return fmt.Sprintf("0-%d", max-1) -} - -// TestMockCPUSet tests mock cpu test cases against the cpuSet functions. -func TestMockCPUSet(t *testing.T) { - for _, tc := range []struct { - testCase mockCPU - isVulnerable bool - }{ - { - testCase: amd8, - isVulnerable: false, - }, - { - testCase: haswell2, - isVulnerable: true, - }, - { - testCase: haswell2core, - isVulnerable: true, - }, - - { - testCase: cascadeLake4, - isVulnerable: true, - }, - } { - t.Run(tc.testCase.name, func(t *testing.T) { - data := tc.testCase.makeCPUString() - vulnerable := func(t thread) bool { - return t.isVulnerable() - } - set, err := newCPUSet([]byte(data), vulnerable) - if err != nil { - t.Fatalf("Failed to ") - } - remaining := set.getRemainingList() - // In the non-vulnerable case, no cores should be shutdown so all should remain. - want := tc.testCase.physicalCores * tc.testCase.cores * tc.testCase.threadsPerCore - if tc.isVulnerable { - want = tc.testCase.physicalCores * tc.testCase.cores - } - - if want != len(remaining) { - t.Fatalf("Failed to shutdown the correct number of cores: want: %d got: %d", want, len(remaining)) - } - - if !tc.isVulnerable { - return - } - - // If the set is vulnerable, we expect only 1 thread per hyperthread pair. - for _, r := range remaining { - if _, ok := set[r.id]; !ok { - t.Fatalf("Entry %+v not in map, there must be two entries in the same thread group.", r) - } - delete(set, r.id) - } - - possible := tc.testCase.makeSysPossibleString() - set, err = newCPUSetFromPossible([]byte(possible)) - if err != nil { - t.Fatalf("Failed to make cpuSet: %v", err) - } - - want = tc.testCase.physicalCores * tc.testCase.cores * tc.testCase.threadsPerCore - got := len(set.getRemainingList()) - if got != want { - t.Fatalf("Returned the wrong number of CPUs want: %d got: %d", want, got) - } - }) - } -} - -// TestGetCPU tests basic parsing of single CPU strings from reading -// /proc/cpuinfo. -func TestGetCPU(t *testing.T) { - data := `processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 85 -physical id: 0 -core id : 0 -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit -` - want := thread{ - processorNumber: 0, - vendorID: "GenuineIntel", - cpuFamily: 6, - model: 85, - id: cpuID{ - physicalID: 0, - coreID: 0, - }, - bugs: map[string]struct{}{ - "cpu_meltdown": struct{}{}, - "spectre_v1": struct{}{}, - "spectre_v2": struct{}{}, - "spec_store_bypass": struct{}{}, - "l1tf": struct{}{}, - "mds": struct{}{}, - "swapgs": struct{}{}, - "taa": struct{}{}, - "itlb_multihit": struct{}{}, - }, - } - - got, err := newThread(data) - if err != nil { - t.Fatalf("getCpu failed with error: %v", err) - } - - if !want.similarTo(got) { - t.Fatalf("Failed cpus not similar: got: %+v, want: %+v", got, want) - } - - if !got.isVulnerable() { - t.Fatalf("Failed: cpu should be vulnerable.") - } -} - -func TestInvalid(t *testing.T) { - result, err := getThreads(`something not a processor`) - if err == nil { - t.Fatalf("getCPU set didn't return an error: %+v", result) - } - - if !strings.Contains(err.Error(), "no cpus") { - t.Fatalf("Incorrect error returned: %v", err) - } -} - -// TestCPUSet tests getting the right number of CPUs from -// parsing full output of /proc/cpuinfo. -func TestCPUSet(t *testing.T) { - data := `processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 63 -model name : Intel(R) Xeon(R) CPU @ 2.30GHz -stepping : 0 -microcode : 0x1 -cpu MHz : 2299.998 -cache size : 46080 KB -physical id : 0 -siblings : 2 -core id : 0 -cpu cores : 1 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat md_clear arch_capabilities -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs -bogomips : 4599.99 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 63 -model name : Intel(R) Xeon(R) CPU @ 2.30GHz -stepping : 0 -microcode : 0x1 -cpu MHz : 2299.998 -cache size : 46080 KB -physical id : 0 -siblings : 2 -core id : 0 -cpu cores : 1 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat md_clear arch_capabilities -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs -bogomips : 4599.99 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: -` - cpuSet, err := getThreads(data) - if err != nil { - t.Fatalf("getCPUSet failed: %v", err) - } - - wantCPULen := 2 - if len(cpuSet) != wantCPULen { - t.Fatalf("Num CPU mismatch: want: %d, got: %d", wantCPULen, len(cpuSet)) - } - - wantCPU := thread{ - vendorID: "GenuineIntel", - cpuFamily: 6, - model: 63, - bugs: map[string]struct{}{ - "cpu_meltdown": struct{}{}, - "spectre_v1": struct{}{}, - "spectre_v2": struct{}{}, - "spec_store_bypass": struct{}{}, - "l1tf": struct{}{}, - "mds": struct{}{}, - "swapgs": struct{}{}, - }, - } - - for _, c := range cpuSet { - if !wantCPU.similarTo(c) { - t.Fatalf("Failed cpus not equal: got: %+v, want: %+v", c, wantCPU) - } - } -} - -// TestReadFile is a smoke test for parsing methods. -func TestReadFile(t *testing.T) { - data, err := ioutil.ReadFile("/proc/cpuinfo") - if err != nil { - t.Fatalf("Failed to read cpuinfo: %v", err) - } - - vulnerable := func(t thread) bool { - return t.isVulnerable() - } - - set, err := newCPUSet(data, vulnerable) - if err != nil { - t.Fatalf("Failed to parse CPU data %v\n%s", err, data) - } - - if len(set) < 1 { - t.Fatalf("Failed to parse any CPUs: %d", len(set)) - } - - t.Log(set) -} - -// TestVulnerable tests if the isVulnerable method is correct -// among known CPUs in GCP. -func TestVulnerable(t *testing.T) { - const haswell = `processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 63 -model name : Intel(R) Xeon(R) CPU @ 2.30GHz -stepping : 0 -microcode : 0x1 -cpu MHz : 2299.998 -cache size : 46080 KB -physical id : 0 -siblings : 4 -core id : 0 -cpu cores : 2 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat md_clear arch_capabilities -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs -bogomips : 4599.99 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management:` - - const skylake = `processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 85 -model name : Intel(R) Xeon(R) CPU @ 2.00GHz -stepping : 3 -microcode : 0x1 -cpu MHz : 2000.180 -cache size : 39424 KB -physical id : 0 -siblings : 2 -core id : 0 -cpu cores : 1 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat md_clear arch_capabilities -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa -bogomips : 4000.36 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management:` - - const cascade = `processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 85 -model name : Intel(R) Xeon(R) CPU -stepping : 7 -microcode : 0x1 -cpu MHz : 2800.198 -cache size : 33792 KB -physical id : 0 -siblings : 2 -core id : 0 -cpu cores : 1 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 - ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmu -lqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowpr -efetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid r -tm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves a -rat avx512_vnni md_clear arch_capabilities -bugs : spectre_v1 spectre_v2 spec_store_bypass mds swapgs taa -bogomips : 5600.39 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management:` - - const amd = `processor : 0 -vendor_id : AuthenticAMD -cpu family : 23 -model : 49 -model name : AMD EPYC 7B12 -stepping : 0 -microcode : 0x1000065 -cpu MHz : 2250.000 -cache size : 512 KB -physical id : 0 -siblings : 2 -core id : 0 -cpu cores : 1 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext ssbd ibrs ibpb stibp vmmcall fsgsbase tsc_adjust bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr arat npt nrip_save umip rdpid -bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass -bogomips : 4500.00 -TLB size : 3072 4K pages -clflush size : 64 -cache_alignment : 64 -address sizes : 48 bits physical, 48 bits virtual -power management:` - - for _, tc := range []struct { - name string - cpuString string - vulnerable bool - }{ - { - name: "haswell", - cpuString: haswell, - vulnerable: true, - }, { - name: "skylake", - cpuString: skylake, - vulnerable: true, - }, { - name: "amd", - cpuString: amd, - vulnerable: false, - }, - } { - t.Run(tc.name, func(t *testing.T) { - set, err := getThreads(tc.cpuString) - if err != nil { - t.Fatalf("Failed to getCPUSet:%v\n %s", err, tc.cpuString) - } - - if len(set) < 1 { - t.Fatalf("Returned empty cpu set: %v", set) - } - - for _, c := range set { - got := func() bool { - return c.isVulnerable() - }() - - if got != tc.vulnerable { - t.Fatalf("Mismatch vulnerable for cpu %+s: got %t want: %t", tc.name, tc.vulnerable, got) - } - } - }) - } -} - -func TestReverse(t *testing.T) { - const noParse = "-1-" - for _, tc := range []struct { - name string - output string - wantErr error - wantCount int - }{ - { - name: "base", - output: "0-7", - wantErr: nil, - wantCount: 8, - }, - { - name: "huge", - output: "0-111", - wantErr: nil, - wantCount: 112, - }, - { - name: "not zero", - output: "50-53", - wantErr: nil, - wantCount: 4, - }, - { - name: "small", - output: "0", - wantErr: nil, - wantCount: 1, - }, - { - name: "invalid order", - output: "10-6", - wantErr: fmt.Errorf("invalid cpu bounds from possible: begin: %d end: %d", 10, 6), - }, - { - name: "no parse", - output: noParse, - wantErr: fmt.Errorf(`mismatch regex from /sys/devices/system/cpu/possible: %q`, noParse), - }, - } { - t.Run(tc.name, func(t *testing.T) { - threads, err := getThreadsFromPossible([]byte(tc.output)) - - switch { - case tc.wantErr == nil: - if err != nil { - t.Fatalf("Wanted nil err, got: %v", err) - } - case err == nil: - t.Fatalf("Want error: %v got: %v", tc.wantErr, err) - default: - if tc.wantErr.Error() != err.Error() { - t.Fatalf("Want error: %v got error: %v", tc.wantErr, err) - } - } - - if len(threads) != tc.wantCount { - t.Fatalf("Want count: %d got: %d", tc.wantCount, len(threads)) - } - }) - } -} - -func TestReverseSmoke(t *testing.T) { - data, err := ioutil.ReadFile(allPossibleCPUs) - if err != nil { - t.Fatalf("Failed to read from possible: %v", err) - } - threads, err := getThreadsFromPossible(data) - if err != nil { - t.Fatalf("Could not parse possible output: %v", err) - } - - if len(threads) <= 0 { - t.Fatalf("Didn't get any CPU cores: %d", len(threads)) - } -} diff --git a/runsc/mitigate/mitigate.go b/runsc/mitigate/mitigate.go index 91de623e3..24f67414c 100644 --- a/runsc/mitigate/mitigate.go +++ b/runsc/mitigate/mitigate.go @@ -14,121 +14,440 @@ // Package mitigate provides libraries for the mitigate command. The // mitigate command mitigates side channel attacks such as MDS. Mitigate -// shuts down CPUs via /sys/devices/system/cpu/cpu{N}/online. In addition, -// the mitigate also handles computing available CPU in kubernetes kube_config -// files. +// shuts down CPUs via /sys/devices/system/cpu/cpu{N}/online. package mitigate import ( "fmt" "io/ioutil" - - "gvisor.dev/gvisor/pkg/log" - "gvisor.dev/gvisor/runsc/flag" + "os" + "regexp" + "sort" + "strconv" + "strings" ) const ( - cpuInfo = "/proc/cpuinfo" - allPossibleCPUs = "/sys/devices/system/cpu/possible" + // mds is the only bug we care about. + mds = "mds" + + // Constants for parsing /proc/cpuinfo. + processorKey = "processor" + vendorIDKey = "vendor_id" + cpuFamilyKey = "cpu family" + modelKey = "model" + physicalIDKey = "physical id" + coreIDKey = "core id" + bugsKey = "bugs" + + // Path to shutdown a CPU. + cpuOnlineTemplate = "/sys/devices/system/cpu/cpu%d/online" ) -// Mitigate handles high level mitigate operations provided to runsc. -type Mitigate struct { - dryRun bool // Run the command without changing the underlying system. - reverse bool // Reverse mitigate by turning on all CPU cores. - other mitigate // Struct holds extra mitigate logic. - path string // path to read for each operation (e.g. /proc/cpuinfo). +// CPUSet contains a map of all CPUs on the system, mapped +// by Physical ID and CoreIDs. threads with the same +// Core and Physical ID are Hyperthread pairs. +type CPUSet map[threadID]*ThreadGroup + +// NewCPUSet creates a CPUSet from data read from /proc/cpuinfo. +func NewCPUSet(data []byte, vulnerable func(Thread) bool) (CPUSet, error) { + processors, err := getThreads(string(data)) + if err != nil { + return nil, err + } + + set := make(CPUSet) + for _, p := range processors { + // Each ID is of the form physicalID:coreID. Hyperthread pairs + // have identical physical and core IDs. We need to match + // Hyperthread pairs so that we can shutdown all but one per + // pair. + core, ok := set[p.id] + if !ok { + core = &ThreadGroup{} + set[p.id] = core + } + core.isVulnerable = core.isVulnerable || vulnerable(p) + core.threads = append(core.threads, p) + } + + // We need to make sure we shutdown the lowest number processor per + // thread group. + for _, tg := range set { + sort.Slice(tg.threads, func(i, j int) bool { + return tg.threads[i].processorNumber < tg.threads[j].processorNumber + }) + } + return set, nil } -// Usage implments Usage for cmd.Mitigate. -func (m Mitigate) Usage() string { - usageString := `mitigate [flags] +// NewCPUSetFromPossible makes a cpuSet data read from +// /sys/devices/system/cpu/possible. This is used in enable operations +// where the caller simply wants to enable all CPUS. +func NewCPUSetFromPossible(data []byte) (CPUSet, error) { + threads, err := GetThreadsFromPossible(data) + if err != nil { + return nil, err + } + + // We don't care if a CPU is vulnerable or not, we just + // want to return a list of all CPUs on the host. + set := CPUSet{ + threads[0].id: &ThreadGroup{ + threads: threads, + isVulnerable: false, + }, + } + return set, nil +} -Mitigate mitigates a system to the "MDS" vulnerability by implementing a manual shutdown of SMT. The command checks /proc/cpuinfo for cpus having the MDS vulnerability, and if found, shutdown all but one CPU per hyperthread pair via /sys/devices/system/cpu/cpu{N}/online. CPUs can be restored by writing "2" to each file in /sys/devices/system/cpu/cpu{N}/online or performing a system reboot. +// String implements the String method for CPUSet. +func (c CPUSet) String() string { + ret := "" + for _, tg := range c { + ret += fmt.Sprintf("%s\n", tg) + } + return ret +} -The command can be reversed with --reverse, which reads the total CPUs from /sys/devices/system/cpu/possible and enables all with /sys/devices/system/cpu/cpu{N}/online. -` - return usageString + m.other.usage() +// GetRemainingList returns the list of threads that will remain active +// after mitigation. +func (c CPUSet) GetRemainingList() []Thread { + threads := make([]Thread, 0, len(c)) + for _, core := range c { + // If we're vulnerable, take only one thread from the pair. + if core.isVulnerable { + threads = append(threads, core.threads[0]) + continue + } + // Otherwise don't shutdown anything. + threads = append(threads, core.threads...) + } + return threads } -// SetFlags sets flags for the command Mitigate. -func (m Mitigate) SetFlags(f *flag.FlagSet) { - f.BoolVar(&m.dryRun, "dryrun", false, "run the command without changing system") - f.BoolVar(&m.reverse, "reverse", false, "reverse mitigate by enabling all CPUs") - m.other.setFlags(f) - m.path = cpuInfo - if m.reverse { - m.path = allPossibleCPUs +// GetShutdownList returns the list of threads that will be shutdown on +// mitigation. +func (c CPUSet) GetShutdownList() []Thread { + threads := make([]Thread, 0) + for _, core := range c { + // Only if we're vulnerable do shutdown anything. In this case, + // shutdown all but the first entry. + if core.isVulnerable && len(core.threads) > 1 { + threads = append(threads, core.threads[1:]...) + } } + return threads } -// Execute executes the Mitigate command. -func (m Mitigate) Execute() error { - data, err := ioutil.ReadFile(m.path) - if err != nil { - return fmt.Errorf("failed to read %s: %v", m.path, err) +// ThreadGroup represents Hyperthread pairs on the same physical/core ID. +type ThreadGroup struct { + threads []Thread + isVulnerable bool +} + +// String implements the String method for threadGroup. +func (c ThreadGroup) String() string { + ret := fmt.Sprintf("ThreadGroup:\nIsVulnerable: %t\n", c.isVulnerable) + for _, processor := range c.threads { + ret += fmt.Sprintf("%s\n", processor) } + return ret +} - if m.reverse { - err := m.doReverse(data) +// getThreads returns threads structs from reading /proc/cpuinfo. +func getThreads(data string) ([]Thread, error) { + // Each processor entry should start with the + // processor key. Find the beginings of each. + r := buildRegex(processorKey, `\d+`) + indices := r.FindAllStringIndex(data, -1) + if len(indices) < 1 { + return nil, fmt.Errorf("no cpus found for: %q", data) + } + + // Add the ending index for last entry. + indices = append(indices, []int{len(data), -1}) + + // Valid cpus are now defined by strings in between + // indexes (e.g. data[index[i], index[i+1]]). + // There should be len(indicies) - 1 CPUs + // since the last index is the end of the string. + cpus := make([]Thread, 0, len(indices)) + // Find each string that represents a CPU. These begin "processor". + for i := 1; i < len(indices); i++ { + start := indices[i-1][0] + end := indices[i][0] + // Parse the CPU entry, which should be between start/end. + c, err := newThread(data[start:end]) if err != nil { - return fmt.Errorf("reverse operation failed: %v", err) + return nil, err } - return nil + cpus = append(cpus, c) + } + return cpus, nil +} + +// GetThreadsFromPossible makes threads from data read from /sys/devices/system/cpu/possible. +func GetThreadsFromPossible(data []byte) ([]Thread, error) { + possibleRegex := regexp.MustCompile(`(?m)^(\d+)(-(\d+))?$`) + matches := possibleRegex.FindStringSubmatch(string(data)) + if len(matches) != 4 { + return nil, fmt.Errorf("mismatch regex from possible: %q", string(data)) + } + + // If matches[3] is empty, we only have one cpu entry. + if matches[3] == "" { + matches[3] = matches[1] } - set, err := m.doMitigate(data) + begin, err := strconv.ParseInt(matches[1], 10, 64) if err != nil { - return fmt.Errorf("mitigate operation failed: %v", err) + return nil, fmt.Errorf("failed to parse begin: %v", err) } - return m.other.execute(set, m.dryRun) + end, err := strconv.ParseInt(matches[3], 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse end: %v", err) + } + if begin > end || begin < 0 || end < 0 { + return nil, fmt.Errorf("invalid cpu bounds from possible: begin: %d end: %d", begin, end) + } + + ret := make([]Thread, 0, end-begin) + for i := begin; i <= end; i++ { + ret = append(ret, Thread{ + processorNumber: i, + id: threadID{ + physicalID: 0, // we don't care about id for enable ops. + coreID: 0, + }, + }) + } + + return ret, nil +} + +// threadID for each thread is defined by the physical and +// core IDs. If equal, two threads are Hyperthread pairs. +type threadID struct { + physicalID int64 + coreID int64 } -func (m Mitigate) doMitigate(data []byte) (cpuSet, error) { - set, err := newCPUSet(data, m.other.vulnerable) +// Thread represents pertinent info about a single hyperthread in a pair. +type Thread struct { + processorNumber int64 // the processor number of this CPU. + vendorID string // the vendorID of CPU (e.g. AuthenticAMD). + cpuFamily int64 // CPU family number (e.g. 6 for CascadeLake/Skylake). + model int64 // CPU model number (e.g. 85 for CascadeLake/Skylake). + id threadID // id for this thread + bugs map[string]struct{} // map of vulnerabilities parsed from the 'bugs' field. +} + +// newThread parses a CPU from a single cpu entry from /proc/cpuinfo. +func newThread(data string) (Thread, error) { + empty := Thread{} + processor, err := parseProcessor(data) if err != nil { - return nil, err + return empty, err } - log.Infof("Mitigate found the following CPUs...") - log.Infof("%s", set) + vendorID, err := parseVendorID(data) + if err != nil { + return empty, err + } - disableList := set.getShutdownList() - log.Infof("Disabling threads on thread pairs.") - for _, t := range disableList { - log.Infof("Disable thread: %s", t) - if m.dryRun { - continue - } - if err := t.disable(); err != nil { - return nil, fmt.Errorf("error disabling thread: %s err: %v", t, err) - } + cpuFamily, err := parseCPUFamily(data) + if err != nil { + return empty, err } - log.Infof("Shutdown successful.") - return set, nil + + model, err := parseModel(data) + if err != nil { + return empty, err + } + + physicalID, err := parsePhysicalID(data) + if err != nil { + return empty, err + } + + coreID, err := parseCoreID(data) + if err != nil { + return empty, err + } + + bugs, err := parseBugs(data) + if err != nil { + return empty, err + } + + return Thread{ + processorNumber: processor, + vendorID: vendorID, + cpuFamily: cpuFamily, + model: model, + id: threadID{ + physicalID: physicalID, + coreID: coreID, + }, + bugs: bugs, + }, nil +} + +// String implements the String method for thread. +func (t Thread) String() string { + template := `CPU: %d +CPU ID: %+v +Vendor: %s +Family/Model: %d/%d +Bugs: %s +` + bugs := make([]string, 0) + for bug := range t.bugs { + bugs = append(bugs, bug) + } + + return fmt.Sprintf(template, t.processorNumber, t.id, t.vendorID, t.cpuFamily, t.model, strings.Join(bugs, ",")) +} + +// Enable turns on the CPU by writing 1 to /sys/devices/cpu/cpu{N}/online. +func (t Thread) Enable() error { + // Linux ensures that "cpu0" is always online. + if t.processorNumber == 0 { + return nil + } + cpuPath := fmt.Sprintf(cpuOnlineTemplate, t.processorNumber) + f, err := os.OpenFile(cpuPath, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return fmt.Errorf("failed to open file %s: %v", cpuPath, err) + } + if _, err = f.Write([]byte{'1'}); err != nil { + return fmt.Errorf("failed to write '1' to %s: %v", cpuPath, err) + } + return nil +} + +// Disable turns off the CPU by writing 0 to /sys/devices/cpu/cpu{N}/online. +func (t Thread) Disable() error { + // The core labeled "cpu0" can never be taken offline via this method. + // Linux will return EPERM if the user even creates a file at the /sys + // path above. + if t.processorNumber == 0 { + return fmt.Errorf("invalid shutdown operation: cpu0 cannot be disabled") + } + cpuPath := fmt.Sprintf(cpuOnlineTemplate, t.processorNumber) + return ioutil.WriteFile(cpuPath, []byte{'0'}, 0644) } -func (m Mitigate) doReverse(data []byte) error { - set, err := newCPUSetFromPossible(data) +// IsVulnerable checks if a CPU is vulnerable to mds. +func (t Thread) IsVulnerable() bool { + _, ok := t.bugs[mds] + return ok +} + +// isActive checks if a CPU is active from /sys/devices/system/cpu/cpu{N}/online +// If the file does not exist (ioutil returns in error), we assume the CPU is on. +func (t Thread) isActive() bool { + cpuPath := fmt.Sprintf(cpuOnlineTemplate, t.processorNumber) + data, err := ioutil.ReadFile(cpuPath) if err != nil { - return err + return true } + return len(data) > 0 && data[0] != '0' +} - log.Infof("Reverse mitigate found the following CPUs...") - log.Infof("%s", set) +// SimilarTo checks family/model/bugs fields for equality of two +// processors. +func (t Thread) SimilarTo(other Thread) bool { + if t.vendorID != other.vendorID { + return false + } - enableList := set.getRemainingList() + if other.cpuFamily != t.cpuFamily { + return false + } - log.Infof("Enabling all CPUs...") - for _, t := range enableList { - log.Infof("Enabling thread: %s", t) - if m.dryRun { - continue - } - if err := t.enable(); err != nil { - return fmt.Errorf("error enabling thread: %s err: %v", t, err) + if other.model != t.model { + return false + } + + if len(other.bugs) != len(t.bugs) { + return false + } + + for bug := range t.bugs { + if _, ok := other.bugs[bug]; !ok { + return false } } - log.Infof("Enable successful.") - return nil + return true +} + +// parseProcessor grabs the processor field from /proc/cpuinfo output. +func parseProcessor(data string) (int64, error) { + return parseIntegerResult(data, processorKey) +} + +// parseVendorID grabs the vendor_id field from /proc/cpuinfo output. +func parseVendorID(data string) (string, error) { + return parseRegex(data, vendorIDKey, `[\w\d]+`) +} + +// parseCPUFamily grabs the cpu family field from /proc/cpuinfo output. +func parseCPUFamily(data string) (int64, error) { + return parseIntegerResult(data, cpuFamilyKey) +} + +// parseModel grabs the model field from /proc/cpuinfo output. +func parseModel(data string) (int64, error) { + return parseIntegerResult(data, modelKey) +} + +// parsePhysicalID parses the physical id field. +func parsePhysicalID(data string) (int64, error) { + return parseIntegerResult(data, physicalIDKey) +} + +// parseCoreID parses the core id field. +func parseCoreID(data string) (int64, error) { + return parseIntegerResult(data, coreIDKey) +} + +// parseBugs grabs the bugs field from /proc/cpuinfo output. +func parseBugs(data string) (map[string]struct{}, error) { + result, err := parseRegex(data, bugsKey, `[\d\w\s]*`) + if err != nil { + return nil, err + } + bugs := strings.Split(result, " ") + ret := make(map[string]struct{}, len(bugs)) + for _, bug := range bugs { + ret[bug] = struct{}{} + } + return ret, nil +} + +// parseIntegerResult parses fields expecting an integer. +func parseIntegerResult(data, key string) (int64, error) { + result, err := parseRegex(data, key, `\d+`) + if err != nil { + return 0, err + } + return strconv.ParseInt(result, 0, 64) +} + +// buildRegex builds a regex for parsing each CPU field. +func buildRegex(key, match string) *regexp.Regexp { + reg := fmt.Sprintf(`(?m)^%s\s*:\s*(.*)$`, key) + return regexp.MustCompile(reg) +} + +// parseRegex parses data with key inserted into a standard regex template. +func parseRegex(data, key, match string) (string, error) { + r := buildRegex(key, match) + matches := r.FindStringSubmatch(data) + if len(matches) < 2 { + return "", fmt.Errorf("failed to match key %q: %q", key, data) + } + return matches[1], nil } diff --git a/runsc/mitigate/mitigate_conf.go b/runsc/mitigate/mitigate_conf.go deleted file mode 100644 index ee326324b..000000000 --- a/runsc/mitigate/mitigate_conf.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mitigate - -import ( - "gvisor.dev/gvisor/runsc/flag" -) - -type mitigate struct { -} - -// usage returns the usage string portion for the mitigate. -func (m mitigate) usage() string { return "" } - -// setFlags sets additional flags for the Mitigate command. -func (m mitigate) setFlags(f *flag.FlagSet) {} - -// execute performs additional parts of Execute for Mitigate. -func (m mitigate) execute(set cpuSet, dryrun bool) error { - return nil -} - -func (m mitigate) vulnerable(other thread) bool { - return other.isVulnerable() -} diff --git a/runsc/mitigate/mitigate_test.go b/runsc/mitigate/mitigate_test.go index b3a9a9b18..fbd8eb886 100644 --- a/runsc/mitigate/mitigate_test.go +++ b/runsc/mitigate/mitigate_test.go @@ -17,138 +17,519 @@ package mitigate import ( "fmt" "io/ioutil" - "os" "strings" "testing" + + "gvisor.dev/gvisor/runsc/mitigate/mock" ) -type executeTestCase struct { - name string - mitigateData string - mitigateError error - reverseData string - reverseError error +// TestMockCPUSet tests mock cpu test cases against the cpuSet functions. +func TestMockCPUSet(t *testing.T) { + for _, tc := range []struct { + testCase mock.CPU + isVulnerable bool + }{ + { + testCase: mock.AMD8, + isVulnerable: false, + }, + { + testCase: mock.Haswell2, + isVulnerable: true, + }, + { + testCase: mock.Haswell2core, + isVulnerable: true, + }, + { + testCase: mock.CascadeLake2, + isVulnerable: true, + }, + { + testCase: mock.CascadeLake4, + isVulnerable: true, + }, + } { + t.Run(tc.testCase.Name, func(t *testing.T) { + data := tc.testCase.MakeCPUString() + vulnerable := func(t Thread) bool { + return t.IsVulnerable() + } + set, err := NewCPUSet([]byte(data), vulnerable) + if err != nil { + t.Fatalf("Failed to create cpuSet: %v", err) + } + + for _, tg := range set { + if err := checkSorted(tg.threads); err != nil { + t.Fatalf("Failed to sort cpuSet: %v", err) + } + } + + remaining := set.GetRemainingList() + // In the non-vulnerable case, no cores should be shutdown so all should remain. + want := tc.testCase.PhysicalCores * tc.testCase.Cores * tc.testCase.ThreadsPerCore + if tc.isVulnerable { + want = tc.testCase.PhysicalCores * tc.testCase.Cores + } + + if want != len(remaining) { + t.Fatalf("Failed to shutdown the correct number of cores: want: %d got: %d", want, len(remaining)) + } + + if !tc.isVulnerable { + return + } + + // If the set is vulnerable, we expect only 1 thread per hyperthread pair. + for _, r := range remaining { + if _, ok := set[r.id]; !ok { + t.Fatalf("Entry %+v not in map, there must be two entries in the same thread group.", r) + } + delete(set, r.id) + } + + possible := tc.testCase.MakeSysPossibleString() + set, err = NewCPUSetFromPossible([]byte(possible)) + if err != nil { + t.Fatalf("Failed to make cpuSet: %v", err) + } + + want = tc.testCase.PhysicalCores * tc.testCase.Cores * tc.testCase.ThreadsPerCore + got := len(set.GetRemainingList()) + if got != want { + t.Fatalf("Returned the wrong number of CPUs want: %d got: %d", want, got) + } + }) + } } -func TestExecute(t *testing.T) { +// TestGetCPU tests basic parsing of single CPU strings from reading +// /proc/cpuinfo. +func TestGetCPU(t *testing.T) { + data := `processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 85 +physical id: 0 +core id : 0 +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit +` + want := Thread{ + processorNumber: 0, + vendorID: "GenuineIntel", + cpuFamily: 6, + model: 85, + id: threadID{ + physicalID: 0, + coreID: 0, + }, + bugs: map[string]struct{}{ + "cpu_meltdown": struct{}{}, + "spectre_v1": struct{}{}, + "spectre_v2": struct{}{}, + "spec_store_bypass": struct{}{}, + "l1tf": struct{}{}, + "mds": struct{}{}, + "swapgs": struct{}{}, + "taa": struct{}{}, + "itlb_multihit": struct{}{}, + }, + } - partial := `processor : 1 -vendor_id : AuthenticAMD -cpu family : 23 -model : 49 -model name : AMD EPYC 7B12 -physical id : 0 -bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass + got, err := newThread(data) + if err != nil { + t.Fatalf("getCpu failed with error: %v", err) + } + + if !want.SimilarTo(got) { + t.Fatalf("Failed cpus not similar: got: %+v, want: %+v", got, want) + } + + if !got.IsVulnerable() { + t.Fatalf("Failed: cpu should be vulnerable.") + } +} + +func TestInvalid(t *testing.T) { + result, err := getThreads(`something not a processor`) + if err == nil { + t.Fatalf("getCPU set didn't return an error: %+v", result) + } + + if !strings.Contains(err.Error(), "no cpus") { + t.Fatalf("Incorrect error returned: %v", err) + } +} + +// TestCPUSet tests getting the right number of CPUs from +// parsing full output of /proc/cpuinfo. +func TestCPUSet(t *testing.T) { + data := `processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 63 +model name : Intel(R) Xeon(R) CPU @ 2.30GHz +stepping : 0 +microcode : 0x1 +cpu MHz : 2299.998 +cache size : 46080 KB +physical id : 0 +siblings : 2 +core id : 0 +cpu cores : 1 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat md_clear arch_capabilities +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs +bogomips : 4599.99 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: + +processor : 1 +vendor_id : GenuineIntel +cpu family : 6 +model : 63 +model name : Intel(R) Xeon(R) CPU @ 2.30GHz +stepping : 0 +microcode : 0x1 +cpu MHz : 2299.998 +cache size : 46080 KB +physical id : 0 +siblings : 2 +core id : 0 +cpu cores : 1 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat md_clear arch_capabilities +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs +bogomips : 4599.99 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual power management: ` + cpuSet, err := getThreads(data) + if err != nil { + t.Fatalf("getCPUSet failed: %v", err) + } - for _, tc := range []executeTestCase{ - { - name: "CascadeLake4", - mitigateData: cascadeLake4.makeCPUString(), - reverseData: cascadeLake4.makeSysPossibleString(), - }, - { - name: "Empty", - mitigateData: "", - mitigateError: fmt.Errorf(`mitigate operation failed: no cpus found for: ""`), - reverseData: "", - reverseError: fmt.Errorf(`reverse operation failed: mismatch regex from %s: ""`, allPossibleCPUs), + wantCPULen := 2 + if len(cpuSet) != wantCPULen { + t.Fatalf("Num CPU mismatch: want: %d, got: %d", wantCPULen, len(cpuSet)) + } + + wantCPU := Thread{ + vendorID: "GenuineIntel", + cpuFamily: 6, + model: 63, + bugs: map[string]struct{}{ + "cpu_meltdown": struct{}{}, + "spectre_v1": struct{}{}, + "spectre_v2": struct{}{}, + "spec_store_bypass": struct{}{}, + "l1tf": struct{}{}, + "mds": struct{}{}, + "swapgs": struct{}{}, }, - { - name: "Partial", - mitigateData: `processor : 0 + } + + for _, c := range cpuSet { + if !wantCPU.SimilarTo(c) { + t.Fatalf("Failed cpus not equal: got: %+v, want: %+v", c, wantCPU) + } + } +} + +// TestReadFile is a smoke test for parsing methods. +func TestReadFile(t *testing.T) { + data, err := ioutil.ReadFile("/proc/cpuinfo") + if err != nil { + t.Fatalf("Failed to read cpuinfo: %v", err) + } + + vulnerable := func(t Thread) bool { + return t.IsVulnerable() + } + + set, err := NewCPUSet(data, vulnerable) + if err != nil { + t.Fatalf("Failed to parse CPU data %v\n%s", err, data) + } + + for _, tg := range set { + if err := checkSorted(tg.threads); err != nil { + t.Fatalf("Failed to sort cpuSet: %v", err) + } + } + + if len(set) < 1 { + t.Fatalf("Failed to parse any CPUs: %d", len(set)) + } + + t.Log(set) +} + +// TestVulnerable tests if the isVulnerable method is correct +// among known CPUs in GCP. +func TestVulnerable(t *testing.T) { + const haswell = `processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 63 +model name : Intel(R) Xeon(R) CPU @ 2.30GHz +stepping : 0 +microcode : 0x1 +cpu MHz : 2299.998 +cache size : 46080 KB +physical id : 0 +siblings : 4 +core id : 0 +cpu cores : 2 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat md_clear arch_capabilities +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs +bogomips : 4599.99 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management:` + + const skylake = `processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 85 +model name : Intel(R) Xeon(R) CPU @ 2.00GHz +stepping : 3 +microcode : 0x1 +cpu MHz : 2000.180 +cache size : 39424 KB +physical id : 0 +siblings : 2 +core id : 0 +cpu cores : 1 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat md_clear arch_capabilities +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa +bogomips : 4000.36 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management:` + + const cascade = `processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 85 +model name : Intel(R) Xeon(R) CPU +stepping : 7 +microcode : 0x1 +cpu MHz : 2800.198 +cache size : 33792 KB +physical id : 0 +siblings : 2 +core id : 0 +cpu cores : 1 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 + ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmu +lqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowpr +efetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid r +tm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves a +rat avx512_vnni md_clear arch_capabilities +bugs : spectre_v1 spectre_v2 spec_store_bypass mds swapgs taa +bogomips : 5600.39 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management:` + + const amd = `processor : 0 vendor_id : AuthenticAMD cpu family : 23 model : 49 model name : AMD EPYC 7B12 +stepping : 0 +microcode : 0x1000065 +cpu MHz : 2250.000 +cache size : 512 KB physical id : 0 +siblings : 2 core id : 0 cpu cores : 1 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext ssbd ibrs ibpb stibp vmmcall fsgsbase tsc_adjust bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr arat npt nrip_save umip rdpid bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass -power management: +bogomips : 4500.00 +TLB size : 3072 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management:` -` + partial, - mitigateError: fmt.Errorf(`mitigate operation failed: failed to match key "core id": %q`, partial), - reverseData: "1-", - reverseError: fmt.Errorf(`reverse operation failed: mismatch regex from %s: %q`, allPossibleCPUs, "1-"), + for _, tc := range []struct { + name string + cpuString string + vulnerable bool + }{ + { + name: "haswell", + cpuString: haswell, + vulnerable: true, + }, { + name: "skylake", + cpuString: skylake, + vulnerable: true, + }, { + name: "amd", + cpuString: amd, + vulnerable: false, }, } { - doExecuteTest(t, Mitigate{}, tc) + t.Run(tc.name, func(t *testing.T) { + set, err := getThreads(tc.cpuString) + if err != nil { + t.Fatalf("Failed to getCPUSet:%v\n %s", err, tc.cpuString) + } + + if len(set) < 1 { + t.Fatalf("Returned empty cpu set: %v", set) + } + + for _, c := range set { + got := func() bool { + return c.IsVulnerable() + }() + + if got != tc.vulnerable { + t.Fatalf("Mismatch vulnerable for cpu %+s: got %t want: %t", tc.name, tc.vulnerable, got) + } + } + }) } } -func TestExecuteSmoke(t *testing.T) { - smokeMitigate, err := ioutil.ReadFile(cpuInfo) +func TestReverse(t *testing.T) { + const noParse = "-1-" + for _, tc := range []struct { + name string + output string + wantErr error + wantCount int + }{ + { + name: "base", + output: "0-7", + wantErr: nil, + wantCount: 8, + }, + { + name: "huge", + output: "0-111", + wantErr: nil, + wantCount: 112, + }, + { + name: "not zero", + output: "50-53", + wantErr: nil, + wantCount: 4, + }, + { + name: "small", + output: "0", + wantErr: nil, + wantCount: 1, + }, + { + name: "invalid order", + output: "10-6", + wantErr: fmt.Errorf("invalid cpu bounds from possible: begin: %d end: %d", 10, 6), + }, + { + name: "no parse", + output: noParse, + wantErr: fmt.Errorf(`mismatch regex from possible: %q`, noParse), + }, + } { + t.Run(tc.name, func(t *testing.T) { + threads, err := GetThreadsFromPossible([]byte(tc.output)) + + switch { + case tc.wantErr == nil: + if err != nil { + t.Fatalf("Wanted nil err, got: %v", err) + } + case err == nil: + t.Fatalf("Want error: %v got: %v", tc.wantErr, err) + default: + if tc.wantErr.Error() != err.Error() { + t.Fatalf("Want error: %v got error: %v", tc.wantErr, err) + } + } + + if len(threads) != tc.wantCount { + t.Fatalf("Want count: %d got: %d", tc.wantCount, len(threads)) + } + }) + } +} + +func TestReverseSmoke(t *testing.T) { + data, err := ioutil.ReadFile("/sys/devices/system/cpu/possible") if err != nil { - t.Fatalf("Failed to read %s: %v", cpuInfo, err) + t.Fatalf("Failed to read from possible: %v", err) } - smokeReverse, err := ioutil.ReadFile(allPossibleCPUs) + threads, err := GetThreadsFromPossible(data) if err != nil { - t.Fatalf("Failed to read %s: %v", allPossibleCPUs, err) + t.Fatalf("Could not parse possible output: %v", err) } - doExecuteTest(t, Mitigate{}, executeTestCase{ - name: "SmokeTest", - mitigateData: string(smokeMitigate), - reverseData: string(smokeReverse), - }) + if len(threads) <= 0 { + t.Fatalf("Didn't get any CPU cores: %d", len(threads)) + } } -// doExecuteTest runs Execute with the mitigate operation and reverse operation. -func doExecuteTest(t *testing.T, m Mitigate, tc executeTestCase) { - t.Run("Mitigate"+tc.name, func(t *testing.T) { - m.dryRun = true - file, err := ioutil.TempFile("", "outfile.txt") - if err != nil { - t.Fatalf("Failed to create tmpfile: %v", err) - } - defer os.Remove(file.Name()) - - if _, err := file.WriteString(tc.mitigateData); err != nil { - t.Fatalf("Failed to write to file: %v", err) - } - - m.path = file.Name() - - got := m.Execute() - if err = checkErr(tc.mitigateError, got); err != nil { - t.Fatalf("Mitigate error mismatch: %v", err) - } - }) - t.Run("Reverse"+tc.name, func(t *testing.T) { - m.dryRun = true - m.reverse = true - - file, err := ioutil.TempFile("", "outfile.txt") - if err != nil { - t.Fatalf("Failed to create tmpfile: %v", err) - } - defer os.Remove(file.Name()) - - if _, err := file.WriteString(tc.reverseData); err != nil { - t.Fatalf("Failed to write to file: %v", err) - } - - m.path = file.Name() - got := m.Execute() - if err = checkErr(tc.reverseError, got); err != nil { - t.Fatalf("Mitigate error mismatch: %v", err) +func checkSorted(threads []Thread) error { + if len(threads) < 2 { + return nil + } + last := threads[0].processorNumber + for _, t := range threads[1:] { + if last >= t.processorNumber { + return fmt.Errorf("threads out of order: thread %d before %d", t.processorNumber, last) } - }) - -} - -// checkErr checks error for equality. -func checkErr(want, got error) error { - switch { - case want == nil && got == nil: - case want != nil && got == nil: - fallthrough - case want == nil && got != nil: - fallthrough - case want.Error() != strings.Trim(got.Error(), " "): - return fmt.Errorf("got: %v want: %v", got, want) + last = t.processorNumber } return nil } diff --git a/runsc/mitigate/mock/BUILD b/runsc/mitigate/mock/BUILD new file mode 100644 index 000000000..5019ff9ee --- /dev/null +++ b/runsc/mitigate/mock/BUILD @@ -0,0 +1,11 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "mock", + srcs = ["mock.go"], + visibility = [ + "//runsc:__subpackages__", + ], +) diff --git a/runsc/mitigate/mock/mock.go b/runsc/mitigate/mock/mock.go new file mode 100644 index 000000000..2db718cb9 --- /dev/null +++ b/runsc/mitigate/mock/mock.go @@ -0,0 +1,141 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mock contains mock CPUs for mitigate tests. +package mock + +import "fmt" + +// CPU represents data from CPUs that will be mitigated. +type CPU struct { + Name string + VendorID string + Family int + Model int + ModelName string + Bugs string + PhysicalCores int + Cores int + ThreadsPerCore int +} + +// CascadeLake2 is a two core Intel CascadeLake machine. +var CascadeLake2 = CPU{ + Name: "CascadeLake", + VendorID: "GenuineIntel", + Family: 6, + Model: 85, + ModelName: "Intel(R) Xeon(R) CPU", + Bugs: "spectre_v1 spectre_v2 spec_store_bypass mds swapgs taa", + PhysicalCores: 1, + Cores: 1, + ThreadsPerCore: 2, +} + +// CascadeLake4 is a four core Intel CascadeLake machine. +var CascadeLake4 = CPU{ + Name: "CascadeLake", + VendorID: "GenuineIntel", + Family: 6, + Model: 85, + ModelName: "Intel(R) Xeon(R) CPU", + Bugs: "spectre_v1 spectre_v2 spec_store_bypass mds swapgs taa", + PhysicalCores: 1, + Cores: 2, + ThreadsPerCore: 2, +} + +// Haswell2 is a two core Intel Haswell machine. +var Haswell2 = CPU{ + Name: "Haswell", + VendorID: "GenuineIntel", + Family: 6, + Model: 63, + ModelName: "Intel(R) Xeon(R) CPU", + Bugs: "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs", + PhysicalCores: 1, + Cores: 1, + ThreadsPerCore: 2, +} + +// Haswell2core is a 2 core Intel Haswell machine with no hyperthread pairs. +var Haswell2core = CPU{ + Name: "Haswell2Physical", + VendorID: "GenuineIntel", + Family: 6, + Model: 63, + ModelName: "Intel(R) Xeon(R) CPU", + Bugs: "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs", + PhysicalCores: 2, + Cores: 1, + ThreadsPerCore: 1, +} + +// AMD8 is an eight core AMD machine. +var AMD8 = CPU{ + Name: "AMD", + VendorID: "AuthenticAMD", + Family: 23, + Model: 49, + ModelName: "AMD EPYC 7B12", + Bugs: "sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass", + PhysicalCores: 4, + Cores: 1, + ThreadsPerCore: 2, +} + +// MakeCPUString makes a string formated like /proc/cpuinfo for each cpuTestCase +func (tc CPU) MakeCPUString() string { + template := `processor : %d +vendor_id : %s +cpu family : %d +model : %d +model name : %s +physical id : %d +core id : %d +cpu cores : %d +bugs : %s + +` + + ret := `` + for i := 0; i < tc.PhysicalCores; i++ { + for j := 0; j < tc.Cores; j++ { + for k := 0; k < tc.ThreadsPerCore; k++ { + processorNum := (i*tc.Cores+j)*tc.ThreadsPerCore + k + ret += fmt.Sprintf(template, + processorNum, /*processor*/ + tc.VendorID, /*vendor_id*/ + tc.Family, /*cpu family*/ + tc.Model, /*model*/ + tc.ModelName, /*model name*/ + i, /*physical id*/ + j, /*core id*/ + tc.Cores*tc.PhysicalCores, /*cpu cores*/ + tc.Bugs, /*bugs*/ + ) + } + } + } + return ret +} + +// MakeSysPossibleString makes a string representing a the contents of /sys/devices/system/cpu/possible. +func (tc CPU) MakeSysPossibleString() string { + max := tc.PhysicalCores * tc.Cores * tc.ThreadsPerCore + if max == 1 { + return "0" + } + return fmt.Sprintf("0-%d", max-1) +} diff --git a/test/benchmarks/fs/BUILD b/test/benchmarks/fs/BUILD index b4f967441..c94caab60 100644 --- a/test/benchmarks/fs/BUILD +++ b/test/benchmarks/fs/BUILD @@ -11,6 +11,7 @@ benchmark_test( "//pkg/test/dockerutil", "//test/benchmarks/harness", "//test/benchmarks/tools", + "@com_github_docker_docker//api/types/mount:go_default_library", ], ) diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go index 8baeff0db..7ced963f6 100644 --- a/test/benchmarks/fs/bazel_test.go +++ b/test/benchmarks/fs/bazel_test.go @@ -25,6 +25,13 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) +// Dimensions here are clean/dirty cache (do or don't drop caches) +// and if the mount on which we are compiling is a tmpfs/bind mount. +type benchmark struct { + clearCache bool // clearCache drops caches before running. + fstype string // type of filesystem to use. +} + // Note: CleanCache versions of this test require running with root permissions. func BenchmarkBuildABSL(b *testing.B) { runBuildBenchmark(b, "benchmarks/absl", "/abseil-cpp", "absl/base/...") @@ -45,17 +52,18 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { } defer machine.CleanUp() - // Dimensions here are clean/dirty cache (do or don't drop caches) - // and if the mount on which we are compiling is a tmpfs/bind mount. - benchmarks := []struct { - clearCache bool // clearCache drops caches before running. - tmpfs bool // tmpfs will run compilation on a tmpfs. - }{ - {clearCache: true, tmpfs: false}, - {clearCache: false, tmpfs: false}, - {clearCache: true, tmpfs: true}, - {clearCache: false, tmpfs: true}, + benchmarks := make([]benchmark, 0, 6) + for _, filesys := range []string{harness.BindFS, harness.TmpFS, harness.RootFS} { + benchmarks = append(benchmarks, benchmark{ + clearCache: true, + fstype: filesys, + }) + benchmarks = append(benchmarks, benchmark{ + clearCache: false, + fstype: filesys, + }) } + for _, bm := range benchmarks { pageCache := tools.Parameter{ Name: "page_cache", @@ -67,10 +75,7 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { filesystem := tools.Parameter{ Name: "filesystem", - Value: "bind", - } - if bm.tmpfs { - filesystem.Value = "tmpfs" + Value: bm.fstype, } name, err := tools.ParametersToName(pageCache, filesystem) if err != nil { @@ -83,21 +88,25 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { container := machine.GetContainer(ctx, b) defer container.CleanUp(ctx) + mts, prefix, cleanup, err := harness.MakeMount(machine, bm.fstype) + if err != nil { + b.Fatalf("Failed to make mount: %v", err) + } + defer cleanup() + + runOpts := dockerutil.RunOpts{ + Image: image, + Mounts: mts, + } + // Start a container and sleep. - if err := container.Spawn(ctx, dockerutil.RunOpts{ - Image: image, - }, "sleep", fmt.Sprintf("%d", 1000000)); err != nil { + if err := container.Spawn(ctx, runOpts, "sleep", fmt.Sprintf("%d", 1000000)); err != nil { b.Fatalf("run failed with: %v", err) } - // If we are running on a tmpfs, copy to /tmp which is a tmpfs. - prefix := "" - if bm.tmpfs { - if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, - "cp", "-r", workdir, "/tmp/."); err != nil { - b.Fatalf("failed to copy directory: %v (%s)", err, out) - } - prefix = "/tmp" + if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, + "cp", "-rf", workdir, prefix+"/."); err != nil { + b.Fatalf("failed to copy directory: %v (%s)", err, out) } b.ResetTimer() @@ -118,7 +127,7 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { WorkDir: prefix + workdir, }, "bazel", "build", "-c", "opt", target) if err != nil { - b.Fatalf("build failed with: %v", err) + b.Fatalf("build failed with: %v logs: %s", err, got) } b.StopTimer() diff --git a/test/benchmarks/fs/fio_test.go b/test/benchmarks/fs/fio_test.go index cc2d1cbbc..f783a2b33 100644 --- a/test/benchmarks/fs/fio_test.go +++ b/test/benchmarks/fs/fio_test.go @@ -21,7 +21,6 @@ import ( "strings" "testing" - "github.com/docker/docker/api/types/mount" "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/test/benchmarks/harness" "gvisor.dev/gvisor/test/benchmarks/tools" @@ -70,7 +69,7 @@ func BenchmarkFio(b *testing.B) { } defer machine.CleanUp() - for _, fsType := range []mount.Type{mount.TypeBind, mount.TypeTmpfs} { + for _, fsType := range []string{harness.BindFS, harness.TmpFS, harness.RootFS} { for _, tc := range testCases { operation := tools.Parameter{ Name: "operation", @@ -82,7 +81,7 @@ func BenchmarkFio(b *testing.B) { } filesystem := tools.Parameter{ Name: "filesystem", - Value: string(fsType), + Value: fsType, } name, err := tools.ParametersToName(operation, blockSize, filesystem) if err != nil { @@ -95,13 +94,7 @@ func BenchmarkFio(b *testing.B) { container := machine.GetContainer(ctx, b) defer container.CleanUp(ctx) - // Directory and filename inside container where fio will read/write. - outdir := "/data" - outfile := filepath.Join(outdir, "test.txt") - - // Make the required mount and grab a cleanup for bind mounts - // as they are backed by a temp directory (mktemp). - mnt, mountCleanup, err := makeMount(machine, fsType, outdir) + mnts, outdir, mountCleanup, err := harness.MakeMount(machine, fsType) if err != nil { b.Fatalf("failed to make mount: %v", err) } @@ -109,12 +102,9 @@ func BenchmarkFio(b *testing.B) { // Start the container with the mount. if err := container.Spawn( - ctx, - dockerutil.RunOpts{ - Image: "benchmarks/fio", - Mounts: []mount.Mount{ - mnt, - }, + ctx, dockerutil.RunOpts{ + Image: "benchmarks/fio", + Mounts: mnts, }, // Sleep on the order of b.N. "sleep", fmt.Sprintf("%d", 1000*b.N), @@ -122,6 +112,9 @@ func BenchmarkFio(b *testing.B) { b.Fatalf("failed to start fio container with: %v", err) } + // Directory and filename inside container where fio will read/write. + outfile := filepath.Join(outdir, "test.txt") + // For reads, we need a file to read so make one inside the container. if strings.Contains(tc.Test, "read") { fallocateCmd := fmt.Sprintf("fallocate -l %dK %s", tc.Size, outfile) @@ -135,6 +128,7 @@ func BenchmarkFio(b *testing.B) { if err := harness.DropCaches(machine); err != nil { b.Skipf("failed to drop caches with %v. You probably need root.", err) } + cmd := tc.MakeCmd(outfile) if err := harness.DropCaches(machine); err != nil { @@ -154,39 +148,6 @@ func BenchmarkFio(b *testing.B) { } } -// makeMount makes a mount and cleanup based on the requested type. Bind -// and volume mounts are backed by a temp directory made with mktemp. -// tmpfs mounts require no such backing and are just made. -// It is up to the caller to call the returned cleanup. -func makeMount(machine harness.Machine, mountType mount.Type, target string) (mount.Mount, func(), error) { - switch mountType { - case mount.TypeVolume, mount.TypeBind: - dir, err := machine.RunCommand("mktemp", "-d") - if err != nil { - return mount.Mount{}, func() {}, fmt.Errorf("failed to create tempdir: %v", err) - } - dir = strings.TrimSuffix(dir, "\n") - - out, err := machine.RunCommand("chmod", "777", dir) - if err != nil { - machine.RunCommand("rm", "-rf", dir) - return mount.Mount{}, func() {}, fmt.Errorf("failed modify directory: %v %s", err, out) - } - return mount.Mount{ - Target: target, - Source: dir, - Type: mount.TypeBind, - }, func() { machine.RunCommand("rm", "-rf", dir) }, nil - case mount.TypeTmpfs: - return mount.Mount{ - Target: target, - Type: mount.TypeTmpfs, - }, func() {}, nil - default: - return mount.Mount{}, func() {}, fmt.Errorf("illegal mount time not supported: %v", mountType) - } -} - // TestMain is the main method for package fs. func TestMain(m *testing.M) { harness.Init() diff --git a/test/benchmarks/harness/BUILD b/test/benchmarks/harness/BUILD index c2e316709..116610938 100644 --- a/test/benchmarks/harness/BUILD +++ b/test/benchmarks/harness/BUILD @@ -14,5 +14,6 @@ go_library( deps = [ "//pkg/test/dockerutil", "//pkg/test/testutil", + "@com_github_docker_docker//api/types/mount:go_default_library", ], ) diff --git a/test/benchmarks/harness/util.go b/test/benchmarks/harness/util.go index aeac7ebff..36abe1069 100644 --- a/test/benchmarks/harness/util.go +++ b/test/benchmarks/harness/util.go @@ -18,8 +18,10 @@ import ( "context" "fmt" "net" + "strings" "testing" + "github.com/docker/docker/api/types/mount" "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/pkg/test/testutil" ) @@ -55,3 +57,53 @@ func DebugLog(b *testing.B, msg string, args ...interface{}) { b.Logf(msg, args...) } } + +const ( + // BindFS indicates a bind mount should be created. + BindFS = "bindfs" + // TmpFS indicates a tmpfs mount should be created. + TmpFS = "tmpfs" + // RootFS indicates no mount should be created and the root mount should be used. + RootFS = "rootfs" +) + +// MakeMount makes a mount and cleanup based on the requested type. Bind +// and volume mounts are backed by a temp directory made with mktemp. +// tmpfs mounts require no such backing and are just made. +// rootfs mounts do not make a mount, but instead return a target direectory at root. +// It is up to the caller to call the returned cleanup. +func MakeMount(machine Machine, fsType string) ([]mount.Mount, string, func(), error) { + mounts := make([]mount.Mount, 0, 1) + switch fsType { + case BindFS: + dir, err := machine.RunCommand("mktemp", "-d") + if err != nil { + return mounts, "", func() {}, fmt.Errorf("failed to create tempdir: %v", err) + } + dir = strings.TrimSuffix(dir, "\n") + + out, err := machine.RunCommand("chmod", "777", dir) + if err != nil { + machine.RunCommand("rm", "-rf", dir) + return mounts, "", func() {}, fmt.Errorf("failed modify directory: %v %s", err, out) + } + target := "/data" + mounts = append(mounts, mount.Mount{ + Target: target, + Source: dir, + Type: mount.TypeBind, + }) + return mounts, target, func() { machine.RunCommand("rm", "-rf", dir) }, nil + case RootFS: + return mounts, "/", func() {}, nil + case TmpFS: + target := "/data" + mounts = append(mounts, mount.Mount{ + Target: target, + Type: mount.TypeTmpfs, + }) + return mounts, target, func() {}, nil + default: + return mounts, "", func() {}, fmt.Errorf("illegal mount type not supported: %v", fsType) + } +} |