Implement cgroupfs.

A skeleton implementation of cgroupfs. It supports trivial cpu and
memory controllers with no support for hierarchies.

PiperOrigin-RevId: 366561126

5 files changed, 408 insertions, 0 deletions

diff --git a/test/util/BUILD b/test/util/BUILD
index e561f3daa..383de00ed 100644
--- a/test/util/BUILD
+++ b/test/util/BUILD
@@ -94,6 +94,7 @@ cc_library(
         ":file_descriptor",
         ":posix_error",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/time",
         gtest,
     ],
 )
@@ -368,3 +369,20 @@ cc_library(
     testonly = 1,
     hdrs = ["temp_umask.h"],
 )
+
+cc_library(
+    name = "cgroup_util",
+    testonly = 1,
+    srcs = ["cgroup_util.cc"],
+    hdrs = ["cgroup_util.h"],
+    deps = [
+        ":cleanup",
+        ":fs_util",
+        ":mount_util",
+        ":posix_error",
+        ":temp_path",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
+    ],
+)
diff --git a/test/util/cgroup_util.cc b/test/util/cgroup_util.cc
new file mode 100644
index 000000000..65d9c4986
--- /dev/null
+++ b/test/util/cgroup_util.cc
@@ -0,0 +1,223 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/util/cgroup_util.h"
+
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#include "absl/strings/str_split.h"
+#include "test/util/fs_util.h"
+#include "test/util/mount_util.h"
+
+namespace gvisor {
+namespace testing {
+
+Cgroup::Cgroup(std::string path) : cgroup_path_(path) {
+  id_ = ++Cgroup::next_id_;
+  std::cerr << absl::StreamFormat("[cg#%d] <= %s", id_, cgroup_path_)
+            << std::endl;
+}
+
+PosixErrorOr<std::string> Cgroup::ReadControlFile(
+    absl::string_view name) const {
+  std::string buf;
+  RETURN_IF_ERRNO(GetContents(Relpath(name), &buf));
+
+  const std::string alias_path = absl::StrFormat("[cg#%d]/%s", id_, name);
+  std::cerr << absl::StreamFormat("<contents of %s>", alias_path) << std::endl;
+  std::cerr << buf;
+  std::cerr << absl::StreamFormat("<end of %s>", alias_path) << std::endl;
+
+  return buf;
+}
+
+PosixErrorOr<int64_t> Cgroup::ReadIntegerControlFile(
+    absl::string_view name) const {
+  ASSIGN_OR_RETURN_ERRNO(const std::string buf, ReadControlFile(name));
+  ASSIGN_OR_RETURN_ERRNO(const int64_t val, Atoi<int64_t>(buf));
+  return val;
+}
+
+PosixErrorOr<absl::flat_hash_set<pid_t>> Cgroup::Procs() const {
+  ASSIGN_OR_RETURN_ERRNO(std::string buf, ReadControlFile("cgroup.procs"));
+  return ParsePIDList(buf);
+}
+
+PosixErrorOr<absl::flat_hash_set<pid_t>> Cgroup::Tasks() const {
+  ASSIGN_OR_RETURN_ERRNO(std::string buf, ReadControlFile("tasks"));
+  return ParsePIDList(buf);
+}
+
+PosixError Cgroup::ContainsCallingProcess() const {
+  ASSIGN_OR_RETURN_ERRNO(const absl::flat_hash_set<pid_t> procs, Procs());
+  ASSIGN_OR_RETURN_ERRNO(const absl::flat_hash_set<pid_t> tasks, Tasks());
+  const pid_t pid = getpid();
+  const pid_t tid = syscall(SYS_gettid);
+  if (!procs.contains(pid)) {
+    return PosixError(
+        ENOENT, absl::StrFormat("Cgroup doesn't contain process %d", pid));
+  }
+  if (!tasks.contains(tid)) {
+    return PosixError(ENOENT,
+                      absl::StrFormat("Cgroup doesn't contain task %d", tid));
+  }
+  return NoError();
+}
+
+PosixErrorOr<absl::flat_hash_set<pid_t>> Cgroup::ParsePIDList(
+    absl::string_view data) const {
+  absl::flat_hash_set<pid_t> res;
+  std::vector<absl::string_view> lines = absl::StrSplit(data, '\n');
+  for (const std::string_view& line : lines) {
+    if (line.empty()) {
+      continue;
+    }
+    ASSIGN_OR_RETURN_ERRNO(const int32_t pid, Atoi<int32_t>(line));
+    res.insert(static_cast<pid_t>(pid));
+  }
+  return res;
+}
+
+int64_t Cgroup::next_id_ = 0;
+
+PosixErrorOr<Cgroup> Mounter::MountCgroupfs(std::string mopts) {
+  ASSIGN_OR_RETURN_ERRNO(TempPath mountpoint,
+                         TempPath::CreateDirIn(root_.path()));
+  ASSIGN_OR_RETURN_ERRNO(
+      Cleanup mount, Mount("none", mountpoint.path(), "cgroup", 0, mopts, 0));
+  const std::string mountpath = mountpoint.path();
+  std::cerr << absl::StreamFormat(
+                   "Mount(\"none\", \"%s\", \"cgroup\", 0, \"%s\", 0) => OK",
+                   mountpath, mopts)
+            << std::endl;
+  Cgroup cg = Cgroup(mountpath);
+  mountpoints_[cg.id()] = std::move(mountpoint);
+  mounts_[cg.id()] = std::move(mount);
+  return cg;
+}
+
+PosixError Mounter::Unmount(const Cgroup& c) {
+  auto mount = mounts_.find(c.id());
+  auto mountpoint = mountpoints_.find(c.id());
+
+  if (mount == mounts_.end() || mountpoint == mountpoints_.end()) {
+    return PosixError(
+        ESRCH, absl::StrFormat("No mount found for cgroupfs containing cg#%d",
+                               c.id()));
+  }
+
+  std::cerr << absl::StreamFormat("Unmount([cg#%d])", c.id()) << std::endl;
+
+  // Simply delete the entries, their destructors will unmount and delete the
+  // mountpoint. Note the order is important to avoid errors: mount then
+  // mountpoint.
+  mounts_.erase(mount);
+  mountpoints_.erase(mountpoint);
+
+  return NoError();
+}
+
+constexpr char kProcCgroupsHeader[] =
+    "#subsys_name\thierarchy\tnum_cgroups\tenabled";
+
+PosixErrorOr<absl::flat_hash_map<std::string, CgroupsEntry>>
+ProcCgroupsEntries() {
+  std::string content;
+  RETURN_IF_ERRNO(GetContents("/proc/cgroups", &content));
+
+  bool found_header = false;
+  absl::flat_hash_map<std::string, CgroupsEntry> entries;
+  std::vector<std::string> lines = absl::StrSplit(content, '\n');
+  std::cerr << "<contents of /proc/cgroups>" << std::endl;
+  for (const std::string& line : lines) {
+    std::cerr << line << std::endl;
+
+    if (!found_header) {
+      EXPECT_EQ(line, kProcCgroupsHeader);
+      found_header = true;
+      continue;
+    }
+    if (line.empty()) {
+      continue;
+    }
+
+    // Parse a single entry from /proc/cgroups.
+    //
+    // Example entries, fields are tab separated in the real file:
+    //
+    // #subsys_name    hierarchy       num_cgroups     enabled
+    // cpuset  12      35      1
+    // cpu     3       222     1
+    //   ^     ^       ^       ^
+    //   0     1       2       3
+
+    CgroupsEntry entry;
+    std::vector<std::string> fields =
+        StrSplit(line, absl::ByAnyChar(": \t"), absl::SkipEmpty());
+
+    entry.subsys_name = fields[0];
+    ASSIGN_OR_RETURN_ERRNO(entry.hierarchy, Atoi<uint32_t>(fields[1]));
+    ASSIGN_OR_RETURN_ERRNO(entry.num_cgroups, Atoi<uint64_t>(fields[2]));
+    ASSIGN_OR_RETURN_ERRNO(const int enabled, Atoi<int>(fields[3]));
+    entry.enabled = enabled != 0;
+
+    entries[entry.subsys_name] = entry;
+  }
+  std::cerr << "<end of /proc/cgroups>" << std::endl;
+
+  return entries;
+}
+
+PosixErrorOr<absl::flat_hash_map<std::string, PIDCgroupEntry>>
+ProcPIDCgroupEntries(pid_t pid) {
+  const std::string path = absl::StrFormat("/proc/%d/cgroup", pid);
+  std::string content;
+  RETURN_IF_ERRNO(GetContents(path, &content));
+
+  absl::flat_hash_map<std::string, PIDCgroupEntry> entries;
+  std::vector<std::string> lines = absl::StrSplit(content, '\n');
+
+  std::cerr << absl::StreamFormat("<contents of %s>", path) << std::endl;
+  for (const std::string& line : lines) {
+    std::cerr << line << std::endl;
+
+    if (line.empty()) {
+      continue;
+    }
+
+    // Parse a single entry from /proc/<pid>/cgroup.
+    //
+    // Example entries:
+    //
+    // 2:cpu:/path/to/cgroup
+    // 1:memory:/
+
+    PIDCgroupEntry entry;
+    std::vector<std::string> fields =
+        absl::StrSplit(line, absl::ByChar(':'), absl::SkipEmpty());
+
+    ASSIGN_OR_RETURN_ERRNO(entry.hierarchy, Atoi<uint32_t>(fields[0]));
+    entry.controllers = fields[1];
+    entry.path = fields[2];
+
+    entries[entry.controllers] = entry;
+  }
+  std::cerr << absl::StreamFormat("<end of %s>", path) << std::endl;
+
+  return entries;
+}
+
+}  // namespace testing
+}  // namespace gvisor
diff --git a/test/util/cgroup_util.h b/test/util/cgroup_util.h
new file mode 100644
index 000000000..b049559df
--- /dev/null
+++ b/test/util/cgroup_util.h
@@ -0,0 +1,111 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GVISOR_TEST_UTIL_CGROUP_UTIL_H_
+#define GVISOR_TEST_UTIL_CGROUP_UTIL_H_
+
+#include <unistd.h>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/string_view.h"
+#include "test/util/cleanup.h"
+#include "test/util/fs_util.h"
+#include "test/util/temp_path.h"
+
+namespace gvisor {
+namespace testing {
+
+// Cgroup represents a cgroup directory on a mounted cgroupfs.
+class Cgroup {
+ public:
+  Cgroup(std::string path);
+
+  uint64_t id() const { return id_; }
+
+  std::string Relpath(absl::string_view leaf) const {
+    return JoinPath(cgroup_path_, leaf);
+  }
+
+  // Returns the contents of a cgroup control file with the given name.
+  PosixErrorOr<std::string> ReadControlFile(absl::string_view name) const;
+
+  // Reads the contents of a cgroup control with the given name, and attempts
+  // to parse it as an integer.
+  PosixErrorOr<int64_t> ReadIntegerControlFile(absl::string_view name) const;
+
+  // Returns the thread ids of the leaders of thread groups managed by this
+  // cgroup.
+  PosixErrorOr<absl::flat_hash_set<pid_t>> Procs() const;
+
+  PosixErrorOr<absl::flat_hash_set<pid_t>> Tasks() const;
+
+  // ContainsCallingProcess checks whether the calling process is part of the
+  PosixError ContainsCallingProcess() const;
+
+ private:
+  PosixErrorOr<absl::flat_hash_set<pid_t>> ParsePIDList(
+      absl::string_view data) const;
+
+  static int64_t next_id_;
+  int64_t id_;
+  const std::string cgroup_path_;
+};
+
+// Mounter is a utility for creating cgroupfs mounts. It automatically manages
+// the lifetime of created mounts.
+class Mounter {
+ public:
+  Mounter(TempPath root) : root_(std::move(root)) {}
+
+  PosixErrorOr<Cgroup> MountCgroupfs(std::string mopts);
+
+  PosixError Unmount(const Cgroup& c);
+
+ private:
+  // The destruction order of these members avoids errors during cleanup. We
+  // first unmount (by executing the mounts_ cleanups), then delete the
+  // mountpoint subdirs, then delete the root.
+  TempPath root_;
+  absl::flat_hash_map<int64_t, TempPath> mountpoints_;
+  absl::flat_hash_map<int64_t, Cleanup> mounts_;
+};
+
+// Represents a line from /proc/cgroups.
+struct CgroupsEntry {
+  std::string subsys_name;
+  uint32_t hierarchy;
+  uint64_t num_cgroups;
+  bool enabled;
+};
+
+// Returns a parsed representation of /proc/cgroups.
+PosixErrorOr<absl::flat_hash_map<std::string, CgroupsEntry>>
+ProcCgroupsEntries();
+
+// Represents a line from /proc/<pid>/cgroup.
+struct PIDCgroupEntry {
+  uint32_t hierarchy;
+  std::string controllers;
+  std::string path;
+};
+
+// Returns a parsed representation of /proc/<pid>/cgroup.
+PosixErrorOr<absl::flat_hash_map<std::string, PIDCgroupEntry>>
+ProcPIDCgroupEntries(pid_t pid);
+
+}  // namespace testing
+}  // namespace gvisor
+
+#endif  // GVISOR_TEST_UTIL_CGROUP_UTIL_H_
diff --git a/test/util/fs_util.cc b/test/util/fs_util.cc
index 5f1ce0d8a..483ae848d 100644
--- a/test/util/fs_util.cc
+++ b/test/util/fs_util.cc
@@ -28,6 +28,8 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_split.h"
 #include "absl/strings/string_view.h"
+#include "absl/time/clock.h"
+#include "absl/time/time.h"
 #include "test/util/cleanup.h"
 #include "test/util/file_descriptor.h"
 #include "test/util/posix_error.h"
@@ -366,6 +368,48 @@ PosixErrorOr<std::vector<std::string>> ListDir(absl::string_view abspath,
   return files;
 }
 
+PosixError DirContains(absl::string_view path,
+                       const std::vector<std::string>& expect,
+                       const std::vector<std::string>& exclude) {
+  ASSIGN_OR_RETURN_ERRNO(auto listing, ListDir(path, false));
+
+  for (auto& expected_entry : expect) {
+    auto cursor = std::find(listing.begin(), listing.end(), expected_entry);
+    if (cursor == listing.end()) {
+      return PosixError(ENOENT, absl::StrFormat("Failed to find '%s' in '%s'",
+                                                expected_entry, path));
+    }
+  }
+  for (auto& excluded_entry : exclude) {
+    auto cursor = std::find(listing.begin(), listing.end(), excluded_entry);
+    if (cursor != listing.end()) {
+      return PosixError(ENOENT, absl::StrCat("File '", excluded_entry,
+                                             "' found in path '", path, "'"));
+    }
+  }
+  return NoError();
+}
+
+PosixError EventuallyDirContains(absl::string_view path,
+                                 const std::vector<std::string>& expect,
+                                 const std::vector<std::string>& exclude) {
+  constexpr int kRetryCount = 100;
+  const absl::Duration kRetryDelay = absl::Milliseconds(100);
+
+  for (int i = 0; i < kRetryCount; ++i) {
+    auto res = DirContains(path, expect, exclude);
+    if (res.ok()) {
+      return res;
+    }
+    if (i < kRetryCount - 1) {
+      // Sleep if this isn't the final iteration.
+      absl::SleepFor(kRetryDelay);
+    }
+  }
+  return PosixError(ETIMEDOUT,
+                    "Timed out while waiting for directory to contain files ");
+}
+
 PosixError RecursivelyDelete(absl::string_view path, int* undeleted_dirs,
                              int* undeleted_files) {
   ASSIGN_OR_RETURN_ERRNO(bool exists, Exists(path));
diff --git a/test/util/fs_util.h b/test/util/fs_util.h
index 2190c3bca..bb2d1d3c8 100644
--- a/test/util/fs_util.h
+++ b/test/util/fs_util.h
@@ -129,6 +129,18 @@ PosixError WalkTree(
 PosixErrorOr<std::vector<std::string>> ListDir(absl::string_view abspath,
                                                bool skipdots);
 
+// Check that a directory contains children nodes named in expect, and does not
+// contain any children nodes named in exclude.
+PosixError DirContains(absl::string_view path,
+                       const std::vector<std::string>& expect,
+                       const std::vector<std::string>& exclude);
+
+// Same as DirContains, but adds a retry. Suitable for checking a directory
+// being modified asynchronously.
+PosixError EventuallyDirContains(absl::string_view path,
+                                 const std::vector<std::string>& expect,
+                                 const std::vector<std::string>& exclude);
+
 // Attempt to recursively delete a directory or file. Returns an error and
 // the number of undeleted directories and files. If either
 // undeleted_dirs or undeleted_files is nullptr then it will not be used.