Fix deadlock between epoll_wait and getdents

epoll_wait acquires EventPoll.listsMu (in EventPoll.ReadEvents) and
then calls Inotify.Readiness which tries to acquire Inotify.evMu.

getdents acquires Inotify.evMu (in Inotify.queueEvent) and then calls
readyCallback.Callback which tries to acquire EventPoll.listsMu.

The fix is to release Inotify.evMu before calling Queue.Notify. Queue
is thread-safe and doesn't require Inotify.evMu to be held.

Closes #121

PiperOrigin-RevId: 227066695
Change-Id: Id29364bb940d1727f33a5dff9a3c52f390c15761

3 files changed, 59 insertions, 28 deletions

diff --git a/test/syscalls/linux/BUILD b/test/syscalls/linux/BUILD
index 0f9b406d8..ae33d14da 100644
--- a/test/syscalls/linux/BUILD
+++ b/test/syscalls/linux/BUILD
@@ -496,6 +496,7 @@ cc_binary(
     srcs = ["epoll.cc"],
     linkstatic = 1,
     deps = [
+        "//test/util:epoll_util",
         "//test/util:file_descriptor",
         "//test/util:posix_error",
         "//test/util:test_main",
@@ -844,6 +845,7 @@ cc_binary(
     srcs = ["inotify.cc"],
     linkstatic = 1,
     deps = [
+        "//test/util:epoll_util",
         "//test/util:file_descriptor",
         "//test/util:fs_util",
         "//test/util:temp_path",
@@ -852,6 +854,7 @@ cc_binary(
         "//test/util:thread_util",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/test/syscalls/linux/epoll.cc b/test/syscalls/linux/epoll.cc
index 9ae87c00b..46fba7b2d 100644
--- a/test/syscalls/linux/epoll.cc
+++ b/test/syscalls/linux/epoll.cc
@@ -25,6 +25,7 @@
 #include <unistd.h>
 
 #include "gtest/gtest.h"
+#include "test/util/epoll_util.h"
 #include "test/util/file_descriptor.h"
 #include "test/util/posix_error.h"
 #include "test/util/test_util.h"
@@ -37,18 +38,6 @@ namespace {
 constexpr int kFDsPerEpoll = 3;
 constexpr uint64_t kMagicConstant = 0x0102030405060708;
 
-// Returns a new epoll file descriptor.
-PosixErrorOr<FileDescriptor> NewEpollFD() {
-  // "Since Linux 2.6.8, the size argument is ignored, but must be greater than
-  // zero." - epoll_create(2)
-  int fd = epoll_create(/* size = */ 1);
-  MaybeSave();
-  if (fd < 0) {
-    return PosixError(errno, "epoll_create");
-  }
-  return FileDescriptor(fd);
-}
-
 // Returns a new eventfd.
 PosixErrorOr<FileDescriptor> NewEventFD() {
   int fd = eventfd(/* initval = */ 0, /* flags = */ 0);
@@ -59,22 +48,6 @@ PosixErrorOr<FileDescriptor> NewEventFD() {
   return FileDescriptor(fd);
 }
 
-// Registers `target_fd` with the epoll instance represented by `epoll_fd` for
-// the epoll events `events`. Events on `target_fd` will be indicated by setting
-// data.u64 to `data` in the returned epoll_event.
-PosixError RegisterEpollFD(int epoll_fd, int target_fd, int events,
-                           uint64_t data) {
-  struct epoll_event event;
-  event.events = events;
-  event.data.u64 = data;
-  int rc = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, target_fd, &event);
-  MaybeSave();
-  if (rc < 0) {
-    return PosixError(errno, "epoll_ctl");
-  }
-  return NoError();
-}
-
 uint64_t ms_elapsed(const struct timespec* begin, const struct timespec* end) {
   return (end->tv_sec - begin->tv_sec) * 1000 +
          (end->tv_nsec - begin->tv_nsec) / 1000000;
diff --git a/test/syscalls/linux/inotify.cc b/test/syscalls/linux/inotify.cc
index 0e361496c..167ca44a8 100644
--- a/test/syscalls/linux/inotify.cc
+++ b/test/syscalls/linux/inotify.cc
@@ -14,9 +14,12 @@
 
 #include <fcntl.h>
 #include <libgen.h>
+#include <sched.h>
+#include <sys/epoll.h>
 #include <sys/inotify.h>
 #include <sys/ioctl.h>
 
+#include <atomic>
 #include <list>
 #include <string>
 #include <vector>
@@ -24,6 +27,9 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
+#include "absl/time/clock.h"
+#include "absl/time/time.h"
+#include "test/util/epoll_util.h"
 #include "test/util/file_descriptor.h"
 #include "test/util/fs_util.h"
 #include "test/util/temp_path.h"
@@ -1512,6 +1518,55 @@ TEST(Inotify, ControlEvents) {
   ASSERT_THAT(events2, Are({}));
 }
 
+// Regression test to ensure epoll and directory access doesn't deadlock.
+TEST(Inotify, EpollNoDeadlock) {
+  const DisableSave ds;  // Too many syscalls.
+
+  const FileDescriptor fd =
+      ASSERT_NO_ERRNO_AND_VALUE(InotifyInit1(IN_NONBLOCK));
+
+  // Create lots of directories and watch all of them.
+  const TempPath root = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
+  std::vector<TempPath> children;
+  for (size_t i = 0; i < 1000; ++i) {
+    auto child = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDirIn(root.path()));
+    ASSERT_NO_ERRNO_AND_VALUE(
+        InotifyAddWatch(fd.get(), child.path(), IN_ACCESS));
+    children.emplace_back(std::move(child));
+  }
+
+  // Run epoll_wait constantly in a separate thread.
+  std::atomic<bool> done(false);
+  ScopedThread th([&fd, &done] {
+    for (auto start = absl::Now(); absl::Now() - start < absl::Seconds(5);) {
+      FileDescriptor epoll_fd = ASSERT_NO_ERRNO_AND_VALUE(NewEpollFD());
+      ASSERT_NO_ERRNO(RegisterEpollFD(epoll_fd.get(), fd.get(),
+                                      EPOLLIN | EPOLLOUT | EPOLLET, 0));
+      struct epoll_event result[1];
+      EXPECT_THAT(RetryEINTR(epoll_wait)(epoll_fd.get(), result, 1, -1),
+                  SyscallSucceedsWithValue(1));
+
+      sched_yield();
+    }
+    done = true;
+  });
+
+  // While epoll thread is running, constantly access all directories to
+  // generate inotify events.
+  while (!done) {
+    std::vector<std::string> files =
+        ASSERT_NO_ERRNO_AND_VALUE(ListDir(root.path(), false));
+    ASSERT_EQ(files.size(), 1002);
+    for (const auto& child : files) {
+      if (child == "." || child == "..") {
+        continue;
+      }
+      ASSERT_NO_ERRNO_AND_VALUE(ListDir(JoinPath(root.path(), child), false));
+    }
+    sched_yield();
+  }
+}
+
 }  // namespace
 }  // namespace testing
 }  // namespace gvisor