Extend integration test to test sequence of FUSE operation

Original FUSE integration test has limited capabilities. To test more
situations, the new integration test framework introduces a protocol
to communicate between testing thread and the FUSE server. In summary,
this change includes:

1. Remove CompareResult() and break SetExpected() into
   SetServerResponse() and GetServerActualRequest(). We no longer set
   up an expected request because we want to retrieve the actual FUSE
   request made to the FUSE server and check in the testing thread.

2. Declare a serial buffer data structure to save the received requests
   and expected responses sequentially. The data structure contains a
   cursor to indicate the progress of accessing. This change makes
   sequential SetServerResponse() and GetServerActualRequest() possible.

3. Replace 2 single directional pipes with 1 bi-directional socketpair.
   A protocol which starts with FuseTestCmd is used between the testing
   thread and the FUSE server to provide various functionality.

Fixes #3405

6 files changed, 477 insertions, 231 deletions

diff --git a/pkg/sentry/fsimpl/fuse/dev.go b/pkg/sentry/fsimpl/fuse/dev.go
index e522ff9a0..0efd2d90d 100644
--- a/pkg/sentry/fsimpl/fuse/dev.go
+++ b/pkg/sentry/fsimpl/fuse/dev.go
@@ -307,6 +307,14 @@ func (fd *DeviceFD) writeLocked(ctx context.Context, src usermem.IOSequence, opt
 
 // Readiness implements vfs.FileDescriptionImpl.Readiness.
 func (fd *DeviceFD) Readiness(mask waiter.EventMask) waiter.EventMask {
+	fd.mu.Lock()
+	defer fd.mu.Unlock()
+	return fd.readinessLocked(mask)
+}
+
+// readinessLocked implements checking the readiness of the fuse device while
+// locked with DeviceFD.mu.
+func (fd *DeviceFD) readinessLocked(mask waiter.EventMask) waiter.EventMask {
 	var ready waiter.EventMask
 	ready |= waiter.EventOut // FD is always writable
 	if !fd.queue.Empty() {
diff --git a/test/fuse/BUILD b/test/fuse/BUILD
index 56157c96b..385920e17 100644
--- a/test/fuse/BUILD
+++ b/test/fuse/BUILD
@@ -5,5 +5,4 @@ package(licenses = ["notice"])
 syscall_test(
     fuse = "True",
     test = "//test/fuse/linux:stat_test",
-    vfs2 = "True",
 )
diff --git a/test/fuse/README.md b/test/fuse/README.md
index 734c3a4e3..c5909a166 100644
--- a/test/fuse/README.md
+++ b/test/fuse/README.md
@@ -1,55 +1,90 @@
 # gVisor FUSE Test Suite
 
-This is an integration test suite for fuse(4) filesystem. It runs under both
-gVisor and Linux, and ensures compatibility between the two. This test suite is
-based on system calls test.
+This is an integration test suite for fuse(4) filesystem. It runs under gVisor
+sandbox container with VFS2 and FUSE function enabled.
 
-This document describes the framework of fuse integration test and the
-guidelines that should be followed when adding new fuse tests.
+This document describes the framework of FUSE integration test, how to use it,
+and the guidelines that should be followed when adding new testing features.
 
 ## Integration Test Framework
 
-Please refer to the figure below. `>` is entering the function, `<` is leaving
-the function, and `=` indicates sequentially entering and leaving.
+By inheriting the `FuseTest` class defined in `linux/fuse_base.h`, every test
+fixture can run in an environment with `mount_point_` mounted by a fake FUSE
+server. It creates a `socketpair(2)` to send and receive control commands and
+data between the client and the server. Because the FUSE server runs in the
+background thread, gTest cannot catch its assertion failure immediately. Thus,
+`TearDown()` function sends command to the FUSE server to check if all gTest
+assertion in the server are successful and all requests and preset responses
+are consumed.
+
+## Communication Diagram
+
+Diagram below describes how a testing thread communicates with the FUSE server
+to achieve integration test.
+
+For the following diagram, `>` means entering the function, `<` is leaving the
+function, and `=` indicates sequentially entering and leaving. Not necessarily
+follow exactly the below diagram due to the nature of a multi-threaded system,
+however, it is still helpful to know when the client waits for the server to
+complete a command and when the server awaits the next instruction.
 
 ```
- |  Client (Test Main Process)         |  Server (FUSE Daemon)
+ |  Client (Testing Thread)            |  Server (FUSE Server Thread)
  |                                     |
  |  >TEST_F()                          |
  |    >SetUp()                         |
  |      =MountFuse()                   |
  |      >SetUpFuseServer()             |
- |        [create communication pipes] |
- |        =fork()                      |        =fork()
- |        >WaitCompleted()             |
- |          [wait for MarkDone()]      |
- |                                     |        =ConsumeFuseInit()
- |                                     |        =MarkDone()
- |        <WaitCompleted()             |
+ |        [create communication socket]|
+ |        =fork()                      |      =fork()
+ |        [wait server complete]       |
+ |                                     |      =ServerConsumeFuseInit()
+ |                                     |      =ServerCompleteWith()
  |      <SetUpFuseServer()             |
  |    <SetUp()                         |
- |    >SetExpected()                   |
- |      [construct expected reaction]  |
- |                                     |        >FuseLoop()
- |                                     |          >ReceiveExpected()
- |                                     |            [wait data from pipe]
- |      [write data to pipe]           |
- |      [wait for MarkDone()]          |
+ |    [testing main]                   |
+ |                                     |      >ServerFuseLoop()
+ |                                     |        [poll on socket and fd]
+ |    >SetServerResponse()             |
+ |      [write data to socket]         |
+ |      [wait server complete]         |
+ |                                     |        [socket event occurs]
+ |                                     |        >ServerHandleCommand()
+ |                                     |          >ServerReceiveResponse()
+ |                                     |            [read data from socket]
  |                                     |            [save data to memory]
- |                                     |            =MarkDone()
- |    <SetExpected()                   |
- |                                     |          <ReceiveExpected()
- |                                     |          >read()
- |                                     |            [wait for fs operation]
+ |                                     |          <ServerReceiveResponse()
+ |                                     |          =ServerCompleteWith()
+ |    <SetServerResponse()             |
+ |                                     |        <ServerHandleCommand()
  |    >[Do fs operation]               |
  |      [wait for fs response]         |
- |                                     |          <read()
- |                                     |          =CompareRequest()
- |                                     |          =write() [write fs response]
+ |                                     |        [fd event occurs]
+ |                                     |        >ServerProcessFuseRequest()
+ |                                     |          =[read fs request]
+ |                                     |          =[save fs request to memory]
+ |                                     |          =[write fs response]
  |    <[Do fs operation]               |
+ |                                     |        <ServerProcessFuseRequest()
+ |                                     |
  |    =[Test fs operation result]      |
- |    =[wait for MarkDone()]           |
- |                                     |          =MarkDone()
+ |                                     |
+ |    >GetServerActualRequest()        |
+ |      [write data to socket]         |
+ |      [wait data from server]        |
+ |                                     |        [socket event occurs]
+ |                                     |        >ServerHandleCommand()
+ |                                     |          >ServerSendReceivedRequest()
+ |                                     |            [write data to socket]
+ |      [read data from socket]        |
+ |      [wait server complete]         |
+ |                                     |          <ServerSendReceivedRequest()
+ |                                     |          =ServerCompleteWith()
+ |    <GetServerActualRequest()        |
+ |                                     |        <ServerHandleCommand()
+ |                                     |
+ |    =[Test actual request]           |
+ |                                     |
  |    >TearDown()                      |
  |      =UnmountFuse()                 |
  |    <TearDown()                      |
@@ -58,8 +93,8 @@ the function, and `=` indicates sequentially entering and leaving.
 
 ## Running the tests
 
-Based on syscall tests, fuse tests can run in different environments. To enable
-fuse testing environment, the test targets should be appended with `_fuse`.
+Based on syscall tests, FUSE tests generate targets only with vfs2 and fuse
+enabled. The corresponding targets end in `_fuse`.
 
 For example, to run fuse test in `stat_test.cc`:
 
@@ -75,19 +110,17 @@ $ bazel test --test_tag_filters=fuse //test/fuse/...
 
 ## Writing a new FUSE test
 
-1.  Add test targets in `BUILD` and `linux/BUILD`.
-2.  Inherit your test from `FuseTest` base class. It allows you to:
-    -   Run a fake FUSE server in background during each test setup.
-    -   Create pipes for communication and provide utility functions.
-    -   Stop FUSE server after test completes.
-3.  Customize your comparison function for request assessment in FUSE server.
-4.  Add the mapping of the size of structs if you are working on new FUSE
-    opcode.
-    -   Please update `FuseTest::GetPayloadSize()` for each new FUSE opcode.
-5.  Build the expected request-response pair of your FUSE operation.
-6.  Call `SetExpected()` function to inject the expected reaction.
-7.  Check the response and/or errors.
-8.  Finally call `WaitCompleted()` to ensure the FUSE server acts correctly.
+1. Add test targets in `BUILD` and `linux/BUILD`.
+2. Inherit your test from `FuseTest` base class. It allows you to:
+  - Fork a fake FUSE server in background during each test setup.
+  - Create a pair of sockets for communication and provide utility functions.
+  - Stop FUSE server and check if error occurs in it after test completes.
+3. Build the expected opcode-response pairs of your FUSE operation.
+4. Call `SetServerResponse()` to preset the next expected opcode and response.
+5. Do real filesystem operations (FUSE is mounted at `mount_point_`).
+6. Check FUSE response and/or errors.
+7. Retrieve FUSE request by `GetServerActualRequest()`.
+8. Check if the request is as expected.
 
 A few customized matchers used in syscalls test are encouraged to test the
 outcome of filesystem operations. Such as:
@@ -101,3 +134,41 @@ SyscallFailsWithErrno(...)
 
 Please refer to [test/syscalls/README.md](../syscalls/README.md) for further
 details.
+
+## Writing a new FuseTestCmd
+
+A `FuseTestCmd` is a control protocol used in the communication between the
+testing thread and the FUSE server. Such commands are sent from the testing
+thread to the FUSE server to set up, control, or inspect the behavior of the
+FUSE server in response to a sequence of FUSE requests.
+
+The lifecycle of a command contains following steps:
+
+1. The testing thread sends a `FuseTestCmd` via socket and waits for completion.
+2. The FUSE server receives the command and does corresponding action.
+3. (Optional) The testing thread reads data from socket.
+4. The FUSE server sends a success indicator via socket after processing.
+5. The testing thread gets the success signal and continues testing.
+
+The success indicator, i.e. `WaitServerComplete()`, is crucial at the end of
+each `FuseTestCmd` sent from the testing thread. Because we don't want to begin
+filesystem operation if the requests have not been completely set up. Also, to
+test FUSE interactions in a sequential manner, concurrent requests are not
+supported now.
+
+To add a new `FuseTestCmd`, one must comply with following format:
+
+1. Add a new `FuseTestCmd` enum class item defined in `linux/fuse_base.h`
+2. Add a `SetServerXXX()` or `GetServerXXX()` public function in `FuseTest`.
+   This is how the testing thread will call to send control message. Define how
+   many bytes you want to send along with the command and what you will expect
+   to receive. Finally it should block and wait for a success indicator from
+   the FUSE server.
+3. Add a `ServerReceiveXXX()` or `ServerSendXXX()` private function in
+   `FuseTest`. It is mandatory to set it private since only the FUSE server
+   (forked from `FuseTest` base class) can call it. This is the handler of a
+   specific `FuseTestCmd` and the format of the data should be consistent with
+   what client expects in the previous step.
+4. Add a case in the switch condition of `ServerHandleCommand()` to route the
+   command to the server handler described in the previous step.
+
diff --git a/test/fuse/linux/fuse_base.cc b/test/fuse/linux/fuse_base.cc
index 9c3124472..b7d8b2a1f 100644
--- a/test/fuse/linux/fuse_base.cc
+++ b/test/fuse/linux/fuse_base.cc
@@ -16,17 +16,16 @@
 
 #include <fcntl.h>
 #include <linux/fuse.h>
-#include <string.h>
+#include <poll.h>
 #include <sys/mount.h>
+#include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/uio.h>
 #include <unistd.h>
 
-#include <iostream>
-
-#include "gtest/gtest.h"
 #include "absl/strings/str_format.h"
+#include "gtest/gtest.h"
 #include "test/util/posix_error.h"
 #include "test/util/temp_path.h"
 #include "test/util/test_util.h"
@@ -41,37 +40,47 @@ void FuseTest::SetUp() {
 
 void FuseTest::TearDown() { UnmountFuse(); }
 
-// Since CompareRequest is running in background thread, gTest assertions and
-// expectations won't directly reflect the test result. However, the FUSE
-// background server still connects to the same standard I/O as testing main
-// thread. So EXPECT_XX can still be used to show different results. To
-// ensure failed testing result is observable, return false and the result
-// will be sent to test main thread via pipe.
-bool FuseTest::CompareRequest(void* expected_mem, size_t expected_len,
-                              void* real_mem, size_t real_len) {
-  if (expected_len != real_len) return false;
-  return memcmp(expected_mem, real_mem, expected_len) == 0;
-}
+// Sends 3 parts of data to the FUSE server:
+//   1. The `kSetResponse` command
+//   2. The expected opcode
+//   3. The fake FUSE response
+// Then waits for the FUSE server to notify its completion.
+void FuseTest::SetServerResponse(uint32_t opcode,
+                                 std::vector<struct iovec>& iovecs) {
+  uint32_t cmd = static_cast<uint32_t>(FuseTestCmd::kSetResponse);
+  EXPECT_THAT(RetryEINTR(write)(sock_[0], &cmd, sizeof(cmd)),
+              SyscallSucceedsWithValue(sizeof(cmd)));
+
+  EXPECT_THAT(RetryEINTR(write)(sock_[0], &opcode, sizeof(opcode)),
+              SyscallSucceedsWithValue(sizeof(opcode)));
 
-// SetExpected is called by the testing main thread to set expected request-
-// response pair of a single FUSE operation.
-void FuseTest::SetExpected(struct iovec* iov_in, int iov_in_cnt,
-                           struct iovec* iov_out, int iov_out_cnt) {
-  EXPECT_THAT(RetryEINTR(writev)(set_expected_[1], iov_in, iov_in_cnt),
-              SyscallSucceedsWithValue(::testing::Gt(0)));
-  WaitCompleted();
+  EXPECT_THAT(RetryEINTR(writev)(sock_[0], iovecs.data(), iovecs.size()),
+              SyscallSucceeds());
 
-  EXPECT_THAT(RetryEINTR(writev)(set_expected_[1], iov_out, iov_out_cnt),
-              SyscallSucceedsWithValue(::testing::Gt(0)));
-  WaitCompleted();
+  WaitServerComplete();
 }
 
-// WaitCompleted waits for the FUSE server to finish its job and check if it
+// Waits for the FUSE server to finish its blocking job and check if it
 // completes without errors.
-void FuseTest::WaitCompleted() {
+void FuseTest::WaitServerComplete() {
   char success;
-  EXPECT_THAT(RetryEINTR(read)(done_[0], &success, sizeof(success)),
-              SyscallSucceedsWithValue(1));
+  EXPECT_THAT(RetryEINTR(read)(sock_[0], &success, sizeof(success)),
+              SyscallSucceedsWithValue(sizeof(success)));
+  EXPECT_EQ(success, static_cast<char>(1));
+}
+
+// Sends the `kGetRequest` command to the FUSE server, then reads the next
+// request into iovec struct. The order of calling this function should be
+// the same as the one of SetServerResponse().
+void FuseTest::GetServerActualRequest(std::vector<struct iovec>& iovecs) {
+  uint32_t cmd = static_cast<uint32_t>(FuseTestCmd::kGetRequest);
+  EXPECT_THAT(RetryEINTR(write)(sock_[0], &cmd, sizeof(cmd)),
+              SyscallSucceedsWithValue(sizeof(cmd)));
+
+  EXPECT_THAT(RetryEINTR(readv)(sock_[0], iovecs.data(), iovecs.size()),
+              SyscallSucceeds());
+
+  WaitServerComplete();
 }
 
 void FuseTest::MountFuse() {
@@ -81,7 +90,7 @@ void FuseTest::MountFuse() {
   mount_point_ = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
   EXPECT_THAT(mount("fuse", mount_point_.path().c_str(), "fuse",
                     MS_NODEV | MS_NOSUID, mount_opts.c_str()),
-              SyscallSucceedsWithValue(0));
+              SyscallSucceeds());
 }
 
 void FuseTest::UnmountFuse() {
@@ -89,11 +98,11 @@ void FuseTest::UnmountFuse() {
   // TODO(gvisor.dev/issue/3330): ensure the process is terminated successfully.
 }
 
-// ConsumeFuseInit consumes the first FUSE request and returns the
-// corresponding PosixError.
-PosixError FuseTest::ConsumeFuseInit() {
+// Consumes the first FUSE request and returns the corresponding PosixError.
+PosixError FuseTest::ServerConsumeFuseInit() {
+  std::vector<char> buf(FUSE_MIN_READ_BUFFER);
   RETURN_ERROR_IF_SYSCALL_FAIL(
-      RetryEINTR(read)(dev_fd_, buf_.data(), buf_.size()));
+      RetryEINTR(read)(dev_fd_, buf.data(), buf.size()));
 
   struct iovec iov_out[2];
   struct fuse_out_header out_header = {
@@ -115,60 +124,67 @@ PosixError FuseTest::ConsumeFuseInit() {
   return NoError();
 }
 
-// ReceiveExpected reads 1 pair of expected fuse request-response `iovec`s
-// from pipe and save them into member variables of this testing instance.
-void FuseTest::ReceiveExpected() {
-  // Set expected fuse_in request.
-  EXPECT_THAT(len_in_ = RetryEINTR(read)(set_expected_[0], mem_in_.data(),
-                                         mem_in_.size()),
-              SyscallSucceedsWithValue(::testing::Gt(0)));
-  MarkDone(len_in_ > 0);
+// Reads 1 expected opcode and a fake response from socket and save them into
+// the serial buffer of this testing instance.
+void FuseTest::ServerReceiveResponse() {
+  ssize_t len;
+  uint32_t opcode;
+  std::vector<char> buf(FUSE_MIN_READ_BUFFER);
+  EXPECT_THAT(RetryEINTR(read)(sock_[1], &opcode, sizeof(opcode)),
+              SyscallSucceedsWithValue(sizeof(opcode)));
 
-  // Set expected fuse_out response.
-  EXPECT_THAT(len_out_ = RetryEINTR(read)(set_expected_[0], mem_out_.data(),
-                                          mem_out_.size()),
-              SyscallSucceedsWithValue(::testing::Gt(0)));
-  MarkDone(len_out_ > 0);
+  EXPECT_THAT(len = RetryEINTR(read)(sock_[1], buf.data(), buf.size()),
+              SyscallSucceeds());
+
+  responses_.AddMemBlock(opcode, buf.data(), len);
 }
 
-// MarkDone writes 1 byte of success indicator through pipe.
-void FuseTest::MarkDone(bool success) {
-  char data = success ? 1 : 0;
-  EXPECT_THAT(RetryEINTR(write)(done_[1], &data, sizeof(data)),
-              SyscallSucceedsWithValue(1));
+// Writes 1 byte of success indicator through socket.
+void FuseTest::ServerCompleteWith(bool success) {
+  char data = static_cast<char>(success);
+  EXPECT_THAT(RetryEINTR(write)(sock_[1], &data, sizeof(data)),
+              SyscallSucceedsWithValue(sizeof(data)));
 }
 
-// FuseLoop is the implementation of the fake FUSE server. Read from /dev/fuse,
-// compare the request by CompareRequest (use derived function if specified),
-// and write the expected response to /dev/fuse.
-void FuseTest::FuseLoop() {
-  bool success = true;
-  ssize_t len = 0;
+// ServerFuseLoop is the implementation of the fake FUSE server. Monitors 2
+// file descriptors: /dev/fuse and sock_[1]. Events from /dev/fuse are FUSE
+// requests and events from sock_[1] are FUSE testing commands, leading by
+// a FuseTestCmd data to indicate the command.
+void FuseTest::ServerFuseLoop() {
+  const int nfds = 2;
+  struct pollfd fds[nfds] = {
+      {
+          .fd = dev_fd_,
+          .events = POLL_IN | POLLHUP | POLLERR | POLLNVAL,
+      },
+      {
+          .fd = sock_[1],
+          .events = POLL_IN | POLLHUP | POLLERR | POLLNVAL,
+      },
+  };
+
   while (true) {
-    ReceiveExpected();
+    ASSERT_THAT(poll(fds, nfds, -1), SyscallSucceeds());
 
-    EXPECT_THAT(len = RetryEINTR(read)(dev_fd_, buf_.data(), buf_.size()),
-                SyscallSucceedsWithValue(len_in_));
-    if (len != len_in_) success = false;
+    for (int fd_idx = 0; fd_idx < nfds; ++fd_idx) {
+      if (fds[fd_idx].revents == 0) continue;
 
-    if (!CompareRequest(buf_.data(), len_in_, mem_in_.data(), len_in_)) {
-      std::cerr << "the FUSE request is not expected" << std::endl;
-      success = false;
+      ASSERT_EQ(fds[fd_idx].revents, POLL_IN);
+      if (fds[fd_idx].fd == sock_[1]) {
+        ServerHandleCommand();
+      } else if (fds[fd_idx].fd == dev_fd_) {
+        ServerProcessFuseRequest();
+      }
     }
-
-    EXPECT_THAT(len = RetryEINTR(write)(dev_fd_, mem_out_.data(), len_out_),
-                SyscallSucceedsWithValue(len_out_));
-    if (len != len_out_) success = false;
-    MarkDone(success);
   }
 }
 
-// SetUpFuseServer creates 2 pipes. First is for testing client to send the
-// expected request-response pair, and the other acts as a checkpoint for the
-// FUSE server to notify the client that it can proceed.
+// SetUpFuseServer creates 1 socketpair and fork the process. The parent thread
+// becomes testing thread and the child thread becomes the FUSE server running
+// in background. These 2 threads are connected via socketpair. sock_[0] is
+// opened in testing thread and sock_[1] is opened in the FUSE server.
 void FuseTest::SetUpFuseServer() {
-  ASSERT_THAT(pipe(set_expected_), SyscallSucceedsWithValue(0));
-  ASSERT_THAT(pipe(done_), SyscallSucceedsWithValue(0));
+  ASSERT_THAT(socketpair(AF_UNIX, SOCK_STREAM, 0, sock_), SyscallSucceeds());
 
   switch (fork()) {
     case -1:
@@ -177,31 +193,110 @@ void FuseTest::SetUpFuseServer() {
     case 0:
       break;
     default:
-      ASSERT_THAT(close(set_expected_[0]), SyscallSucceedsWithValue(0));
-      ASSERT_THAT(close(done_[1]), SyscallSucceedsWithValue(0));
-      WaitCompleted();
+      ASSERT_THAT(close(sock_[1]), SyscallSucceeds());
+      WaitServerComplete();
       return;
   }
 
-  ASSERT_THAT(close(set_expected_[1]), SyscallSucceedsWithValue(0));
-  ASSERT_THAT(close(done_[0]), SyscallSucceedsWithValue(0));
-
-  MarkDone(ConsumeFuseInit().ok());
-
-  FuseLoop();
+  // Begin child thread, i.e. the FUSE server.
+  ASSERT_THAT(close(sock_[0]), SyscallSucceeds());
+  ServerCompleteWith(ServerConsumeFuseInit().ok());
+  ServerFuseLoop();
   _exit(0);
 }
 
-// GetPayloadSize is a helper function to get the number of bytes of a
-// specific FUSE operation struct.
-size_t FuseTest::GetPayloadSize(uint32_t opcode, bool in) {
-  switch (opcode) {
-    case FUSE_INIT:
-      return in ? sizeof(struct fuse_init_in) : sizeof(struct fuse_init_out);
+// Reads FuseTestCmd sent from testing thread and routes to correct handler.
+// Since each command should be a blocking operation, a `ServerCompleteWith()`
+// is required after the switch keyword.
+void FuseTest::ServerHandleCommand() {
+  uint32_t cmd;
+  EXPECT_THAT(RetryEINTR(read)(sock_[1], &cmd, sizeof(cmd)),
+              SyscallSucceedsWithValue(sizeof(cmd)));
+
+  switch (static_cast<FuseTestCmd>(cmd)) {
+    case FuseTestCmd::kSetResponse:
+      ServerReceiveResponse();
+      break;
+    case FuseTestCmd::kGetRequest:
+      ServerSendReceivedRequest();
+      break;
     default:
+      FAIL() << "Unknown FuseTestCmd " << cmd;
       break;
   }
-  return 0;
+
+  ServerCompleteWith(!HasFailure());
+}
+
+// Sends the received request pointed by current cursor and advances cursor.
+void FuseTest::ServerSendReceivedRequest() {
+  if (requests_.End()) {
+    FAIL() << "No more received request.";
+    return;
+  }
+  auto mem_block = requests_.Next();
+  EXPECT_THAT(
+      RetryEINTR(write)(sock_[1], requests_.DataAtOffset(mem_block.offset),
+                        mem_block.len),
+      SyscallSucceedsWithValue(mem_block.len));
+}
+
+// Handles FUSE request. Reads request from /dev/fuse, checks if it has the
+// same opcode as expected, and responds with the saved fake FUSE response.
+// The FUSE request is copied to the serial buffer and can be retrieved one-
+// by-one by calling GetServerActualRequest from testing thread.
+void FuseTest::ServerProcessFuseRequest() {
+  ssize_t len;
+  std::vector<char> buf(FUSE_MIN_READ_BUFFER);
+
+  // Read FUSE request.
+  EXPECT_THAT(len = RetryEINTR(read)(dev_fd_, buf.data(), buf.size()),
+              SyscallSucceeds());
+  fuse_in_header* in_header = reinterpret_cast<fuse_in_header*>(buf.data());
+  requests_.AddMemBlock(in_header->opcode, buf.data(), len);
+
+  // Check if there is a corresponding response.
+  if (responses_.End()) {
+    GTEST_NONFATAL_FAILURE_("No more FUSE response is expected");
+    ServerRespondFuseError(in_header->unique);
+    return;
+  }
+  auto mem_block = responses_.Next();
+  if (in_header->opcode != mem_block.opcode) {
+    std::string message = absl::StrFormat("Expect opcode %d but got %d",
+                                          mem_block.opcode, in_header->opcode);
+    GTEST_NONFATAL_FAILURE_(message.c_str());
+    // We won't get correct response if opcode is not expected. Send error
+    // response here to avoid wrong parsing by VFS.
+    ServerRespondFuseError(in_header->unique);
+    return;
+  }
+
+  // Write FUSE response.
+  ServerRespondFuseSuccess(responses_, mem_block, in_header->unique);
+}
+
+void FuseTest::ServerRespondFuseSuccess(FuseMemBuffer& mem_buf,
+                                        const FuseMemBlock& block,
+                                        uint64_t unique) {
+  fuse_out_header* out_header =
+      reinterpret_cast<fuse_out_header*>(mem_buf.DataAtOffset(block.offset));
+
+  // Patch `unique` in fuse_out_header to avoid EINVAL caused by responding
+  // with an unknown `unique`.
+  out_header->unique = unique;
+  EXPECT_THAT(RetryEINTR(write)(dev_fd_, out_header, block.len),
+              SyscallSucceedsWithValue(block.len));
+}
+
+void FuseTest::ServerRespondFuseError(uint64_t unique) {
+  fuse_out_header out_header = {
+      .len = sizeof(struct fuse_out_header),
+      .error = ENOSYS,
+      .unique = unique,
+  };
+  EXPECT_THAT(RetryEINTR(write)(dev_fd_, &out_header, sizeof(out_header)),
+              SyscallSucceedsWithValue(sizeof(out_header)));
 }
 
 }  // namespace testing
diff --git a/test/fuse/linux/fuse_base.h b/test/fuse/linux/fuse_base.h
index 3a2f255a9..b610d0f54 100644
--- a/test/fuse/linux/fuse_base.h
+++ b/test/fuse/linux/fuse_base.h
@@ -16,8 +16,10 @@
 #define GVISOR_TEST_FUSE_FUSE_BASE_H_
 
 #include <linux/fuse.h>
+#include <string.h>
 #include <sys/uio.h>
 
+#include <iostream>
 #include <vector>
 
 #include "gtest/gtest.h"
@@ -29,68 +31,156 @@ namespace testing {
 
 constexpr char kMountOpts[] = "rootmode=755,user_id=0,group_id=0";
 
-class FuseTest : public ::testing::Test {
+// Internal commands used to communicate between testing thread and the FUSE
+// server. See test/fuse/README.md for further detail.
+enum class FuseTestCmd {
+  kSetResponse = 0,
+  kGetRequest,
+};
+
+// Holds the information of a memory block in a serial buffer.
+struct FuseMemBlock {
+  uint32_t opcode;
+  size_t offset;
+  size_t len;
+};
+
+// A wrapper of a simple serial buffer that can be used with read(2) and
+// write(2). Contains a cursor to indicate accessing. This class is not thread-
+// safe and can only be used in single-thread version.
+class FuseMemBuffer {
  public:
-  FuseTest() {
-    buf_.resize(FUSE_MIN_READ_BUFFER);
-    mem_in_.resize(FUSE_MIN_READ_BUFFER);
-    mem_out_.resize(FUSE_MIN_READ_BUFFER);
+  FuseMemBuffer() : cursor_(0) {
+    // To read from /dev/fuse, a buffer needs at least FUSE_MIN_READ_BUFFER
+    // bytes to avoid EINVAL. FuseMemBuffer holds memory that can accommodate
+    // a sequence of FUSE request/response, so it is initiated with double
+    // minimal requirement.
+    mem_.resize(FUSE_MIN_READ_BUFFER * 2);
   }
+
+  // Returns whether there is no memory block.
+  bool Empty() { return blocks_.empty(); }
+
+  // Returns if there is no more remaining memory blocks.
+  bool End() { return cursor_ == blocks_.size(); }
+
+  // Returns how many bytes that have been received.
+  size_t UsedBytes() {
+    return Empty() ? 0 : blocks_.back().offset + blocks_.back().len;
+  }
+
+  // Returns the available bytes remains in the serial buffer.
+  size_t AvailBytes() { return mem_.size() - UsedBytes(); }
+
+  // Appends a memory block information that starts at the tail of the serial
+  // buffer. /dev/fuse requires at least FUSE_MIN_READ_BUFFER bytes to read, or
+  // it will issue EINVAL. If it is not enough, just double the buffer length.
+  void AddMemBlock(uint32_t opcode, void* data, size_t len) {
+    if (AvailBytes() < FUSE_MIN_READ_BUFFER) {
+      mem_.resize(mem_.size() << 1);
+    }
+    size_t offset = UsedBytes();
+    memcpy(mem_.data() + offset, data, len);
+    blocks_.push_back(FuseMemBlock{opcode, offset, len});
+  }
+
+  // Returns the memory address at a specific offset. Used with read(2) or
+  // write(2).
+  char* DataAtOffset(size_t offset) { return mem_.data() + offset; }
+
+  // Returns current memory block pointed by the cursor and increase by 1.
+  FuseMemBlock Next() {
+    if (End()) {
+      std::cerr << "Buffer is already exhausted." << std::endl;
+      return FuseMemBlock{};
+    }
+    return blocks_[cursor_++];
+  }
+
+  // Returns the number of the blocks that has not been requested.
+  size_t RemainingBlocks() { return blocks_.size() - cursor_; }
+
+ private:
+  size_t cursor_;
+  std::vector<FuseMemBlock> blocks_;
+  std::vector<char> mem_;
+};
+
+// FuseTest base class is useful in FUSE integration test. Inherit this class
+// to automatically set up a fake FUSE server and use the member functions
+// to manipulate with it. Refer to test/fuse/README.md for detailed explanation.
+class FuseTest : public ::testing::Test {
+ public:
   void SetUp() override;
   void TearDown() override;
 
-  // CompareRequest is used by the FUSE server and should be implemented to
-  // compare different FUSE operations. It compares the actual FUSE input
-  // request with the expected one set by `SetExpected()`.
-  virtual bool CompareRequest(void* expected_mem, size_t expected_len,
-                              void* real_mem, size_t real_len);
-
-  // SetExpected is called by the testing main thread. Writes a request-
-  // response pair into FUSE server's member variables via pipe.
-  void SetExpected(struct iovec* iov_in, int iov_in_cnt, struct iovec* iov_out,
-                   int iov_out_cnt);
+  // Called by the testing thread to set up a fake response for an expected
+  // opcode via socket. This can be used multiple times to define a sequence of
+  // expected FUSE reactions.
+  void SetServerResponse(uint32_t opcode, std::vector<struct iovec>& iovecs);
 
-  // WaitCompleted waits for FUSE server to complete its processing. It
-  // complains if the FUSE server responds failure during tests.
-  void WaitCompleted();
+  // Called by the testing thread to ask the FUSE server for its next received
+  // FUSE request. Be sure to use the corresponding struct of iovec to receive
+  // data from server.
+  void GetServerActualRequest(std::vector<struct iovec>& iovecs);
 
  protected:
   TempPath mount_point_;
 
  private:
+  // Opens /dev/fuse and inherit the file descriptor for the FUSE server.
   void MountFuse();
+
+  // Unmounts the mountpoint of the FUSE server.
   void UnmountFuse();
 
-  // ConsumeFuseInit is only used during FUSE server setup.
-  PosixError ConsumeFuseInit();
+  // Creates a socketpair for communication and forks FUSE server.
+  void SetUpFuseServer();
 
-  // ReceiveExpected is the FUSE server side's corresponding code of
-  // `SetExpected()`. Save the request-response pair into its memory.
-  void ReceiveExpected();
+  // Waits for FUSE server to complete its processing. Complains if the FUSE
+  // server responds any failure during tests.
+  void WaitServerComplete();
 
-  // MarkDone is used by the FUSE server to tell testing main if it's OK to
-  // proceed next command.
-  void MarkDone(bool success);
+  // The FUSE server stays here and waits next command or FUSE request until it
+  // is terminated.
+  void ServerFuseLoop();
 
-  // FuseLoop is where the FUSE server stay until it is terminated.
-  void FuseLoop();
+  // Used by the FUSE server to tell testing thread if it is OK to proceed next
+  // command. Will be issued after processing each FuseTestCmd.
+  void ServerCompleteWith(bool success);
 
-  // SetUpFuseServer creates 2 pipes for communication and forks FUSE server.
-  void SetUpFuseServer();
+  // Consumes the first FUSE request when mounting FUSE. Replies with a
+  // response with empty payload.
+  PosixError ServerConsumeFuseInit();
+
+  // A command switch that dispatch different FuseTestCmd to its handler.
+  void ServerHandleCommand();
+
+  // The FUSE server side's corresponding code of `SetServerResponse()`.
+  // Handles `kSetResponse` command. Saves the fake response into its output
+  // memory queue.
+  void ServerReceiveResponse();
+
+  // The FUSE server side's corresponding code of `GetServerActualRequest()`.
+  // Handles `kGetRequest` command. Sends the next received request pointed by
+  // the cursor.
+  void ServerSendReceivedRequest();
 
-  // GetPayloadSize is a helper function to get the number of bytes of a
-  // specific FUSE operation struct.
-  size_t GetPayloadSize(uint32_t opcode, bool in);
+  // Handles FUSE request sent to /dev/fuse by its saved responses.
+  void ServerProcessFuseRequest();
+
+  // Responds to FUSE request with a saved data.
+  void ServerRespondFuseSuccess(FuseMemBuffer& mem_buf,
+                                const FuseMemBlock& block, uint64_t unique);
+
+  // Responds an error header to /dev/fuse when bad thing happens.
+  void ServerRespondFuseError(uint64_t unique);
 
   int dev_fd_;
-  int set_expected_[2];
-  int done_[2];
-
-  std::vector<char> buf_;
-  std::vector<char> mem_in_;
-  std::vector<char> mem_out_;
-  ssize_t len_in_;
-  ssize_t len_out_;
+  int sock_[2];
+
+  FuseMemBuffer requests_;
+  FuseMemBuffer responses_;
 };
 
 }  // namespace testing
diff --git a/test/fuse/linux/stat_test.cc b/test/fuse/linux/stat_test.cc
index 172e09867..c2e5bd1cf 100644
--- a/test/fuse/linux/stat_test.cc
+++ b/test/fuse/linux/stat_test.cc
@@ -33,20 +33,6 @@ namespace {
 
 class StatTest : public FuseTest {
  public:
-  bool CompareRequest(void* expected_mem, size_t expected_len, void* real_mem,
-                      size_t real_len) override {
-    if (expected_len != real_len) return false;
-    struct fuse_in_header* real_header =
-        reinterpret_cast<fuse_in_header*>(real_mem);
-
-    if (real_header->opcode != FUSE_GETATTR) {
-      std::cerr << "expect header opcode " << FUSE_GETATTR << " but got "
-                << real_header->opcode << std::endl;
-      return false;
-    }
-    return true;
-  }
-
   bool StatsAreEqual(struct stat expected, struct stat actual) {
     // device number will be dynamically allocated by kernel, we cannot know
     // in advance
@@ -56,25 +42,9 @@ class StatTest : public FuseTest {
 };
 
 TEST_F(StatTest, StatNormal) {
-  struct iovec iov_in[2];
-  struct iovec iov_out[2];
-
-  struct fuse_in_header in_header = {
-      .len = sizeof(struct fuse_in_header) + sizeof(struct fuse_getattr_in),
-      .opcode = FUSE_GETATTR,
-      .unique = 4,
-      .nodeid = 1,
-      .uid = 0,
-      .gid = 0,
-      .pid = 4,
-      .padding = 0,
-  };
-  struct fuse_getattr_in in_payload = {0};
-  iov_in[0].iov_len = sizeof(in_header);
-  iov_in[0].iov_base = &in_header;
-  iov_in[1].iov_len = sizeof(in_payload);
-  iov_in[1].iov_base = &in_payload;
-
+  // Set up fixture.
+  std::vector<struct iovec> iov_in(2);
+  std::vector<struct iovec> iov_out(2);
   mode_t expected_mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
   struct timespec atime = {.tv_sec = 1595436289, .tv_nsec = 134150844};
   struct timespec mtime = {.tv_sec = 1595436290, .tv_nsec = 134150845};
@@ -82,7 +52,6 @@ TEST_F(StatTest, StatNormal) {
   struct fuse_out_header out_header = {
       .len = sizeof(struct fuse_out_header) + sizeof(struct fuse_attr_out),
       .error = 0,
-      .unique = 4,
   };
   struct fuse_attr attr = {
       .ino = 1,
@@ -109,11 +78,13 @@ TEST_F(StatTest, StatNormal) {
   iov_out[1].iov_len = sizeof(out_payload);
   iov_out[1].iov_base = &out_payload;
 
-  SetExpected(iov_in, 2, iov_out, 2);
+  SetServerResponse(FUSE_GETATTR, iov_out);
 
+  // Do integration test.
   struct stat stat_buf;
   EXPECT_THAT(stat(mount_point_.path().c_str(), &stat_buf), SyscallSucceeds());
 
+  // Check filesystem operation result.
   struct stat expected_stat = {
       .st_ino = attr.ino,
       .st_nlink = attr.nlink,
@@ -129,38 +100,50 @@ TEST_F(StatTest, StatNormal) {
       .st_ctim = ctime,
   };
   EXPECT_TRUE(StatsAreEqual(stat_buf, expected_stat));
-  WaitCompleted();
-}
-
-TEST_F(StatTest, StatNotFound) {
-  struct iovec iov_in[2];
-  struct iovec iov_out[2];
 
-  struct fuse_in_header in_header = {
-      .len = sizeof(struct fuse_in_header) + sizeof(struct fuse_getattr_in),
-      .opcode = FUSE_GETATTR,
-      .unique = 4,
-  };
-  struct fuse_getattr_in in_payload = {0};
+  // Check FUSE request.
+  struct fuse_in_header in_header;
+  struct fuse_getattr_in in_payload;
   iov_in[0].iov_len = sizeof(in_header);
   iov_in[0].iov_base = &in_header;
   iov_in[1].iov_len = sizeof(in_payload);
   iov_in[1].iov_base = &in_payload;
 
+  GetServerActualRequest(iov_in);
+  EXPECT_EQ(in_header.opcode, FUSE_GETATTR);
+  EXPECT_EQ(in_payload.getattr_flags, 0);
+  EXPECT_EQ(in_payload.fh, 0);
+}
+
+TEST_F(StatTest, StatNotFound) {
+  // Set up fixture.
+  std::vector<struct iovec> iov_in(2);
+  std::vector<struct iovec> iov_out(1);
   struct fuse_out_header out_header = {
       .len = sizeof(struct fuse_out_header),
       .error = -ENOENT,
-      .unique = 4,
   };
   iov_out[0].iov_len = sizeof(out_header);
   iov_out[0].iov_base = &out_header;
+  SetServerResponse(FUSE_GETATTR, iov_out);
 
-  SetExpected(iov_in, 2, iov_out, 1);
-
+  // Do integration test.
   struct stat stat_buf;
   EXPECT_THAT(stat(mount_point_.path().c_str(), &stat_buf),
               SyscallFailsWithErrno(ENOENT));
-  WaitCompleted();
+
+  // Check FUSE request.
+  struct fuse_in_header in_header;
+  struct fuse_getattr_in in_payload;
+  iov_in[0].iov_len = sizeof(in_header);
+  iov_in[0].iov_base = &in_header;
+  iov_in[1].iov_len = sizeof(in_payload);
+  iov_in[1].iov_base = &in_payload;
+
+  GetServerActualRequest(iov_in);
+  EXPECT_EQ(in_header.opcode, FUSE_GETATTR);
+  EXPECT_EQ(in_payload.getattr_flags, 0);
+  EXPECT_EQ(in_payload.fh, 0);
 }
 
 }  // namespace