Add functions in FUSE integration test to get metrics from FUSE server

This commit adds 3 utility functions to ensure all received requests
and preset responses are consumed.

1. Get number of unconsumed requests (received by the FUSE server but
   not consumed by the testing thread).
2. Get number of unsent responses (set by the testing thread but not
   processed by the FUSE server).
3. Get total bytes of the received requests (to ensure some operations
   don't trigger FUSE requests).

Fixes #3607

3 files changed, 96 insertions, 13 deletions

diff --git a/test/fuse/README.md b/test/fuse/README.md
index c5909a166..7a1839714 100644
--- a/test/fuse/README.md
+++ b/test/fuse/README.md
@@ -86,6 +86,20 @@ complete a command and when the server awaits the next instruction.
  |    =[Test actual request]           |
  |                                     |
  |    >TearDown()                      |
+ |      ...                            |
+ |      >GetServerNumUnsentResponses() |
+ |        [write data to socket]       |
+ |        [wait server complete]       |
+ |                                     |        [socket event arrive]
+ |                                     |        >ServerHandleCommand()
+ |                                     |          >ServerSendData()
+ |                                     |            [write data to socket]
+ |                                     |          <ServerSendData()
+ |                                     |          =ServerCompleteWith()
+ |        [read data from socket]      |
+ |        [test if all succeeded]      |
+ |      <GetServerNumUnsentResponses() |
+ |                                     |        <ServerHandleCommand()
  |      =UnmountFuse()                 |
  |    <TearDown()                      |
  |  <TEST_F()                          |
@@ -164,11 +178,10 @@ To add a new `FuseTestCmd`, one must comply with following format:
    many bytes you want to send along with the command and what you will expect
    to receive. Finally it should block and wait for a success indicator from
    the FUSE server.
-3. Add a `ServerReceiveXXX()` or `ServerSendXXX()` private function in
-   `FuseTest`. It is mandatory to set it private since only the FUSE server
-   (forked from `FuseTest` base class) can call it. This is the handler of a
-   specific `FuseTestCmd` and the format of the data should be consistent with
-   what client expects in the previous step.
-4. Add a case in the switch condition of `ServerHandleCommand()` to route the
-   command to the server handler described in the previous step.
+3. Add a handler logic in the switch condition of `ServerHandleCommand()`. Use
+   `ServerSendData()` or declare a new private function such as
+   `ServerReceiveXXX()` or `ServerSendXXX()`. It is mandatory to set it private
+   since only the FUSE server (forked from `FuseTest` base class) can call it.
+   This is the server part of the specific `FuseTestCmd` and the format of the
+   data should be consistent with what the client expects in the previous step.
 
diff --git a/test/fuse/linux/fuse_base.cc b/test/fuse/linux/fuse_base.cc
index b7d8b2a1f..b1897cf88 100644
--- a/test/fuse/linux/fuse_base.cc
+++ b/test/fuse/linux/fuse_base.cc
@@ -38,7 +38,11 @@ void FuseTest::SetUp() {
   SetUpFuseServer();
 }
 
-void FuseTest::TearDown() { UnmountFuse(); }
+void FuseTest::TearDown() {
+  EXPECT_EQ(GetServerNumUnconsumedRequests(), 0);
+  EXPECT_EQ(GetServerNumUnsentResponses(), 0);
+  UnmountFuse();
+}
 
 // Sends 3 parts of data to the FUSE server:
 //   1. The `kSetResponse` command
@@ -63,10 +67,10 @@ void FuseTest::SetServerResponse(uint32_t opcode,
 // Waits for the FUSE server to finish its blocking job and check if it
 // completes without errors.
 void FuseTest::WaitServerComplete() {
-  char success;
+  uint32_t success;
   EXPECT_THAT(RetryEINTR(read)(sock_[0], &success, sizeof(success)),
               SyscallSucceedsWithValue(sizeof(success)));
-  EXPECT_EQ(success, static_cast<char>(1));
+  ASSERT_EQ(success, 1);
 }
 
 // Sends the `kGetRequest` command to the FUSE server, then reads the next
@@ -83,6 +87,35 @@ void FuseTest::GetServerActualRequest(std::vector<struct iovec>& iovecs) {
   WaitServerComplete();
 }
 
+// Sends a FuseTestCmd command to the FUSE server, reads from the socket, and
+// returns the corresponding data.
+uint32_t FuseTest::GetServerData(uint32_t cmd) {
+  uint32_t data;
+  EXPECT_THAT(RetryEINTR(write)(sock_[0], &cmd, sizeof(cmd)),
+              SyscallSucceedsWithValue(sizeof(cmd)));
+
+  EXPECT_THAT(RetryEINTR(read)(sock_[0], &data, sizeof(data)),
+              SyscallSucceedsWithValue(sizeof(data)));
+
+  WaitServerComplete();
+  return data;
+}
+
+uint32_t FuseTest::GetServerNumUnconsumedRequests() {
+  return GetServerData(
+      static_cast<uint32_t>(FuseTestCmd::kGetNumUnconsumedRequests));
+}
+
+uint32_t FuseTest::GetServerNumUnsentResponses() {
+  return GetServerData(
+      static_cast<uint32_t>(FuseTestCmd::kGetNumUnsentResponses));
+}
+
+uint32_t FuseTest::GetServerTotalReceivedBytes() {
+  return GetServerData(
+      static_cast<uint32_t>(FuseTestCmd::kGetTotalReceivedBytes));
+}
+
 void FuseTest::MountFuse() {
   EXPECT_THAT(dev_fd_ = open("/dev/fuse", O_RDWR), SyscallSucceeds());
 
@@ -141,9 +174,8 @@ void FuseTest::ServerReceiveResponse() {
 
 // Writes 1 byte of success indicator through socket.
 void FuseTest::ServerCompleteWith(bool success) {
-  char data = static_cast<char>(success);
-  EXPECT_THAT(RetryEINTR(write)(sock_[1], &data, sizeof(data)),
-              SyscallSucceedsWithValue(sizeof(data)));
+  uint32_t data = success ? 1 : 0;
+  ServerSendData(data);
 }
 
 // ServerFuseLoop is the implementation of the fake FUSE server. Monitors 2
@@ -205,6 +237,11 @@ void FuseTest::SetUpFuseServer() {
   _exit(0);
 }
 
+void FuseTest::ServerSendData(uint32_t data) {
+  EXPECT_THAT(RetryEINTR(write)(sock_[1], &data, sizeof(data)),
+              SyscallSucceedsWithValue(sizeof(data)));
+}
+
 // Reads FuseTestCmd sent from testing thread and routes to correct handler.
 // Since each command should be a blocking operation, a `ServerCompleteWith()`
 // is required after the switch keyword.
@@ -220,6 +257,15 @@ void FuseTest::ServerHandleCommand() {
     case FuseTestCmd::kGetRequest:
       ServerSendReceivedRequest();
       break;
+    case FuseTestCmd::kGetTotalReceivedBytes:
+      ServerSendData(static_cast<uint32_t>(requests_.UsedBytes()));
+      break;
+    case FuseTestCmd::kGetNumUnconsumedRequests:
+      ServerSendData(static_cast<uint32_t>(requests_.RemainingBlocks()));
+      break;
+    case FuseTestCmd::kGetNumUnsentResponses:
+      ServerSendData(static_cast<uint32_t>(responses_.RemainingBlocks()));
+      break;
     default:
       FAIL() << "Unknown FuseTestCmd " << cmd;
       break;
diff --git a/test/fuse/linux/fuse_base.h b/test/fuse/linux/fuse_base.h
index b610d0f54..3f2522977 100644
--- a/test/fuse/linux/fuse_base.h
+++ b/test/fuse/linux/fuse_base.h
@@ -36,6 +36,9 @@ constexpr char kMountOpts[] = "rootmode=755,user_id=0,group_id=0";
 enum class FuseTestCmd {
   kSetResponse = 0,
   kGetRequest,
+  kGetNumUnconsumedRequests,
+  kGetNumUnsentResponses,
+  kGetTotalReceivedBytes,
 };
 
 // Holds the information of a memory block in a serial buffer.
@@ -124,6 +127,21 @@ class FuseTest : public ::testing::Test {
   // data from server.
   void GetServerActualRequest(std::vector<struct iovec>& iovecs);
 
+  // Called by the testing thread to query the number of unconsumed requests in
+  // the requests_ serial buffer of the FUSE server. TearDown() ensures all
+  // FUSE requests received by the FUSE server were consumed by the testing
+  // thread.
+  uint32_t GetServerNumUnconsumedRequests();
+
+  // Called by the testing thread to query the number of unsent responses in
+  // the responses_ serial buffer of the FUSE server. TearDown() ensures all
+  // preset FUSE responses were sent out by the FUSE server.
+  uint32_t GetServerNumUnsentResponses();
+
+  // Called by the testing thread to ask the FUSE server for its total received
+  // bytes from /dev/fuse.
+  uint32_t GetServerTotalReceivedBytes();
+
  protected:
   TempPath mount_point_;
 
@@ -137,6 +155,9 @@ class FuseTest : public ::testing::Test {
   // Creates a socketpair for communication and forks FUSE server.
   void SetUpFuseServer();
 
+  // Sends a FuseTestCmd and gets a uint32_t data from the FUSE server.
+  inline uint32_t GetServerData(uint32_t cmd);
+
   // Waits for FUSE server to complete its processing. Complains if the FUSE
   // server responds any failure during tests.
   void WaitServerComplete();
@@ -166,6 +187,9 @@ class FuseTest : public ::testing::Test {
   // the cursor.
   void ServerSendReceivedRequest();
 
+  // Sends a uint32_t data via socket.
+  inline void ServerSendData(uint32_t data);
+
   // Handles FUSE request sent to /dev/fuse by its saved responses.
   void ServerProcessFuseRequest();