From 3733499952c056cc8496beb01c72dcf53177048e Mon Sep 17 00:00:00 2001
From: Zach Koopmans <zkoopmans@google.com>
Date: Fri, 21 Feb 2020 13:17:44 -0800
Subject: Fix master installer.

Sometimes, when we start a new instance, the file
lock on "apt" is locked. Add a loop to the master
installer.

In addition, the "apt-get install" fails to register
runsc in docker, so run the appropriate scripts to
get that to happen.

Also, add some helpful log messages.

PiperOrigin-RevId: 296497357
---
 benchmarks/harness/machine.py        |  9 ++++++---
 benchmarks/harness/ssh_connection.py |  9 +++++++--
 tools/installers/master.sh           | 17 ++++++++++++++++-
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/benchmarks/harness/machine.py b/benchmarks/harness/machine.py
index 3d32d3dda..5bdc4aa85 100644
--- a/benchmarks/harness/machine.py
+++ b/benchmarks/harness/machine.py
@@ -43,6 +43,8 @@ from benchmarks.harness import machine_mocks
 from benchmarks.harness import ssh_connection
 from benchmarks.harness import tunnel_dispatcher
 
+log = logging.getLogger(__name__)
+
 
 class Machine(object):
   """The machine object is the primary object for benchmarks.
@@ -236,9 +238,10 @@ class RemoteMachine(Machine):
           archive=archive, dir=harness.REMOTE_INSTALLERS_PATH))
       self._has_installers = True
 
-      # Execute the remote installer.
-      self.run("sudo {dir}/{file}".format(
-          dir=harness.REMOTE_INSTALLERS_PATH, file=installer))
+    # Execute the remote installer.
+    self.run("sudo {dir}/{file}".format(
+        dir=harness.REMOTE_INSTALLERS_PATH, file=installer))
+
     if results:
       results[index] = True
 
diff --git a/benchmarks/harness/ssh_connection.py b/benchmarks/harness/ssh_connection.py
index a50e34293..b8c8e42d4 100644
--- a/benchmarks/harness/ssh_connection.py
+++ b/benchmarks/harness/ssh_connection.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 """SSHConnection handles the details of SSH connections."""
 
-
+import logging
 import os
 import warnings
 
@@ -24,6 +24,8 @@ from benchmarks import harness
 # Get rid of paramiko Cryptography Warnings.
 warnings.filterwarnings(action="ignore", module=".*paramiko.*")
 
+log = logging.getLogger(__name__)
+
 
 def send_one_file(client: paramiko.SSHClient, path: str,
                   remote_dir: str) -> str:
@@ -94,10 +96,13 @@ class SSHConnection:
       The contents of stdout and stderr.
     """
     with self._client() as client:
+      log.info("running command: %s", cmd)
       _, stdout, stderr = client.exec_command(command=cmd)
-      stdout.channel.recv_exit_status()
+      log.info("returned status: %d", stdout.channel.recv_exit_status())
       stdout = stdout.read().decode("utf-8")
       stderr = stderr.read().decode("utf-8")
+      log.info("stdout: %s", stdout)
+      log.info("stderr: %s", stderr)
     return stdout, stderr
 
   def send_workload(self, name: str) -> str:
diff --git a/tools/installers/master.sh b/tools/installers/master.sh
index 7b1956454..52f9734a6 100755
--- a/tools/installers/master.sh
+++ b/tools/installers/master.sh
@@ -15,6 +15,21 @@
 # limitations under the License.
 
 # Install runsc from the master branch.
+set -e
+
 curl -fsSL https://gvisor.dev/archive.key | sudo apt-key add -
 add-apt-repository "deb https://storage.googleapis.com/gvisor/releases release main"
-apt-get update && apt-get install -y runsc
+while true; do
+  if apt-get update; then
+    apt-get install -y runsc
+    break
+  fi
+  result=$?
+  # Check if apt update failed to aquire the file lock.
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
+runsc install
+service docker restart
+
-- 
cgit v1.2.3