Fix apt-get reliability issues.

This is frequently causing the core build scripts to fail. The core ubuntu
distribution will perform an auto-update at first start, which may cause the
lock file to be held. All apt-get commands may be done in a loop in order to
retry to avoid this issue. We may want to consider retrying other pieces, but
for now this should avoid the most frequent cause of build flakes.

PiperOrigin-RevId: 297704789

8 files changed, 96 insertions, 16 deletions

diff --git a/scripts/build.sh b/scripts/build.sh
index 4c042af6c..7c9c99800 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -17,7 +17,7 @@
 source $(dirname $0)/common.sh
 
 # Install required packages for make_repository.sh et al.
-sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils xz-utils
+apt_install dpkg-sig coreutils apt-utils xz-utils
 
 # Build runsc.
 runsc=$(build -c opt //runsc)
diff --git a/scripts/common.sh b/scripts/common.sh
index 3ca699e4a..735a383de 100755
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -84,3 +84,17 @@ function install_runsc() {
   # Restart docker to pick up the new runtime configuration.
   sudo systemctl restart docker
 }
+
+# Installs the given packages. Note that the package names should be verified to
+# be correct, otherwise this may result in a loop that spins until time out.
+function apt_install() {
+  while true; do
+    if (sudo apt-get update && sudo apt-get install -y "$@"); then
+      break
+    fi
+    result=$?
+    if [[ $result -ne 100 ]]; then
+      return $result
+    fi
+  done
+}
diff --git a/tools/images/ubuntu1604/10_core.sh b/tools/images/ubuntu1604/10_core.sh
index 46dda6bb1..cd518d6ac 100755
--- a/tools/images/ubuntu1604/10_core.sh
+++ b/tools/images/ubuntu1604/10_core.sh
@@ -17,7 +17,20 @@
 set -xeo pipefail
 
 # Install all essential build tools.
-apt-get update && apt-get -y install make git-core build-essential linux-headers-$(uname -r) pkg-config
+while true; do
+  if (apt-get update && apt-get install -y \
+      make \
+      git-core \
+      build-essential \
+      linux-headers-$(uname -r) \
+      pkg-config); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Install a recent go toolchain.
 if ! [[ -d /usr/local/go ]]; then
diff --git a/tools/images/ubuntu1604/20_bazel.sh b/tools/images/ubuntu1604/20_bazel.sh
index b33e1656c..bb7afa676 100755
--- a/tools/images/ubuntu1604/20_bazel.sh
+++ b/tools/images/ubuntu1604/20_bazel.sh
@@ -19,7 +19,17 @@ set -xeo pipefail
 declare -r BAZEL_VERSION=2.0.0
 
 # Install bazel dependencies.
-apt-get update && apt-get install -y openjdk-8-jdk-headless unzip
+while true; do
+  if (apt-get update && apt-get install -y \
+      openjdk-8-jdk-headless \
+      unzip); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Use the release installer.
 curl -L -o bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
diff --git a/tools/images/ubuntu1604/25_docker.sh b/tools/images/ubuntu1604/25_docker.sh
index 1d3defcd3..11eea2d72 100755
--- a/tools/images/ubuntu1604/25_docker.sh
+++ b/tools/images/ubuntu1604/25_docker.sh
@@ -15,12 +15,20 @@
 # limitations under the License.
 
 # Add dependencies.
-apt-get update && apt-get -y install \
-    apt-transport-https \
-    ca-certificates \
-    curl \
-    gnupg-agent \
-    software-properties-common
+while true; do
+  if (apt-get update && apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg-agent \
+      software-properties-common); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Install the key.
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
@@ -32,4 +40,15 @@ add-apt-repository \
    stable"
 
 # Install docker.
-apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io
+while true; do
+  if (apt-get update && apt-get install -y \
+      docker-ce \
+      docker-ce-cli \
+      containerd.io); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
diff --git a/tools/images/ubuntu1604/30_containerd.sh b/tools/images/ubuntu1604/30_containerd.sh
index a7472bd1c..fb3699c12 100755
--- a/tools/images/ubuntu1604/30_containerd.sh
+++ b/tools/images/ubuntu1604/30_containerd.sh
@@ -34,7 +34,17 @@ install_helper() {
 }
 
 # Install dependencies for the crictl tests.
-apt-get install -y btrfs-tools libseccomp-dev
+while true; do
+  if (apt-get update && apt-get install -y \
+      btrfs-tools \
+      libseccomp-dev); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Install containerd & cri-tools.
 GOPATH=$(mktemp -d --tmpdir gopathXXXXX)
diff --git a/tools/images/ubuntu1604/40_kokoro.sh b/tools/images/ubuntu1604/40_kokoro.sh
index 5f2dfc858..06a1e6c48 100755
--- a/tools/images/ubuntu1604/40_kokoro.sh
+++ b/tools/images/ubuntu1604/40_kokoro.sh
@@ -23,7 +23,22 @@ declare -r ssh_public_keys=(
 )
 
 # Install dependencies.
-apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip python3-pip zip
+while true; do
+  if (apt-get update && apt-get install -y \
+      rsync \
+      coreutils \
+      python-psutil \
+      qemu-kvm \
+      python-pip \
+      python3-pip \
+      zip); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # junitparser is used to merge junit xml files.
 pip install junitparser
diff --git a/tools/installers/master.sh b/tools/installers/master.sh
index 52f9734a6..2c6001c6c 100755
--- a/tools/installers/master.sh
+++ b/tools/installers/master.sh
@@ -19,17 +19,16 @@ set -e
 
 curl -fsSL https://gvisor.dev/archive.key | sudo apt-key add -
 add-apt-repository "deb https://storage.googleapis.com/gvisor/releases release main"
+
 while true; do
-  if apt-get update; then
-    apt-get install -y runsc
+  if (apt-get update && apt-get install -y runsc); then
     break
   fi
   result=$?
-  # Check if apt update failed to aquire the file lock.
   if [[ $result -ne 100 ]]; then
     exit $result
   fi
 done
+
 runsc install
 service docker restart
-