Fix repository build scripts.

This fixes a number of issues with the repository build process:

 * Fix the overall structure of the repository.
 * Fix the debian package description.
 * Fix the broken version number for packages.
 * Update the digest algorithm used for signing the release.

I've validated that installation works from a separate staging bucket.

Updates #852

PiperOrigin-RevId: 278716914

5 files changed, 111 insertions, 69 deletions

diff --git a/kokoro/build.cfg b/kokoro/build.cfg
index 6c1d262d4..c9ceda947 100644
--- a/kokoro/build.cfg
+++ b/kokoro/build.cfg
@@ -19,5 +19,6 @@ action {
     regex: "**/runsc"
     regex: "**/runsc.*"
     regex: "**/dists/**"
+    regex: "**/pool/**"
   }
 }
diff --git a/runsc/BUILD b/runsc/BUILD
index e4e8e64a3..e5587421d 100644
--- a/runsc/BUILD
+++ b/runsc/BUILD
@@ -76,16 +76,24 @@ pkg_tar(
 
 genrule(
     name = "deb-version",
+    # Note that runsc must appear in the srcs parameter and not the tools
+    # parameter, otherwise it will not be stamped. This is reasonable, as tools
+    # may be encoded differently in the build graph (cached more aggressively
+    # because they are assumes to be hermetic).
+    srcs = [":runsc"],
     outs = ["version.txt"],
     cmd = "$(location :runsc) -version | grep 'runsc version' | sed 's/^[^0-9]*//' > $@",
     stamp = 1,
-    tools = [":runsc"],
 )
 
 pkg_deb(
     name = "runsc-debian",
     architecture = "amd64",
     data = ":debian-data",
+    # Note that the description_file will be flatten (all newlines removed),
+    # and therefore it is kept to a simple one-line description. The expected
+    # format for debian packages is "short summary\nLonger explanation of
+    # tool." and this is impossible with the flattening.
     description_file = "debian/description",
     homepage = "https://gvisor.dev/",
     maintainer = "The gVisor Authors <gvisor-dev@googlegroups.com>",
diff --git a/runsc/debian/description b/runsc/debian/description
index 6e3b1b2c0..9e8e08805 100644
--- a/runsc/debian/description
+++ b/runsc/debian/description
@@ -1,5 +1 @@
-gVisor is a user-space kernel, written in Go, that implements a substantial
-portion of the Linux system surface. It includes an Open Container Initiative
-(OCI) runtime called runsc that provides an isolation boundary between the
-application and the host kernel. The runsc runtime integrates with Docker and
-Kubernetes, making it simple to run sandboxed containers.
+gVisor container sandbox runtime
diff --git a/scripts/build.sh b/scripts/build.sh
index 0b3d1b316..8b2094cb0 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -17,63 +17,71 @@
 source $(dirname $0)/common.sh
 
 # Install required packages for make_repository.sh et al.
-sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils
+sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils xz-utils
 
 # Build runsc.
 runsc=$(build -c opt //runsc)
 
 # Build packages.
-pkg=$(build -c opt //runsc:runsc-debian)
+pkgs=$(build -c opt //runsc:runsc-debian)
+
+# Stop here if we have no artifacts directory.
+[[ -v KOKORO_ARTIFACTS_DIR ]] || exit 0
+
+# install_raw installs raw artifacts.
+install_raw() {
+  mkdir -p "$1"
+  cp -f "${runsc}" "$1"/runsc
+  sha512sum "$1"/runsc | awk '{print $1 "  runsc"}' > "$1"/runsc.sha512
+}
 
 # Build a repository, if the key is available.
+#
+# Note that make_repository.sh script will install packages into the provided
+# root, but will output to stdout a directory that can be copied arbitrarily
+# into "${KOKORO_ARTIFACTS_DIR}"/dists/XXX. We do things this way because we
+# will copy the same repository structure into multiple locations, below.
 if [[ -v KOKORO_REPO_KEY ]]; then
-  repo=$(tools/make_repository.sh "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" gvisor-bot@google.com main ${pkg})
+  repo=$(tools/make_repository.sh \
+          "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" \
+          gvisor-bot@google.com \
+          main \
+          "${KOKORO_ARTIFACTS_DIR}" \
+          ${pkgs})
 fi
 
-# Install installs artifacts.
-install() {
-  local -r binaries_dir="$1"
-  local -r repo_dir="$2"
-  mkdir -p "${binaries_dir}"
-  cp -f "${runsc}" "${binaries_dir}"/runsc
-  sha512sum "${binaries_dir}"/runsc | awk '{print $1 "  runsc"}' > "${binaries_dir}"/runsc.sha512
+# install_repo installs a repository.
+#
+# Note that packages are already installed, as noted above.
+install_repo() {
   if [[ -v repo ]]; then
-    rm -rf "${repo_dir}" && mkdir -p "$(dirname "${repo_dir}")"
-    cp -a "${repo}" "${repo_dir}"
+    rm -rf "$1" && mkdir -p "$(dirname "$1")" && cp -a "${repo}" "$1"
   fi
 }
 
-# Move the runsc binary into "latest" directory, and also a directory with the
-# current date. If the current commit happens to correpond to a tag, then we
-# will also move everything into a directory named after the given tag.
-if [[ -v KOKORO_ARTIFACTS_DIR ]]; then
-  if [[ "${KOKORO_BUILD_NIGHTLY:-false}" == "true" ]]; then
-    # The "latest" directory and current date.
-    stamp="$(date -Idate)"
-    install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" \
-            "${KOKORO_ARTIFACTS_DIR}/dists/nightly/latest"
-    install "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}" \
-            "${KOKORO_ARTIFACTS_DIR}/dists/nightly/${stamp}"
-  else
-    # Is it a tagged release? Build that instead. In that case, we also try to
-    # update the base release directory, in case this is an update. Finally, we
-    # update the "release" directory, which has the last released version.
-    tags="$(git tag --points-at HEAD)"
-    if ! [[ -z "${tags}" ]]; then
-      # Note that a given commit can match any number of tags. We have to
-      # iterate through all possible tags and produce associated artifacts.
-      for tag in ${tags}; do
-        name=$(echo "${tag}" | cut -d'-' -f2)
-        base=$(echo "${name}" | cut -d'.' -f1)
-        install "${KOKORO_ARTIFACTS_DIR}/release/${name}" \
-                "${KOKORO_ARTIFACTS_DIR}/dists/${name}"
-        if [[ "${base}" != "${tag}" ]]; then
-          install "${KOKORO_ARTIFACTS_DIR}/release/${base}" \
-                  "${KOKORO_ARTIFACTS_DIR}/dists/${base}"
-        fi
-        install "${KOKORO_ARTIFACTS_DIR}/release/latest" \
-                "${KOKORO_ARTIFACTS_DIR}/dists/latest"
-      done
-    fi
+# If nightly, install only nightly artifacts.
+if [[ "${KOKORO_BUILD_NIGHTLY:-false}" == "true" ]]; then
+  # The "latest" directory and current date.
+  stamp="$(date -Idate)"
+  install_raw  "${KOKORO_ARTIFACTS_DIR}/nightly/latest"
+  install_raw  "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}"
+  install_repo "${KOKORO_ARTIFACTS_DIR}/dists/nightly"
+else
+  # We keep only the latest master raw release.
+  install_raw  "${KOKORO_ARTIFACTS_DIR}/master/latest"
+  install_repo "${KOKORO_ARTIFACTS_DIR}/dists/master"
+
+  # Is it a tagged release? Build that too.
+  tags="$(git tag --points-at HEAD)"
+  if ! [[ -z "${tags}" ]]; then
+    # Note that a given commit can match any number of tags. We have to iterate
+    # through all possible tags and produce associated artifacts.
+    for tag in ${tags}; do
+      name=$(echo "${tag}" | cut -d'-' -f2)
+      base=$(echo "${name}" | cut -d'.' -f1)
+      install_raw  "${KOKORO_ARTIFACTS_DIR}/release/${name}"
+      install_repo "${KOKORO_ARTIFACTS_DIR}/dists/release"
+      install_repo "${KOKORO_ARTIFACTS_DIR}/dists/${base}"
+    done
   fi
 fi
diff --git a/tools/make_repository.sh b/tools/make_repository.sh
index 071f72b74..27ffbc9f3 100755
--- a/tools/make_repository.sh
+++ b/tools/make_repository.sh
@@ -17,13 +17,13 @@
 # Parse arguments. We require more than two arguments, which are the private
 # keyring, the e-mail associated with the signer, and the list of packages.
 if [ "$#" -le 3 ]; then
-  echo "usage: $0 <private-key> <signer-email> <component> <packages...>"
+  echo "usage: $0 <private-key> <signer-email> <component> <root> <packages...>"
   exit 1
 fi
-declare -r private_key=$(readlink -e "$1")
-declare -r signer="$2"
-declare -r component="$3"
-shift; shift; shift
+declare -r private_key=$(readlink -e "$1"); shift
+declare -r signer="$1"; shift
+declare -r component="$1"; shift
+declare -r root="$1"; shift
 
 # Verbose from this point.
 set -xeo pipefail
@@ -40,7 +40,7 @@ cleanup() {
 trap cleanup EXIT
 gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" >&2
 
-# Copy the packages, and ensure permissions are correct.
+# Copy the packages into the root.
 for pkg in "$@"; do
   name=$(basename "${pkg}" .deb)
   name=$(basename "${name}" .changes)
@@ -48,32 +48,61 @@ for pkg in "$@"; do
   if [[ "${name}" == "${arch}" ]]; then
     continue # Not a regular package.
   fi
-  mkdir -p "${tmpdir}"/"${component}"/binary-"${arch}"
-  cp -a "${pkg}" "${tmpdir}"/"${component}"/binary-"${arch}"
+  if [[ "${pkg}" =~ ^.*\.deb$ ]]; then
+    # Extract from the debian file.
+    version=$(dpkg --info "${pkg}" | grep -E 'Version:' | cut -d':' -f2)
+  elif [[ "${pkg}" =~ ^.*\.changes$ ]]; then
+    # Extract from the changes file.
+    version=$(grep -E 'Version:' "${pkg}" | cut -d':' -f2)
+  else
+    # Unsupported file type.
+    echo "Unknown file type: ${pkg}"
+    exit 1
+  fi
+  version=${version// /} # Trim whitespace.
+  mkdir -p "${root}"/pool/"${version}"/binary-"${arch}"
+  cp -a "${pkg}" "${root}"/pool/"${version}"/binary-"${arch}"
 done
-find "${tmpdir}" -type f -exec chmod 0644 {} \;
 
-# Ensure there are no symlinks hanging around; these may be remnants of the
-# build process. They may be useful for other things, but we are going to build
-# an index of the actual packages here.
-find "${tmpdir}" -type l -exec rm -f {} \;
+# Ensure all permissions are correct.
+find "${root}"/pool -type f -exec chmod 0644 {} \;
 
 # Sign all packages.
-for file in "${tmpdir}"/"${component}"/binary-*/*.deb; do
+for file in "${root}"/pool/*/binary-*/*.deb; do
   dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" >&2
 done
 
 # Build the package list.
-for dir in "${tmpdir}"/"${component}"/binary-*; do
-  (cd "${dir}" && apt-ftparchive packages . | gzip > Packages.gz)
+declare arches=()
+for dir in "${root}"/pool/*/binary-*; do
+  name=$(basename "${dir}")
+  arch=${name##binary-}
+  arches+=("${arch}")
+  repo_packages="${tmpdir}"/"${component}"/"${name}"
+  mkdir -p "${repo_packages}"
+  (cd "${root}" && apt-ftparchive --arch "${arch}" packages pool > "${repo_packages}"/Packages)
+  (cd "${repo_packages}" && cat Packages | gzip > Packages.gz)
+  (cd "${repo_packages}" && cat Packages | xz > Packages.xz)
 done
 
 # Build the release list.
-(cd "${tmpdir}" && apt-ftparchive release . > Release)
+cat > "${tmpdir}"/apt.conf <<EOF
+APT {
+  FTPArchive {
+    Release {
+      Architectures "${arches[@]}";
+      Components "${component}";
+    };
+  };
+};
+EOF
+(cd "${tmpdir}" && apt-ftparchive -c=apt.conf release . > Release)
+rm "${tmpdir}"/apt.conf
 
 # Sign the release.
-(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign -o InRelease Release >&2)
-(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs -o Release.gpg Release >&2)
+declare -r digest_opts=("--digest-algo" "SHA512" "--cert-digest-algo" "SHA512")
+(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign "${digest_opts[@]}" -o InRelease Release >&2)
+(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs "${digest_opts[@]}" -o Release.gpg Release >&2)
 
 # Show the results.
 echo "${tmpdir}"