From 04a1f431e392f62639ef13245ac8e7006c0e5af4 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Fri, 29 May 2020 11:23:50 -0700 Subject: Fix the APT repository structure. This change fixes the apt repository structure to avoid emiting warnings on Ubuntu 18.04 (and potentially other versions). This requires a slight refactor of the repository generation scripts, since we can no longer copy the same release files for different "suites". This should avoid the warning by setting the suite to the distribution: https://github.com/Debian/apt/blob/master/apt-pkg/metaindex.cc#L75 This change also moves over to the standardized Makefile entrypoint, which makes settings clearer and enables local testing. PiperOrigin-RevId: 313817017 --- Makefile | 27 +++++++++ scripts/build.sh | 84 ---------------------------- tools/make_apt.sh | 139 +++++++++++++++++++++++++++++++++++++++++++++++ tools/make_release.sh | 82 ++++++++++++++++++++++++++++ tools/make_repository.sh | 139 ----------------------------------------------- 5 files changed, 248 insertions(+), 223 deletions(-) delete mode 100755 scripts/build.sh create mode 100755 tools/make_apt.sh create mode 100755 tools/make_release.sh delete mode 100755 tools/make_repository.sh diff --git a/Makefile b/Makefile index 2bcb85e9b..80cb6851b 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,33 @@ website-deploy: website-push ## Deploy a new version of the website. @gcloud run deploy $(WEBSITE_SERVICE) --platform=managed --region=$(WEBSITE_REGION) --project=$(WEBSITE_PROJECT) --image=$(WEBSITE_IMAGE) .PHONY: website-push +## +## Repository builders. +## +## This builds a local apt repository. The following variables may be set: +## RELEASE_ROOT - The repository root (default: "repo" directory). +## RELEASE_KEY - The repository GPG private key file (default: dummy key is created). +## RELEASE_NIGHTLY - Set to true if a nightly release (default: false). +## +RELEASE_ROOT := $(CURDIR)/repo +RELEASE_KEY := repo.key +RELEASE_NIGHTLY := false + +$(RELEASE_KEY): + @echo "WARNING: Generating a key for testing ($@); don't use this." + T=$$(mktemp /tmp/keyring.XXXXXX); \ + gpg --no-default-keyring --keyring $$T --batch --passphrase "" --quick-generate-key $(shell whoami) && \ + gpg --export-secret-keys --no-default-keyring --keyring $$T > $@; \ + rc=$$?; rm -f $$T; exit $$rc + +release: $(RELEASE_KEY) ## Builds a release. + @mkdir -p $(RELEASE_ROOT) + @T=$$(mktemp -d /tmp/release.XXXXXX); \ + $(MAKE) copy TARGETS="runsc runsc:runsc-debian" DESTINATION=$$T && \ + NIGHTLY=$(RELEASE_NIGHTLY) tools/make_release.sh $(RELEASE_KEY) $(RELEASE_ROOT) $$T/*; \ + rc=$$?; rm -rf $$T; exit $$rc +.PHONY: release + ## ## Development helpers and tooling. ## diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100755 index e821e8624..000000000 --- a/scripts/build.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/common.sh - -# Build runsc. -runsc=$(build -c opt //runsc) - -# Build packages. -pkgs=$(build -c opt //runsc:runsc-debian) - -# Stop here if we have no artifacts directory. -[[ -v KOKORO_ARTIFACTS_DIR ]] || exit 0 - -# install_raw installs raw artifacts. -install_raw() { - mkdir -p "$1" - cp -f "${runsc}" "$1"/runsc - sha512sum "$1"/runsc | awk '{print $1 " runsc"}' > "$1"/runsc.sha512 -} - -# Build a repository, if the key is available. -# -# Note that make_repository.sh script will install packages into the provided -# root, but will output to stdout a directory that can be copied arbitrarily -# into "${KOKORO_ARTIFACTS_DIR}"/dists/XXX. We do things this way because we -# will copy the same repository structure into multiple locations, below. -if [[ -v KOKORO_REPO_KEY ]]; then - repo=$(tools/make_repository.sh \ - "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" \ - gvisor-bot@google.com \ - "${KOKORO_ARTIFACTS_DIR}" \ - ${pkgs}) -fi - -# install_repo installs a repository. -# -# Note that packages are already installed, as noted above. -install_repo() { - if [[ -v repo ]]; then - rm -rf "$1" && mkdir -p "$(dirname "$1")" && cp -a "${repo}" "$1" - fi -} - -# If nightly, install only nightly artifacts. -if [[ "${KOKORO_BUILD_NIGHTLY:-false}" == "true" ]]; then - # The "latest" directory and current date. - stamp="$(date -Idate)" - install_raw "${KOKORO_ARTIFACTS_DIR}/nightly/latest" - install_raw "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}" - install_repo "${KOKORO_ARTIFACTS_DIR}/dists/nightly" -else - # Is it a tagged release? Build that. - tags="$(git tag --points-at HEAD)" - if ! [[ -z "${tags}" ]]; then - # Note that a given commit can match any number of tags. We have to iterate - # through all possible tags and produce associated artifacts. - for tag in ${tags}; do - name=$(echo "${tag}" | cut -d'-' -f2) - base=$(echo "${name}" | cut -d'.' -f1) - install_raw "${KOKORO_ARTIFACTS_DIR}/release/${name}" - install_raw "${KOKORO_ARTIFACTS_DIR}/release/latest" - install_repo "${KOKORO_ARTIFACTS_DIR}/dists/release" - install_repo "${KOKORO_ARTIFACTS_DIR}/dists/${base}" - done - else - # Otherwise, assume it is a raw master commit. - install_raw "${KOKORO_ARTIFACTS_DIR}/master/latest" - install_repo "${KOKORO_ARTIFACTS_DIR}/dists/master" - fi -fi diff --git a/tools/make_apt.sh b/tools/make_apt.sh new file mode 100755 index 000000000..3fb1066e5 --- /dev/null +++ b/tools/make_apt.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "$#" -le 3 ]]; then + echo "usage: $0 " + exit 1 +fi +declare -r private_key=$(readlink -e "$1"); shift +declare -r suite="$1"; shift +declare -r root="$1"; shift + +# Ensure that we have the correct packages installed. +function apt_install() { + while true; do + sudo apt-get update && + sudo apt-get install -y "$@" && + true + result="${?}" + case $result in + 0) + break + ;; + 100) + # 100 is the error code that apt-get returns. + ;; + *) + exit $result + ;; + esac + done +} +dpkg-sig --help >/dev/null 2>&1 || apt_install dpkg-sig +apt-ftparchive --help >/dev/null 2>&1 || apt_install apt-utils +xz --help >/dev/null 2>&1 || apt_install xz-utils + +# Verbose from this point. +set -xeo pipefail + +# Create a directory for the release. +declare -r release="${root}/dists/${suite}" +mkdir -p "${release}" + +# Create a temporary keyring, and ensure it is cleaned up. +declare -r keyring=$(mktemp /tmp/keyringXXXXXX.gpg) +cleanup() { + rm -f "${keyring}" +} +trap cleanup EXIT + +# We attempt the import twice because the first one will fail if the public key +# is not found. This isn't actually a failure for us, because we don't require +# the public (this may be stored separately). The second import will succeed +# because, in reality, the first import succeeded and it's a no-op. +gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" || \ + gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" + +# Copy the packages into the root. +for pkg in "$@"; do + ext=${pkg##*.} + name=$(basename "${pkg}" ".${ext}") + arch=${name##*_} + if [[ "${name}" == "${arch}" ]]; then + continue # Not a regular package. + fi + if [[ "${pkg}" =~ ^.*\.deb$ ]]; then + # Extract from the debian file. + version=$(dpkg --info "${pkg}" | grep -E 'Version:' | cut -d':' -f2) + elif [[ "${pkg}" =~ ^.*\.changes$ ]]; then + # Extract from the changes file. + version=$(grep -E 'Version:' "${pkg}" | cut -d':' -f2) + else + # Unsupported file type. + echo "Unknown file type: ${pkg}" + exit 1 + fi + + # The package may already exist, in which case we leave it alone. + version=${version// /} # Trim whitespace. + destdir="${root}/pool/${version}/binary-${arch}" + target="${destdir}/${name}.${ext}" + if [[ -f "${target}" ]]; then + continue + fi + + # Copy & sign the package. + mkdir -p "${destdir}" + cp -a "${pkg}" "${target}" + chmod 0644 "${target}" + if [[ "${ext}" == "deb" ]]; then + dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${target}" + fi +done + +# Build the package list. +declare arches=() +for dir in "${root}"/pool/*/binary-*; do + name=$(basename "${dir}") + arch=${name##binary-} + arches+=("${arch}") + repo_packages="${release}"/main/"${name}" + mkdir -p "${repo_packages}" + (cd "${root}" && apt-ftparchive --arch "${arch}" packages pool > "${repo_packages}"/Packages) + (cd "${repo_packages}" && cat Packages | gzip > Packages.gz) + (cd "${repo_packages}" && cat Packages | xz > Packages.xz) +done + +# Build the release list. +cat > "${release}"/apt.conf < Release) +rm "${release}"/apt.conf + +# Sign the release. +declare -r digest_opts=("--digest-algo" "SHA512" "--cert-digest-algo" "SHA512") +(cd "${release}" && rm -f Release.gpg InRelease) +(cd "${release}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign "${digest_opts[@]}" -o InRelease Release) +(cd "${release}" && gpg --no-default-keyring --keyring "${keyring}" -abs "${digest_opts[@]}" -o Release.gpg Release) diff --git a/tools/make_release.sh b/tools/make_release.sh new file mode 100755 index 000000000..b1cdd47b0 --- /dev/null +++ b/tools/make_release.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "$#" -le 2 ]]; then + echo "usage: $0 " + echo "The environment variable NIGHTLY may be set to control" + echo "whether the nightly packages are produced or not." + exit 1 +fi + +set -xeo pipefail +declare -r private_key="$1"; shift +declare -r root="$1"; shift +declare -a binaries +declare -a pkgs + +# Collect binaries & packages. +for arg in "$@"; do + if [[ "${arg}" == *.deb ]] || [[ "${arg}" == *.changes ]]; then + pkgs+=("${arg}") + else + binaries+=("${arg}") + fi +done + +# install_raw installs raw artifacts. +install_raw() { + mkdir -p "${root}/$1" + for binary in "${binaries[@]}"; do + # Copy the raw file & generate a sha512sum. + name=$(basename "${binary}") + cp -f "${binary}" "${root}/$1" + sha512sum "${root}/$1/${name}" | \ + awk "{print $$1 \" ${name}\"}" > "${root}/$1/${name}.sha512" + done +} + +# install_apt installs an apt repository. +install_apt() { + tools/make_apt.sh "${private_key}" "$1" "${root}" "${pkgs[@]}" +} + +# If nightly, install only nightly artifacts. +if [[ "${NIGHTLY:-false}" == "true" ]]; then + # The "latest" directory and current date. + stamp="$(date -Idate)" + install_raw "nightly/latest" + install_raw "nightly/${stamp}" + install_apt "nightly" +else + # Is it a tagged release? Build that. + tags="$(git tag --points-at HEAD 2>/dev/null || true)" + if ! [[ -z "${tags}" ]]; then + # Note that a given commit can match any number of tags. We have to iterate + # through all possible tags and produce associated artifacts. + for tag in ${tags}; do + name=$(echo "${tag}" | cut -d'-' -f2) + base=$(echo "${name}" | cut -d'.' -f1) + install_raw "release/${name}" + install_raw "release/latest" + install_apt "release" + install_apt "${base}" + done + else + # Otherwise, assume it is a raw master commit. + install_raw "master/latest" + install_apt "master" + fi +fi diff --git a/tools/make_repository.sh b/tools/make_repository.sh deleted file mode 100755 index 32d7b3b1f..000000000 --- a/tools/make_repository.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# We need to be sure that only a repo path is printed on stdout. -exec 50<&1 -exec 1<&2 - -echo_stdout() { - echo "$@" >&50 -} - -# Parse arguments. We require more than two arguments, which are the private -# keyring, the e-mail associated with the signer, and the list of packages. -if [ "$#" -le 3 ]; then - echo "usage: $0 " - exit 1 -fi -declare -r private_key=$(readlink -e "$1"); shift -declare -r signer="$1"; shift -declare -r root="$1"; shift - -# Ensure that we have the correct packages installed. -function apt_install() { - while true; do - sudo apt-get update && - sudo apt-get install -y "$@" && - true - result="${?}" - case $result in - 0) - break - ;; - 100) - # 100 is the error code that apt-get returns. - ;; - *) - exit $result - ;; - esac - done -} -dpkg-sig --help >/dev/null || apt_install dpkg-sig -apt-ftparchive --help >/dev/null || apt_install apt-utils -xz --help >/dev/null || apt_install xz-utils - -# Verbose from this point. -set -xeo pipefail - -# Create a temporary working directory. We don't remove this, as we ultimately -# print this result and allow the caller to copy wherever they would like. -declare -r tmpdir=$(mktemp -d /tmp/repoXXXXXX) - -# Create a temporary keyring, and ensure it is cleaned up. -declare -r keyring=$(mktemp /tmp/keyringXXXXXX.gpg) -cleanup() { - rm -f "${keyring}" -} -trap cleanup EXIT -gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" - -# Copy the packages into the root. -for pkg in "$@"; do - name=$(basename "${pkg}" .deb) - name=$(basename "${name}" .changes) - arch=${name##*_} - if [[ "${name}" == "${arch}" ]]; then - continue # Not a regular package. - fi - if [[ "${pkg}" =~ ^.*\.deb$ ]]; then - # Extract from the debian file. - version=$(dpkg --info "${pkg}" | grep -E 'Version:' | cut -d':' -f2) - elif [[ "${pkg}" =~ ^.*\.changes$ ]]; then - # Extract from the changes file. - version=$(grep -E 'Version:' "${pkg}" | cut -d':' -f2) - else - # Unsupported file type. - echo "Unknown file type: ${pkg}" - exit 1 - fi - version=${version// /} # Trim whitespace. - mkdir -p "${root}"/pool/"${version}"/binary-"${arch}" - cp -a "${pkg}" "${root}"/pool/"${version}"/binary-"${arch}" -done - -# Ensure all permissions are correct. -find "${root}"/pool -type f -exec chmod 0644 {} \; - -# Sign all packages. -for file in "${root}"/pool/*/binary-*/*.deb; do - dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" -done - -# Build the package list. -declare arches=() -for dir in "${root}"/pool/*/binary-*; do - name=$(basename "${dir}") - arch=${name##binary-} - arches+=("${arch}") - repo_packages="${tmpdir}"/main/"${name}" - mkdir -p "${repo_packages}" - (cd "${root}" && apt-ftparchive --arch "${arch}" packages pool > "${repo_packages}"/Packages) - (cd "${repo_packages}" && cat Packages | gzip > Packages.gz) - (cd "${repo_packages}" && cat Packages | xz > Packages.xz) -done - -# Build the release list. -cat > "${tmpdir}"/apt.conf < Release) -rm "${tmpdir}"/apt.conf - -# Sign the release. -declare -r digest_opts=("--digest-algo" "SHA512" "--cert-digest-algo" "SHA512") -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign "${digest_opts[@]}" -o InRelease Release) -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs "${digest_opts[@]}" -o Release.gpg Release) - -# Show the results. -echo_stdout "${tmpdir}" -- cgit v1.2.3