diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/bazel.mk | 10 | ||||
| -rwxr-xr-x | tools/go_branch.sh | 11 | ||||
| -rw-r--r-- | tools/go_generics/generics.go | 4 | ||||
| -rw-r--r-- | tools/issue_reviver/main.go | 21 | ||||
| -rwxr-xr-x | tools/make_apt.sh (renamed from tools/make_repository.sh) | 80 | ||||
| -rwxr-xr-x | tools/make_release.sh | 82 | ||||
| -rwxr-xr-x | tools/tag_release.sh | 19 | ||||
| -rwxr-xr-x | tools/vm/build.sh | 8 | ||||
| -rw-r--r-- | tools/vm/defs.bzl | 2 | ||||
| -rwxr-xr-x | tools/vm/ubuntu1604/10_core.sh | 2 | ||||
| -rwxr-xr-x | tools/vm/ubuntu1604/15_gcloud.sh | 50 | ||||
| -rwxr-xr-x | tools/vm/ubuntu1604/40_kokoro.sh | 16 |
12 files changed, 235 insertions, 70 deletions
diff --git a/tools/bazel.mk b/tools/bazel.mk index 4d9bbf0ee..b774c8fab 100644 --- a/tools/bazel.mk +++ b/tools/bazel.mk @@ -21,7 +21,8 @@ BRANCH_NAME := $(shell (git branch --show-current 2>/dev/null || \ # Bazel container configuration (see below). USER ?= gvisor -DOCKER_NAME ?= gvisor-bazel-$(shell readlink -m $(CURDIR) | md5sum | cut -c1-8) +HASH ?= $(shell readlink -m $(CURDIR) | md5sum | cut -c1-8) +DOCKER_NAME ?= gvisor-bazel-$(HASH) DOCKER_PRIVILEGED ?= --privileged BAZEL_CACHE := $(shell readlink -m ~/.cache/bazel/) GCLOUD_CONFIG := $(shell readlink -m ~/.config/gcloud/) @@ -40,6 +41,7 @@ FULL_DOCKER_RUN_OPTIONS += -v "$(DOCKER_SOCKET):$(DOCKER_SOCKET)" DOCKER_GROUP := $(shell stat -c '%g' $(DOCKER_SOCKET)) ifneq ($(GID),$(DOCKER_GROUP)) USERADD_OPTIONS += --groups $(DOCKER_GROUP) +GROUPADD_DOCKER += groupadd --gid $(DOCKER_GROUP) --non-unique docker-$(HASH) && FULL_DOCKER_RUN_OPTIONS += --group-add $(DOCKER_GROUP) endif endif @@ -59,6 +61,8 @@ SHELL=/bin/bash -o pipefail ## DOCKER_SOCKET - The Docker socket (default: detected). ## bazel-server-start: load-default ## Starts the bazel server. + @mkdir -p $(BAZEL_CACHE) + @mkdir -p $(GCLOUD_CONFIG) docker run -d --rm \ --init \ --name $(DOCKER_NAME) \ @@ -69,10 +73,12 @@ bazel-server-start: load-default ## Starts the bazel server. $(FULL_DOCKER_RUN_OPTIONS) \ gvisor.dev/images/default \ sh -c "groupadd --gid $(GID) --non-unique $(USER) && \ + $(GROUPADD_DOCKER) \ useradd --uid $(UID) --non-unique --no-create-home --gid $(GID) $(USERADD_OPTIONS) -d $(HOME) $(USER) && \ bazel version && \ exec tail --pid=\$$(bazel info server_pid) -f /dev/null" - @while :; do if docker logs $(DOCKER_NAME) 2>/dev/null | grep "Build label:" >/dev/null; then break; fi; sleep 1; done + @while :; do if docker logs $(DOCKER_NAME) 2>/dev/null | grep "Build label:" >/dev/null; then break; fi; \ + if ! docker ps | grep $(DOCKER_NAME); then exit 1; else sleep 1; fi; done .PHONY: bazel-server-start bazel-shutdown: ## Shuts down a running bazel server. diff --git a/tools/go_branch.sh b/tools/go_branch.sh index f97a74aaf..093de89b4 100755 --- a/tools/go_branch.sh +++ b/tools/go_branch.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eo pipefail +set -xeo pipefail # Discovery the package name from the go.mod file. declare -r module=$(cat go.mod | grep -E "^module" | cut -d' ' -f2) @@ -42,7 +42,8 @@ declare -r head=$(git describe --always) # We expect to have an existing go branch that we will use as the basis for # this commit. That branch may be empty, but it must exist. -declare -r go_branch=$(git show-ref --hash origin/go) +git fetch --all +declare -r go_branch=$(git show-ref --hash go) # Clone the current repository to the temporary directory, and check out the # current go_branch directory. We move to the new repository for convenience. @@ -87,6 +88,12 @@ EOF # because they may correspond to unused templates, etc. cp "${repo_orig}"/runsc/*.go runsc/ +# Normalize all permissions. The way bazel constructs the :gopath tree may leave +# some strange permissions on files. We don't have anything in this tree that +# should be execution, only the Go source files, README.md, and ${othersrc}. +find . -type f -exec chmod 0644 {} \; +find . -type d -exec chmod 0755 {} \; + # Update the current working set and commit. git add . && git commit -m "Merge ${head} (automated)" diff --git a/tools/go_generics/generics.go b/tools/go_generics/generics.go index e9cc2c753..0860ca9db 100644 --- a/tools/go_generics/generics.go +++ b/tools/go_generics/generics.go @@ -223,7 +223,9 @@ func main() { } else { switch kind { case globals.KindType, globals.KindVar, globals.KindConst, globals.KindFunction: - ident.Name = *prefix + ident.Name + *suffix + if ident.Name != "_" { + ident.Name = *prefix + ident.Name + *suffix + } case globals.KindTag: // Modify the state tag appropriately. if m := stateTagRegexp.FindStringSubmatch(ident.Name); m != nil { diff --git a/tools/issue_reviver/main.go b/tools/issue_reviver/main.go index 4256f5a6c..47c796b8a 100644 --- a/tools/issue_reviver/main.go +++ b/tools/issue_reviver/main.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "os" + "strings" "gvisor.dev/gvisor/tools/issue_reviver/github" "gvisor.dev/gvisor/tools/issue_reviver/reviver" @@ -35,14 +36,22 @@ var ( // Keep the options simple for now. Supports only a single path and repo. func init() { - flag.StringVar(&owner, "owner", "google", "Github project org/owner to look for issues") - flag.StringVar(&repo, "repo", "gvisor", "Github repo to look for issues") + flag.StringVar(&owner, "owner", "", "Github project org/owner to look for issues") + flag.StringVar(&repo, "repo", "", "Github repo to look for issues") flag.StringVar(&tokenFile, "oauth-token-file", "", "Path to file containing the OAUTH token to be used as credential to github") - flag.StringVar(&path, "path", "", "Path to scan for TODOs") + flag.StringVar(&path, "path", ".", "Path to scan for TODOs") flag.BoolVar(&dryRun, "dry-run", false, "If set to true, no changes are made to issues") } func main() { + // Set defaults from the environment. + repository := os.Getenv("GITHUB_REPOSITORY") + if parts := strings.SplitN(repository, "/", 2); len(parts) == 2 { + owner = parts[0] + repo = parts[1] + } + + // Parse flags. flag.Parse() // Check for mandatory parameters. @@ -62,8 +71,10 @@ func main() { os.Exit(1) } - // Token is passed as a file so it doesn't show up in command line arguments. - var token string + // The access token may be passed as a file so it doesn't show up in + // command line arguments. It also may be provided through the + // environment to faciliate use through GitHub's CI system. + token := os.Getenv("GITHUB_TOKEN") if len(tokenFile) != 0 { bytes, err := ioutil.ReadFile(tokenFile) if err != nil { diff --git a/tools/make_repository.sh b/tools/make_apt.sh index 32d7b3b1f..3fb1066e5 100755 --- a/tools/make_repository.sh +++ b/tools/make_apt.sh @@ -14,22 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# We need to be sure that only a repo path is printed on stdout. -exec 50<&1 -exec 1<&2 - -echo_stdout() { - echo "$@" >&50 -} - -# Parse arguments. We require more than two arguments, which are the private -# keyring, the e-mail associated with the signer, and the list of packages. -if [ "$#" -le 3 ]; then - echo "usage: $0 <private-key> <signer-email> <root> <packages...>" +if [[ "$#" -le 3 ]]; then + echo "usage: $0 <private-key> <suite> <root> <packages...>" exit 1 fi declare -r private_key=$(readlink -e "$1"); shift -declare -r signer="$1"; shift +declare -r suite="$1"; shift declare -r root="$1"; shift # Ensure that we have the correct packages installed. @@ -52,16 +42,16 @@ function apt_install() { esac done } -dpkg-sig --help >/dev/null || apt_install dpkg-sig -apt-ftparchive --help >/dev/null || apt_install apt-utils -xz --help >/dev/null || apt_install xz-utils +dpkg-sig --help >/dev/null 2>&1 || apt_install dpkg-sig +apt-ftparchive --help >/dev/null 2>&1 || apt_install apt-utils +xz --help >/dev/null 2>&1 || apt_install xz-utils # Verbose from this point. set -xeo pipefail -# Create a temporary working directory. We don't remove this, as we ultimately -# print this result and allow the caller to copy wherever they would like. -declare -r tmpdir=$(mktemp -d /tmp/repoXXXXXX) +# Create a directory for the release. +declare -r release="${root}/dists/${suite}" +mkdir -p "${release}" # Create a temporary keyring, and ensure it is cleaned up. declare -r keyring=$(mktemp /tmp/keyringXXXXXX.gpg) @@ -69,12 +59,18 @@ cleanup() { rm -f "${keyring}" } trap cleanup EXIT -gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" + +# We attempt the import twice because the first one will fail if the public key +# is not found. This isn't actually a failure for us, because we don't require +# the public (this may be stored separately). The second import will succeed +# because, in reality, the first import succeeded and it's a no-op. +gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" || \ + gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" # Copy the packages into the root. for pkg in "$@"; do - name=$(basename "${pkg}" .deb) - name=$(basename "${name}" .changes) + ext=${pkg##*.} + name=$(basename "${pkg}" ".${ext}") arch=${name##*_} if [[ "${name}" == "${arch}" ]]; then continue # Not a regular package. @@ -90,17 +86,22 @@ for pkg in "$@"; do echo "Unknown file type: ${pkg}" exit 1 fi - version=${version// /} # Trim whitespace. - mkdir -p "${root}"/pool/"${version}"/binary-"${arch}" - cp -a "${pkg}" "${root}"/pool/"${version}"/binary-"${arch}" -done -# Ensure all permissions are correct. -find "${root}"/pool -type f -exec chmod 0644 {} \; + # The package may already exist, in which case we leave it alone. + version=${version// /} # Trim whitespace. + destdir="${root}/pool/${version}/binary-${arch}" + target="${destdir}/${name}.${ext}" + if [[ -f "${target}" ]]; then + continue + fi -# Sign all packages. -for file in "${root}"/pool/*/binary-*/*.deb; do - dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" + # Copy & sign the package. + mkdir -p "${destdir}" + cp -a "${pkg}" "${target}" + chmod 0644 "${target}" + if [[ "${ext}" == "deb" ]]; then + dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${target}" + fi done # Build the package list. @@ -109,7 +110,7 @@ for dir in "${root}"/pool/*/binary-*; do name=$(basename "${dir}") arch=${name##binary-} arches+=("${arch}") - repo_packages="${tmpdir}"/main/"${name}" + repo_packages="${release}"/main/"${name}" mkdir -p "${repo_packages}" (cd "${root}" && apt-ftparchive --arch "${arch}" packages pool > "${repo_packages}"/Packages) (cd "${repo_packages}" && cat Packages | gzip > Packages.gz) @@ -117,23 +118,22 @@ for dir in "${root}"/pool/*/binary-*; do done # Build the release list. -cat > "${tmpdir}"/apt.conf <<EOF +cat > "${release}"/apt.conf <<EOF APT { FTPArchive { Release { Architectures "${arches[@]}"; + Suite "${suite}"; Components "main"; }; }; }; EOF -(cd "${tmpdir}" && apt-ftparchive -c=apt.conf release . > Release) -rm "${tmpdir}"/apt.conf +(cd "${release}" && apt-ftparchive -c=apt.conf release . > Release) +rm "${release}"/apt.conf # Sign the release. declare -r digest_opts=("--digest-algo" "SHA512" "--cert-digest-algo" "SHA512") -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign "${digest_opts[@]}" -o InRelease Release) -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs "${digest_opts[@]}" -o Release.gpg Release) - -# Show the results. -echo_stdout "${tmpdir}" +(cd "${release}" && rm -f Release.gpg InRelease) +(cd "${release}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign "${digest_opts[@]}" -o InRelease Release) +(cd "${release}" && gpg --no-default-keyring --keyring "${keyring}" -abs "${digest_opts[@]}" -o Release.gpg Release) diff --git a/tools/make_release.sh b/tools/make_release.sh new file mode 100755 index 000000000..b1cdd47b0 --- /dev/null +++ b/tools/make_release.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "$#" -le 2 ]]; then + echo "usage: $0 <private-key> <root> <binaries & packages...>" + echo "The environment variable NIGHTLY may be set to control" + echo "whether the nightly packages are produced or not." + exit 1 +fi + +set -xeo pipefail +declare -r private_key="$1"; shift +declare -r root="$1"; shift +declare -a binaries +declare -a pkgs + +# Collect binaries & packages. +for arg in "$@"; do + if [[ "${arg}" == *.deb ]] || [[ "${arg}" == *.changes ]]; then + pkgs+=("${arg}") + else + binaries+=("${arg}") + fi +done + +# install_raw installs raw artifacts. +install_raw() { + mkdir -p "${root}/$1" + for binary in "${binaries[@]}"; do + # Copy the raw file & generate a sha512sum. + name=$(basename "${binary}") + cp -f "${binary}" "${root}/$1" + sha512sum "${root}/$1/${name}" | \ + awk "{print $$1 \" ${name}\"}" > "${root}/$1/${name}.sha512" + done +} + +# install_apt installs an apt repository. +install_apt() { + tools/make_apt.sh "${private_key}" "$1" "${root}" "${pkgs[@]}" +} + +# If nightly, install only nightly artifacts. +if [[ "${NIGHTLY:-false}" == "true" ]]; then + # The "latest" directory and current date. + stamp="$(date -Idate)" + install_raw "nightly/latest" + install_raw "nightly/${stamp}" + install_apt "nightly" +else + # Is it a tagged release? Build that. + tags="$(git tag --points-at HEAD 2>/dev/null || true)" + if ! [[ -z "${tags}" ]]; then + # Note that a given commit can match any number of tags. We have to iterate + # through all possible tags and produce associated artifacts. + for tag in ${tags}; do + name=$(echo "${tag}" | cut -d'-' -f2) + base=$(echo "${name}" | cut -d'.' -f1) + install_raw "release/${name}" + install_raw "release/latest" + install_apt "release" + install_apt "${base}" + done + else + # Otherwise, assume it is a raw master commit. + install_raw "master/latest" + install_apt "master" + fi +fi diff --git a/tools/tag_release.sh b/tools/tag_release.sh index 4dbfe420a..b0bab74b4 100755 --- a/tools/tag_release.sh +++ b/tools/tag_release.sh @@ -18,10 +18,10 @@ # validate a provided release name, create a tag and push it. It must be # run manually when a release is created. -set -xeu +set -xeuo pipefail # Check arguments. -if [ "$#" -ne 3 ]; then +if [[ "$#" -ne 3 ]]; then echo "usage: $0 <commit|revid> <release.rc> <message-file>" exit 1 fi @@ -30,6 +30,12 @@ declare -r target_commit="$1" declare -r release="$2" declare -r message_file="$3" +if [[ -z "${target_commit}" ]]; then + echo "error: <commit|revid> is empty." +fi +if [[ -z "${release}" ]]; then + echo "error: <release.rc> is empty." +fi if ! [[ -r "${message_file}" ]]; then echo "error: message file '${message_file}' is not readable." exit 1 @@ -68,8 +74,9 @@ if ! [[ "${release}" =~ ^20[0-9]{6}\.[0-9]+$ ]]; then exit 1 fi -# Tag the given commit (annotated, to record the committer). +# Tag the given commit (annotated, to record the committer). Note that the tag +# here is applied as a force, in case the tag already exists and is the same. +# The push will fail in this case (because it is not forced). declare -r tag="release-${release}" -(git tag -F "${message_file}" -a "${tag}" "${commit}" && \ - git push origin tag "${tag}") || \ - (git tag -d "${tag}" && false) +git tag -f -F "${message_file}" -a "${tag}" "${commit}" && \ + git push origin tag "${tag}" diff --git a/tools/vm/build.sh b/tools/vm/build.sh index 5d3dc0bbf..752b2b77b 100755 --- a/tools/vm/build.sh +++ b/tools/vm/build.sh @@ -64,14 +64,14 @@ function cleanup { trap cleanup EXIT # Wait for the instance to become available (up to 5 minutes). -echo -n "Waiting for ${INSTANCE_NAME}" +echo -n "Waiting for ${INSTANCE_NAME}" >&2 declare timeout=300 declare success=0 declare internal="" declare -r start=$(date +%s) declare -r end=$((${start}+${timeout})) while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do - echo -n "." + echo -n "." >&2 if gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then success=$((${success}+1)) elif gcloud compute ssh --internal-ip --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then @@ -81,10 +81,10 @@ while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do done if [[ "${success}" -eq "0" ]]; then - echo "connect timed out after ${timeout} seconds." + echo "connect timed out after ${timeout} seconds." >&2 exit 1 else - echo "done." + echo "done." >&2 fi # Run the install scripts provided. diff --git a/tools/vm/defs.bzl b/tools/vm/defs.bzl index 61feefcbc..0f67cfa92 100644 --- a/tools/vm/defs.bzl +++ b/tools/vm/defs.bzl @@ -60,7 +60,7 @@ def _vm_image_impl(ctx): # Run the builder to generate our output. echo = ctx.actions.declare_file(ctx.label.name) resolved_inputs, argv, runfiles_manifests = ctx.resolve_command( - command = "echo -ne \"#!/bin/bash\\necho $(%s)\\n\" > %s && chmod 0755 %s" % ( + command = "echo -ne \"#!/bin/bash\\nset -e\\nimage=$(%s)\\necho ${image}\\n\" > %s && chmod 0755 %s" % ( ctx.files.builder[0].path, echo.path, echo.path, diff --git a/tools/vm/ubuntu1604/10_core.sh b/tools/vm/ubuntu1604/10_core.sh index cd518d6ac..629f7cf7a 100755 --- a/tools/vm/ubuntu1604/10_core.sh +++ b/tools/vm/ubuntu1604/10_core.sh @@ -40,4 +40,4 @@ if ! [[ -d /usr/local/go ]]; then fi # Link the Go binary from /usr/bin; replacing anything there. -(cd /usr/bin && rm -f go && sudo ln -fs /usr/local/go/bin/go go) +(cd /usr/bin && rm -f go && ln -fs /usr/local/go/bin/go go) diff --git a/tools/vm/ubuntu1604/15_gcloud.sh b/tools/vm/ubuntu1604/15_gcloud.sh new file mode 100755 index 000000000..bc2e5eccc --- /dev/null +++ b/tools/vm/ubuntu1604/15_gcloud.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeo pipefail + +# Install all essential build tools. +while true; do + if (apt-get update && apt-get install -y \ + apt-transport-https \ + ca-certificates \ + gnupg); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done + +# Add gcloud repositories. +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \ + tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + +# Add the appropriate key. +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ + apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + +# Install the gcloud SDK. +while true; do + if (apt-get update && apt-get install -y google-cloud-sdk); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done diff --git a/tools/vm/ubuntu1604/40_kokoro.sh b/tools/vm/ubuntu1604/40_kokoro.sh index 06a1e6c48..2974f156c 100755 --- a/tools/vm/ubuntu1604/40_kokoro.sh +++ b/tools/vm/ubuntu1604/40_kokoro.sh @@ -43,14 +43,14 @@ done # junitparser is used to merge junit xml files. pip install junitparser -# We need a kbuilder user. -if useradd -c "kbuilder user" -m -s /bin/bash kbuilder; then - # User was added successfully; we add the relevant SSH keys here. - mkdir -p ~kbuilder/.ssh - (IFS=$'\n'; echo "${ssh_public_keys[*]}") > ~kbuilder/.ssh/authorized_keys - chmod 0600 ~kbuilder/.ssh/authorized_keys - chown -R kbuilder ~kbuilder/.ssh -fi +# We need a kbuilder user, which may already exist. +useradd -c "kbuilder user" -m -s /bin/bash kbuilder || true + +# We need to provision appropriate keys. +mkdir -p ~kbuilder/.ssh +(IFS=$'\n'; echo "${ssh_public_keys[*]}") > ~kbuilder/.ssh/authorized_keys +chmod 0600 ~kbuilder/.ssh/authorized_keys +chown -R kbuilder ~kbuilder/.ssh # Give passwordless sudo access. cat > /etc/sudoers.d/kokoro <<EOF |