From 5776a7b6f6b52faf6e0735c3f4a892639c1bd773 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Mon, 27 Jan 2020 18:26:26 -0800 Subject: Fix header ordering and format all C++ code. PiperOrigin-RevId: 291844200 --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'CONTRIBUTING.md') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d46168bc..55a1ad0d9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,8 @@ directory tree. All Go code should conform to the [Go style guidelines][gostyle]. C++ code should conform to the [Google C++ Style Guide][cppstyle] and the guidelines -described for [tests][teststyle]. +described for [tests][teststyle]. Note that code may be automatically formatted +per the guidelines when merged. As a secure runtime, we need to maintain the safety of all of code included in gVisor. The following rules help mitigate issues. -- cgit v1.2.3 From afcab8fe9f6fb3504ebdbb95d35299277c2d67ca Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Mon, 10 Feb 2020 14:11:04 -0800 Subject: Clean-up comments in runsc/BUILD and CONTRIBUTING.md. PiperOrigin-RevId: 294300437 --- CONTRIBUTING.md | 2 +- runsc/BUILD | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'CONTRIBUTING.md') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55a1ad0d9..71650a4b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ Definitions for the rules below: `core`: * `//pkg/sentry/...` -* Transitive dependencies in `//pkg/...`, `//third_party/...`. +* Transitive dependencies in `//pkg/...`, etc. `runsc`: diff --git a/runsc/BUILD b/runsc/BUILD index 02a56657a..757f6d44c 100644 --- a/runsc/BUILD +++ b/runsc/BUILD @@ -26,7 +26,7 @@ go_binary( ) # The runsc-race target is a race-compatible BUILD target. This must be built -# via: bazel build --features=race //runsc:runsc-race +# via: bazel build --features=race :runsc-race # # This is neccessary because the race feature must apply to all dependencies # due a bug in gazelle file selection. The pure attribute must be off because -- cgit v1.2.3 From e5d9a4010bdbea10320348b022ee5b761c1eba07 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Tue, 11 Feb 2020 16:01:42 -0800 Subject: Add ability to execute go.mod in gopath context. --- CONTRIBUTING.md | 3 +++ WORKSPACE | 41 ++++++++++++++++++++++++++++++----------- go.mod | 31 ++++++++++++++----------------- go.sum | 29 +++++++++++++++++++---------- tools/go_mod.sh | 29 +++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 38 deletions(-) create mode 100755 tools/go_mod.sh (limited to 'CONTRIBUTING.md') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 71650a4b8..ad8e710da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,9 @@ will need to be added to the appropriate `BUILD` files, and the `:gopath` target will need to be re-run to generate appropriate symlinks in the `GOPATH` directory tree. +Dependencies can be added by using `go mod get`. In order to keep the +`WORKSPACE` file in sync, run `tools/go_mod.sh` in place of `go mod`. + ### Coding Guidelines All Go code should conform to the [Go style guidelines][gostyle]. C++ code diff --git a/WORKSPACE b/WORKSPACE index a15238a2e..995d2c7f1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -20,7 +20,7 @@ http_archive( ], ) -load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains") +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") go_rules_dependencies() @@ -43,8 +43,8 @@ gazelle_dependencies() go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:72l8qCJ1nGxMGH26QVBVIxKd/D34cfGt0OvrPtpemyY=", - version = "v0.0.0-20191220220014-0732a990476f", + sum = "h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=", + version = "v0.0.0-20190215142949-d0b11bdaac8a", ) # Load C++ rules. @@ -68,8 +68,11 @@ http_archive( "https://github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.tar.gz", ], ) + load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + rules_proto_dependencies() + rules_proto_toolchains() # Load python dependencies. @@ -146,9 +149,9 @@ load( # This container is built from the Dockerfile in test/iptables/runner. container_pull( name = "iptables-test", + digest = "sha256:a137d692a2eb9fc7bf95c5f4a568da090e2c31098e93634421ed88f3a3f1db65", registry = "gcr.io", repository = "gvisor-presubmit/iptables-test", - digest = "sha256:a137d692a2eb9fc7bf95c5f4a568da090e2c31098e93634421ed88f3a3f1db65", ) load( @@ -201,6 +204,13 @@ go_repository( version = "v0.0.0-20171129191014-dec09d789f3d", ) +go_repository( + name = "com_github_kr_pretty", + importpath = "github.com/kr/pretty", + sum = "h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=", + version = "v0.2.0", +) + go_repository( name = "com_github_kr_pty", importpath = "github.com/kr/pty", @@ -208,6 +218,13 @@ go_repository( version = "v1.1.1", ) +go_repository( + name = "com_github_kr_text", + importpath = "github.com/kr/text", + sum = "h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=", + version = "v0.1.0", +) + go_repository( name = "com_github_opencontainers_runtime-spec", importpath = "github.com/opencontainers/runtime-spec", @@ -236,6 +253,13 @@ go_repository( version = "v0.0.0-20171111001504-be1fbeda1936", ) +go_repository( + name = "in_gopkg_check_v1", + importpath = "gopkg.in/check.v1", + sum = "h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=", + version = "v1.0.0-20190902080502-41f04d3bba15", +) + go_repository( name = "org_golang_x_crypto", importpath = "golang.org/x/crypto", @@ -257,12 +281,6 @@ go_repository( version = "v0.3.0", ) -go_repository( - name = "org_golang_x_tools", - commit = "36563e24a262", - importpath = "golang.org/x/tools", -) - go_repository( name = "org_golang_x_sync", importpath = "golang.org/x/sync", @@ -272,8 +290,9 @@ go_repository( go_repository( name = "org_golang_x_time", - commit = "c4c64cad1fd0a1a8dab2523e04e61d35308e131e", importpath = "golang.org/x/time", + sum = "h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=", + version = "v0.0.0-20191024005414-555d28b269f0", ) go_repository( diff --git a/go.mod b/go.mod index c4687ed02..3a8b9288d 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,18 @@ module gvisor.dev/gvisor go 1.13 require ( - github.com/cenkalti/backoff v0.0.0-20190506075156-2146c9339422 - github.com/gofrs/flock v0.6.1-0.20180915234121-886344bea079 - github.com/golang/mock v1.3.1 - github.com/golang/protobuf v1.3.1 - github.com/google/btree v1.0.0 - github.com/google/go-cmp v0.2.0 - github.com/google/go-github/v28 v28.1.1 - github.com/google/subcommands v0.0.0-20190508160503-636abe8753b8 - github.com/google/uuid v0.0.0-20171129191014-dec09d789f3d - github.com/kr/pty v1.1.1 - github.com/opencontainers/runtime-spec v0.1.2-0.20171211145439-b2d941ef6a78 - github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 - github.com/vishvananda/netlink v1.0.1-0.20190318003149-adb577d4a45e - github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 - golang.org/x/net v0.0.0-20190311183353-d8887717615a - golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 - golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a + github.com/cenkalti/backoff v0.0.0-20190506075156-2146c9339422 + github.com/gofrs/flock v0.6.1-0.20180915234121-886344bea079 + github.com/golang/protobuf v1.3.1 + github.com/google/btree v1.0.0 + github.com/google/subcommands v0.0.0-20190508160503-636abe8753b8 + github.com/kr/pretty v0.2.0 // indirect + github.com/kr/pty v1.1.1 + github.com/opencontainers/runtime-spec v0.1.2-0.20171211145439-b2d941ef6a78 + github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 + github.com/vishvananda/netlink v1.0.1-0.20190318003149-adb577d4a45e + github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 // indirect + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index 434770beb..f16a549fd 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,30 @@ +github.com/cenkalti/backoff v0.0.0-20190506075156-2146c9339422 h1:+FKjzBIdfBHYDvxCv+djmDJdes/AoDtg8gpcxowBlF8= github.com/cenkalti/backoff v0.0.0-20190506075156-2146c9339422/go.mod h1:b6Nc7NRH5C4aCISLry0tLnTjcuTEvoiqcWDdsU0sOGM= +github.com/gofrs/flock v0.6.1-0.20180915234121-886344bea079 h1:JFTFz3HZTGmgMz4E1TabNBNJljROSYgja1b4l50FNVs= github.com/gofrs/flock v0.6.1-0.20180915234121-886344bea079/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= +github.com/google/subcommands v0.0.0-20190508160503-636abe8753b8 h1:GZGUPQiZfYrd9uOqyqwbQcHPkz/EZJVkZB1MkaO9UBI= github.com/google/subcommands v0.0.0-20190508160503-636abe8753b8/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v0.0.0-20171129191014-dec09d789f3d/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/opencontainers/runtime-spec v0.1.2-0.20171211145439-b2d941ef6a78 h1:d9F+LNYwMyi3BDN4GzZdaSiq4otb8duVEWyZjeUtOQI= github.com/opencontainers/runtime-spec v0.1.2-0.20171211145439-b2d941ef6a78/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/vishvananda/netlink v1.0.1-0.20190318003149-adb577d4a45e h1:/Tdc23Arz1OtdIsBY2utWepGRQ9fEAJlhkdoLzWMK8Q= github.com/vishvananda/netlink v1.0.1-0.20190318003149-adb577d4a45e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 h1:J9gO8RJCAFlln1jsvRba/CWVUnMHwObklfxxjErl1uk= github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/go_mod.sh b/tools/go_mod.sh new file mode 100755 index 000000000..84b779d6d --- /dev/null +++ b/tools/go_mod.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Build the :gopath target. +bazel build //:gopath +declare -r gopathdir="bazel-bin/gopath/src/gvisor.dev/gvisor/" + +# Copy go.mod and execute the command. +cp -a go.mod go.sum "${gopathdir}" +(cd "${gopathdir}" && go mod "$@") +cp -a "${gopathdir}/go.mod" "${gopathdir}/go.sum" . + +# Cleanup the WORKSPACE file. +bazel run //:gazelle -- update-repos -from_file=go.mod -- cgit v1.2.3 From c60613475c92185c9b15468d0de87b321ef2b4d7 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Fri, 24 Apr 2020 14:10:28 -0700 Subject: Standardize all Docker images. This change moves all Docker images to a standard location, and abstracts the build process so that they can be maintained in an automated fashion. This also allows the images to be architecture-independent. All images will now be referred to by the test framework via the canonical `gvisor.dev/images/`, where `` is a function of the path within the source tree. In a subsequent change, continuous integration will be added so that the images will always be correct and available locally. In the end, using `bazel` for Docker containers is simply not possible. Given that we already have the need to use `make` with the base container (for Docker), we extend this approach to get more flexibility. This change also adds a self-documenting and powerful Makefile that is intended to replace the collection of scripts in scripts. Canonical (self-documenting) targets can be added here for targets that understand which images need to be loaded and/or built. PiperOrigin-RevId: 308322438 --- .travis.yml | 11 +- CONTRIBUTING.md | 31 +---- Dockerfile | 9 -- Makefile | 199 ++++++++++++++++++++++----- images/BUILD | 11 ++ images/Makefile | 93 +++++++++++++ images/README.md | 61 ++++++++ images/basic/alpine/Dockerfile | 1 + images/basic/busybox/Dockerfile | 1 + images/basic/httpd/Dockerfile | 1 + images/basic/mysql/Dockerfile | 1 + images/basic/nginx/Dockerfile | 1 + images/basic/python/Dockerfile | 2 + images/basic/resolv/Dockerfile | 1 + images/basic/ruby/Dockerfile | 1 + images/basic/tomcat/Dockerfile | 1 + images/basic/ubuntu/Dockerfile | 1 + images/default/Dockerfile | 11 ++ images/iptables/Dockerfile | 2 + images/packetdrill/Dockerfile | 8 ++ images/packetimpact/Dockerfile | 16 +++ images/runtimes/go1.12/Dockerfile | 4 + images/runtimes/java11/Dockerfile | 22 +++ images/runtimes/nodejs12.4.0/Dockerfile | 21 +++ images/runtimes/php7.3.6/Dockerfile | 19 +++ images/runtimes/python3.7.3/Dockerfile | 21 +++ pkg/test/testutil/testutil.go | 38 +---- scripts/build.sh | 4 - scripts/common.sh | 22 --- scripts/docker_tests.sh | 2 + scripts/hostnet_tests.sh | 2 + scripts/iptables_tests.sh | 6 +- scripts/kvm_tests.sh | 2 + scripts/make_tests.sh | 5 - scripts/overlay_tests.sh | 2 + scripts/packetdrill_tests.sh | 2 + scripts/packetimpact_tests.sh | 2 + scripts/root_tests.sh | 3 +- scripts/swgso_tests.sh | 2 + test/iptables/runner/Dockerfile | 4 - test/packetdrill/Dockerfile | 9 -- test/packetimpact/tests/Dockerfile | 17 --- test/root/BUILD | 1 + test/runtimes/images/Dockerfile_go1.12 | 10 -- test/runtimes/images/Dockerfile_java11 | 30 ---- test/runtimes/images/Dockerfile_nodejs12.4.0 | 28 ---- test/runtimes/images/Dockerfile_php7.3.6 | 27 ---- test/runtimes/images/Dockerfile_python3.7.3 | 30 ---- tools/bazel.mk | 106 ++++++++++++++ tools/installers/BUILD | 8 ++ tools/installers/images.sh | 24 ++++ tools/make_repository.sh | 31 ++++- tools/vm/defs.bzl | 5 +- 53 files changed, 665 insertions(+), 307 deletions(-) delete mode 100644 Dockerfile create mode 100644 images/BUILD create mode 100644 images/Makefile create mode 100644 images/README.md create mode 100644 images/basic/alpine/Dockerfile create mode 100644 images/basic/busybox/Dockerfile create mode 100644 images/basic/httpd/Dockerfile create mode 100644 images/basic/mysql/Dockerfile create mode 100644 images/basic/nginx/Dockerfile create mode 100644 images/basic/python/Dockerfile create mode 100644 images/basic/resolv/Dockerfile create mode 100644 images/basic/ruby/Dockerfile create mode 100644 images/basic/tomcat/Dockerfile create mode 100644 images/basic/ubuntu/Dockerfile create mode 100644 images/default/Dockerfile create mode 100644 images/iptables/Dockerfile create mode 100644 images/packetdrill/Dockerfile create mode 100644 images/packetimpact/Dockerfile create mode 100644 images/runtimes/go1.12/Dockerfile create mode 100644 images/runtimes/java11/Dockerfile create mode 100644 images/runtimes/nodejs12.4.0/Dockerfile create mode 100644 images/runtimes/php7.3.6/Dockerfile create mode 100644 images/runtimes/python3.7.3/Dockerfile delete mode 100644 test/iptables/runner/Dockerfile delete mode 100644 test/packetdrill/Dockerfile delete mode 100644 test/packetimpact/tests/Dockerfile delete mode 100644 test/runtimes/images/Dockerfile_go1.12 delete mode 100644 test/runtimes/images/Dockerfile_java11 delete mode 100644 test/runtimes/images/Dockerfile_nodejs12.4.0 delete mode 100644 test/runtimes/images/Dockerfile_php7.3.6 delete mode 100644 test/runtimes/images/Dockerfile_python3.7.3 create mode 100644 tools/bazel.mk create mode 100755 tools/installers/images.sh (limited to 'CONTRIBUTING.md') diff --git a/.travis.yml b/.travis.yml index acbd3d61b..40c8773fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,19 @@ -language: minimal -sudo: required +language: shell dist: xenial cache: directories: - /home/travis/.cache/bazel/ +os: linux services: - docker -matrix: +jobs: include: - os: linux arch: amd64 - env: RUNSC_PATH=./bazel-bin/runsc/linux_amd64_pure_stripped/runsc - os: linux arch: arm64 - env: RUNSC_PATH=./bazel-bin/runsc/linux_arm64_pure_stripped/runsc script: - - uname -a - - make DOCKER_RUN_OPTIONS="" BAZEL_OPTIONS="build runsc:runsc" bazel && $RUNSC_PATH --alsologtostderr --network none --debug --TESTONLY-unsafe-nonroot=true --rootless do ls + - uname -a && make smoke-test branches: except: # Skip copybara branches. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad8e710da..423cf7a34 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,32 +108,15 @@ ignored. ### Build and test with Docker -`scripts/dev.sh` is a convenient script that builds and installs `runsc` as a -new Docker runtime for you. The scripts tries to extract the runtime name from -your local environment and will print it at the end. You can also customize it. -The script creates one regular runtime and another with debug flags enabled. -Here are a few examples: +Running `make dev` is a convenient way to build and install `runsc` as a Docker +runtime. The output of this command will show the runtimes installed. + +You may use `make refresh` to refresh the binary after any changes. For example: ```bash -# Default case (inside branch my-branch) -$ scripts/dev.sh -... -Runtimes my-branch and my-branch-d (debug enabled) setup. -Use --runtime=my-branch with your Docker command. - docker run --rm --runtime=my-branch --rm hello-world - -If you rebuild, use scripts/dev.sh --refresh. -Logs are in: /tmp/my-branch/logs - -# --refresh just updates the runtime binary and doesn't restart docker. -$ git/my_branch> scripts/dev.sh --refresh - -# Using a custom runtime name -$ git/my_branch> scripts/dev.sh my-runtime -... -Runtimes my-runtime and my-runtime-d (debug enabled) setup. -Use --runtime=my-runtime with your Docker command. - docker run --rm --runtime=my-runtime --rm hello-world +make dev +docker run --rm --runtime=my-branch --rm hello-world +make refresh ``` ### The small print diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0fac71710..000000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM fedora:31 - -RUN dnf install -y dnf-plugins-core && dnf copr enable -y vbatts/bazel - -RUN dnf install -y bazel2 git gcc make golang gcc-c++ glibc-devel python3 which python3-pip python3-devel libffi-devel openssl-devel pkg-config glibc-static libstdc++-static patch - -RUN pip install pycparser - -WORKDIR /gvisor diff --git a/Makefile b/Makefile index d9531fbd5..c56c6ed48 100644 --- a/Makefile +++ b/Makefile @@ -1,50 +1,173 @@ -UID := $(shell id -u ${USER}) -GID := $(shell id -g ${USER}) -GVISOR_BAZEL_CACHE := $(shell readlink -f ~/.cache/bazel/) +#!/usr/bin/make -f -# The --privileged is required to run tests. -DOCKER_RUN_OPTIONS ?= --privileged +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -all: runsc +# Described below. +OPTIONS := +TARGETS := //runsc +ARGS := -docker-build: - docker build -t gvisor-bazel . +default: runsc +.PHONY: default -bazel-shutdown: - docker exec -i gvisor-bazel bazel shutdown && \ - docker kill gvisor-bazel +## usage: make +## or +## make OPTIONS="..." TARGETS="..." ARGS="..." +## +## Basic targets. +## +## This Makefile wraps basic build and test targets for ease-of-use. Bazel +## is run inside a canonical Docker container in order to simplify up-front +## requirements. +## +## There are common arguments that may be passed to targets. These are: +## OPTIONS - Build or test options. +## TARGETS - The bazel targets. +## ARGS - Arguments for run or sudo. +## +## Additionally, the copy target expects a DESTINATION to be provided. +## +## For example, to build runsc using this Makefile, you can run: +## make build OPTIONS="" TARGETS="//runsc"' +## +help: ## Shows all targets and help from the Makefile (this message). + @grep --no-filename -E '^([a-z.A-Z_-]+:.*?|)##' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = "(:.*?|)## ?"}; { \ + if (length($$1) > 0) { \ + printf " \033[36m%-20s\033[0m %s\n", $$1, $$2; \ + } else { \ + printf "%s\n", $$2; \ + } \ + }' +build: ## Builds the given $(TARGETS) with the given $(OPTIONS). E.g. make build TARGETS=runsc +test: ## Tests the given $(TARGETS) with the given $(OPTIONS). E.g. make test TARGETS=pkg/buffer:buffer_test +copy: ## Copies the given $(TARGETS) to the given $(DESTINATION). E.g. make copy TARGETS=runsc DESTINATION=/tmp +run: ## Runs the given $(TARGETS), built with $(OPTIONS), using $(ARGS). E.g. make run TARGETS=runsc ARGS=-version +sudo: ## Runs the given $(TARGETS) as per run, but using "sudo -E". E.g. make sudo TARGETS=test/root:root_test ARGS=-test.v +.PHONY: help build test copy run sudo -bazel-server-start: docker-build - mkdir -p "$(GVISOR_BAZEL_CACHE)" && \ - docker run -d --rm --name gvisor-bazel \ - --user 0:0 \ - -v "$(GVISOR_BAZEL_CACHE):$(HOME)/.cache/bazel/" \ - -v "$(CURDIR):$(CURDIR)" \ - --workdir "$(CURDIR)" \ - --tmpfs /tmp:rw,exec \ - $(DOCKER_RUN_OPTIONS) \ - gvisor-bazel \ - sh -c "while :; do sleep 100; done" && \ - docker exec --user 0:0 -i gvisor-bazel sh -c "groupadd --gid $(GID) --non-unique gvisor && useradd --uid $(UID) --non-unique --gid $(GID) -d $(HOME) gvisor" +# Load all bazel wrappers. +# +# This file should define the basic "build", "test", "run" and "sudo" rules, in +# addition to the $(BRANCH_NAME) variable. +ifneq (,$(wildcard tools/google.mk)) +include tools/google.mk +else +include tools/bazel.mk +endif -bazel-server: - docker exec gvisor-bazel true || \ - $(MAKE) bazel-server-start +## +## Docker image targets. +## +## Images used by the tests must also be built and available locally. +## The canonical test targets defined below will automatically load +## relevant images. These can be loaded or built manually via these +## targets. +## +## (*) Note that you may provide an ARCH parameter in order to build +## and load images from an alternate archiecture (using qemu). When +## bazel is run as a server, this has the effect of running an full +## cross-architecture chain, and can produce cross-compiled binaries. +## +define images +$(1)-%: ## Image tool: $(1) a given image (also may use 'all-images'). + @$(MAKE) -C images $$@ +endef +rebuild-...: ## Rebuild the given image. Also may use 'rebuild-all-images'. +$(eval $(call images,rebuild)) +push-...: ## Push the given image. Also may use 'push-all-images'. +$(eval $(call images,pull)) +pull-...: ## Pull the given image. Also may use 'pull-all-images'. +$(eval $(call images,push)) +load-...: ## Load (pull or rebuild) the given image. Also may use 'load-all-images'. +$(eval $(call images,load)) +list-images: ## List all available images. + @$(MAKE) -C images $$@ -BAZEL_OPTIONS := build runsc -bazel: bazel-server - docker exec -u $(UID):$(GID) -i gvisor-bazel bazel $(BAZEL_OPTIONS) +## +## Canonical build and test targets. +## +## These targets are used by continuous integration and provide +## convenient entrypoints for testing changes. If you're adding a +## new subsystem or workflow, consider adding a new target here. +## +runsc: ## Builds the runsc binary. + @$(MAKE) build TARGETS="//runsc" +.PHONY: runsc -bazel-alias: - @echo "alias bazel='docker exec -u $(UID):$(GID) -i gvisor-bazel bazel'" +smoke-test: ## Runs a simple smoke test after build runsc. + @$(MAKE) run DOCKER_RUN_OPTIONS="" ARGS="--alsologtostderr --network none --debug --TESTONLY-unsafe-nonroot=true --rootless do true" +.PHONY: smoke-tests -runsc: - $(MAKE) BAZEL_OPTIONS="build runsc" bazel +unit-tests: ## Runs all unit tests in pkg runsc and tools. + @$(MAKE) test OPTIONS="pkg/... runsc/... tools/..." +.PHONY: unit-tests -tests: - $(MAKE) BAZEL_OPTIONS="test --test_tag_filters runsc_ptrace //test/syscalls/..." bazel +tests: ## Runs all local ptrace system call tests. + @$(MAKE) test OPTIONS="--test_tag_filter runsc_ptrace test/syscalls/..." +.PHONY: tests -unit-tests: - $(MAKE) BAZEL_OPTIONS="test //pkg/... //runsc/... //tools/..." bazel +## +## Development helpers and tooling. +## +## These targets faciliate local development by automatically +## installing and configuring a runtime. Several variables may +## be used here to tweak the installation: +## RUNTIME - The name of the installed runtime (default: branch). +## RUNTIME_DIR - Where the runtime will be installed (default: temporary directory with the $RUNTIME). +## RUNTIME_BIN - The runtime binary (default: $RUNTIME_DIR/runsc). +## RUNTIME_LOG_DIR - The logs directory (default: $RUNTIME_DIR/logs). +## RUNTIME_LOGS - The log pattern (default: $RUNTIME_LOG_DIR/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%). +## +ifeq (,$(BRANCH_NAME)) +RUNTIME := runsc +RUNTIME_DIR := $(shell dirname $(shell mktemp -u))/runsc +else +RUNTIME := $(BRANCH_NAME) +RUNTIME_DIR := $(shell dirname $(shell mktemp -u))/$(BRANCH_NAME) +endif +RUNTIME_BIN := $(RUNTIME_DIR)/runsc +RUNTIME_LOG_DIR := $(RUNTIME_DIR)/logs +RUNTIME_LOGS := $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND% -.PHONY: docker-build bazel-shutdown bazel-server-start bazel-server bazel runsc tests +dev: ## Installs a set of local runtimes. Requires sudo. + @$(MAKE) refresh ARGS="--net-raw" + @$(MAKE) configure RUNTIME="$(RUNTIME)" ARGS="--net-raw" + @$(MAKE) configure RUNTIME="$(RUNTIME)-d" ARGS="--net-raw --debug --strace --log-packets" + @$(MAKE) configure RUNTIME="$(RUNTIME)-p" ARGS="--net-raw --profile" + @sudo systemctl restart docker +.PHONY: dev + +refresh: ## Refreshes the runtime binary (for development only). Must have called 'dev' or 'test-install' first. + @mkdir -p "$(RUNTIME_DIR)" + @$(MAKE) copy TARGETS=runsc DESTINATION="$(RUNTIME_BIN)" && chmod 0755 "$(RUNTIME_BIN)" +.PHONY: install + +test-install: ## Installs the runtime for testing. Requires sudo. + @$(MAKE) refresh ARGS="--net-raw --TESTONLY-test-name-env=RUNSC_TEST_NAME --debug --strace --log-packets $(ARGS)" + @$(MAKE) configure + @sudo systemctl restart docker +.PHONY: install-test + +configure: ## Configures a single runtime. Requires sudo. Typically called from dev or test-install. + @sudo sudo "$(RUNTIME_BIN)" install --experimental=true --runtime="$(RUNTIME)" -- --debug-log "$(RUNTIME_LOGS)" $(ARGS) + @echo "Installed runtime \"$(RUNTIME)\" @ $(RUNTIME_BIN)" + @echo "Logs are in: $(RUNTIME_LOG_DIR)" + @sudo rm -rf "$(RUNTIME_LOG_DIR)" && mkdir -p "$(RUNTIME_LOG_DIR)" +.PHONY: configure + +test-runtime: ## A convenient wrapper around test that provides the runtime argument. Target must still be provided. + @$(MAKE) test OPTIONS="$(OPTIONS) --test_arg=--runtime=$(RUNTIME)" +.PHONY: runtime-test diff --git a/images/BUILD b/images/BUILD new file mode 100644 index 000000000..a50f388e9 --- /dev/null +++ b/images/BUILD @@ -0,0 +1,11 @@ +package(licenses = ["notice"]) + +# The images filegroup is definitely not a hermetic target, and requires Make +# to do anything meaningful with. However, this will be slurped up and used by +# the tools/installer/images.sh installer, which will ensure that all required +# images are available locally when running vm_tests. +filegroup( + name = "images", + srcs = glob(["**"]), + visibility = ["//tools/installers:__pkg__"], +) diff --git a/images/Makefile b/images/Makefile new file mode 100644 index 000000000..1485607bd --- /dev/null +++ b/images/Makefile @@ -0,0 +1,93 @@ +#!/usr/bin/make -f + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ARCH is the architecture used for the build. This may be overriden at the +# command line in order to perform a cross-build (in a limited capacity). +ARCH := $(shell uname -m) + +# Note that the image prefixes used here must match the image mangling in +# runsc/testutil.MangleImage. Names are mangled in this way to ensure that all +# tests are using locally-defined images (that are consistent and idempotent). +REMOTE_IMAGE_PREFIX ?= gcr.io/gvisor-presubmit +LOCAL_IMAGE_PREFIX ?= gvisor.dev/images +ALL_IMAGES := $(subst /,_,$(subst ./,,$(shell find . -name Dockerfile -exec dirname {} \;))) +ifneq ($(ARCH),$(shell uname -m)) +DOCKER_PLATFORM_ARGS := --platform=$(ARCH) +else +DOCKER_PLATFORM_ARGS := +endif + +list-all-images: + @for image in $(ALL_IMAGES); do echo $${image}; done +.PHONY: list-build-images + +%-all-images: + @$(MAKE) $(patsubst %,$*-%,$(ALL_IMAGES)) + +# tag is a function that returns the tag name, given an image. +# +# The tag constructed is used to memoize the image generated (see README.md). +# This scheme is used to enable aggressive caching in a central repository, but +# ensuring that images will always be sourced using the local files if there +# are changes. +path = $(subst _,/,$(1)) +tag = $(shell find $(call path,$(1)) -type f -print | sort | xargs -n 1 sha256sum | sha256sum - | cut -c 1-16) +remote_image = $(REMOTE_IMAGE_PREFIX)/$(subst _,/,$(1))_$(ARCH):$(call tag,$(1)) +local_image = $(LOCAL_IMAGE_PREFIX)/$(subst _,/,$(1)) + +# rebuild builds the image locally. Only the "remote" tag will be applied. Note +# we need to explicitly repull the base layer in order to ensure that the +# architecture is correct. Note that we use the term "rebuild" here to avoid +# conflicting with the bazel "build" terminology, which is used elsewhere. +rebuild-%: register-cross + FROM=$(shell grep FROM $(call path,$*)/Dockerfile | cut -d' ' -f2-) && \ + docker pull $(DOCKER_PLATFORM_ARGS) $$FROM + T=$$(mktemp -d) && cp -a $(call path,$*)/* $$T && \ + docker build $(DOCKER_PLATFORM_ARGS) -t $(call remote_image,$*) $$T && \ + rm -rf $$T + +# pull will check the "remote" image and pull if necessary. If the remote image +# must be pulled, then it will tag with the latest local target. Note that pull +# may fail if the remote image is not available. +pull-%: + docker pull $(DOCKER_PLATFORM_ARGS) $(call remote_image,$*) + +# load will either pull the "remote" or build it locally. This is the preferred +# entrypoint, as it should never file. The local tag should always be set after +# this returns (either by the pull or the build). +load-%: + docker inspect $(call remote_image,$*) >/dev/null 2>&1 || $(MAKE) pull-$* || $(MAKE) rebuild-$* + docker tag $(call remote_image,$*) $(call local_image,$*) + +# push pushes the remote image, after either pulling (to validate that the tag +# already exists) or building manually. +push-%: load-% + docker push $(call remote_image,$*) + +# register-cross registers the necessary qemu binaries for cross-compilation. +# This may be used by any target that may execute containers that are not the +# native format. +register-cross: +ifneq ($(ARCH),$(shell uname -m)) +ifeq (,$(wildcard /proc/sys/fs/binfmt_misc/qemu-*)) + docker run --rm --privileged multiarch/qemu-user-static --reset --persistent yes +else + @true # Already registered. +endif +else + @true # No cross required. +endif +.PHONY: register-cross diff --git a/images/README.md b/images/README.md new file mode 100644 index 000000000..d2efb5db4 --- /dev/null +++ b/images/README.md @@ -0,0 +1,61 @@ +# Container Images + +This directory contains all images used by tests. + +Note that all these images must be pushed to the testing project hosted on +[Google Container Registry][gcr]. This will happen automatically as part of +continuous integration. This will speed up loading as images will not need to be +built from scratch for each test run. + +Image tooling is accessible via `make`, specifically via `tools/images.mk`. + +## Why make? + +Make is used because it can bootstrap the `default` image, which contains +`bazel` and all other parts of the toolchain. + +## Listing images + +To list all images, use `make list-all-images` from the top-level directory. + +## Loading and referencing images + +To build a specific image, use `make load-` from the top-level directory. +This will ensure that an image `gvisor.dev/images/:latest` is available. + +Images should always be referred to via the `gvisor.dev/images` canonical path. +This tag exists only locally, but serves to decouple tests from the underlying +image infrastructure. + +The continuous integration system can either take fine-grained dependencies on +single images via individual `load` targets, or pull all images via a single +`load-all-images` invocation. + +## Adding new images + +To add a new image, create a new directory under `images` containing a +Dockerfile and any other files that the image requires. You may choose to add to +an existing subdirectory if applicable, or create a new one. + +All images will be tagged and memoized using a hash of the directory contents. +As a result, every image should be made completely reproducible if possible. +This means using fixed tags and fixed versions whenever feasible. + +Notes that images should also be made architecture-independent if possible. The +build scripts will handling loading the appropriate architecture onto the +machine and tagging it with the single canonical tag. + +Add a `load-` dependency in the Makefile if the image is required for a +particular set of tests. This target will pull the tag from the image repository +if available. + +## Building and pushing images + +All images can be built manually by running `build-` and pushed using +`push-`. Note that you can also use `build-all-images` and +`push-all-images`. Note that pushing will require appropriate permissions in the +project. + +The continuous integration system can either take fine-grained dependencies on +individual `push` targets, or ensure all images are up-to-date with a single +`push-all-images` invocation. diff --git a/images/basic/alpine/Dockerfile b/images/basic/alpine/Dockerfile new file mode 100644 index 000000000..12b26040a --- /dev/null +++ b/images/basic/alpine/Dockerfile @@ -0,0 +1 @@ +FROM alpine:3.11.5 diff --git a/images/basic/busybox/Dockerfile b/images/basic/busybox/Dockerfile new file mode 100644 index 000000000..79b3f683a --- /dev/null +++ b/images/basic/busybox/Dockerfile @@ -0,0 +1 @@ +FROM busybox:1.31.1 diff --git a/images/basic/httpd/Dockerfile b/images/basic/httpd/Dockerfile new file mode 100644 index 000000000..83bc0ed88 --- /dev/null +++ b/images/basic/httpd/Dockerfile @@ -0,0 +1 @@ +FROM httpd:2.4.43 diff --git a/images/basic/mysql/Dockerfile b/images/basic/mysql/Dockerfile new file mode 100644 index 000000000..95da9c48d --- /dev/null +++ b/images/basic/mysql/Dockerfile @@ -0,0 +1 @@ +FROM mysql:8.0.19 diff --git a/images/basic/nginx/Dockerfile b/images/basic/nginx/Dockerfile new file mode 100644 index 000000000..af2e62526 --- /dev/null +++ b/images/basic/nginx/Dockerfile @@ -0,0 +1 @@ +FROM nginx:1.17.9 diff --git a/images/basic/python/Dockerfile b/images/basic/python/Dockerfile new file mode 100644 index 000000000..acf07cca9 --- /dev/null +++ b/images/basic/python/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3 +ENTRYPOINT ["python", "-m", "http.server", "8080"] diff --git a/images/basic/resolv/Dockerfile b/images/basic/resolv/Dockerfile new file mode 100644 index 000000000..13665bdaf --- /dev/null +++ b/images/basic/resolv/Dockerfile @@ -0,0 +1 @@ +FROM k8s.gcr.io/busybox:latest diff --git a/images/basic/ruby/Dockerfile b/images/basic/ruby/Dockerfile new file mode 100644 index 000000000..d290418fb --- /dev/null +++ b/images/basic/ruby/Dockerfile @@ -0,0 +1 @@ +FROM ruby:2.7.1 diff --git a/images/basic/tomcat/Dockerfile b/images/basic/tomcat/Dockerfile new file mode 100644 index 000000000..c7db39a36 --- /dev/null +++ b/images/basic/tomcat/Dockerfile @@ -0,0 +1 @@ +FROM tomcat:8.0 diff --git a/images/basic/ubuntu/Dockerfile b/images/basic/ubuntu/Dockerfile new file mode 100644 index 000000000..331b71343 --- /dev/null +++ b/images/basic/ubuntu/Dockerfile @@ -0,0 +1 @@ +FROM ubuntu:trusty diff --git a/images/default/Dockerfile b/images/default/Dockerfile new file mode 100644 index 000000000..2d0bb5ba5 --- /dev/null +++ b/images/default/Dockerfile @@ -0,0 +1,11 @@ +FROM fedora:31 +RUN dnf install -y dnf-plugins-core && dnf copr enable -y vbatts/bazel +RUN dnf install -y git gcc make golang gcc-c++ glibc-devel python3 which python3-pip python3-devel libffi-devel openssl-devel pkg-config glibc-static libstdc++-static patch +RUN pip install pycparser +RUN dnf install -y bazel3 +RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-289.0.0-linux-x86_64.tar.gz | \ + tar zxvf - google-cloud-sdk && \ + google-cloud-sdk/install.sh && \ + ln -s /google-cloud-sdk/bin/gcloud /usr/bin/gcloud +WORKDIR /workspace +ENTRYPOINT ["/usr/bin/bazel"] diff --git a/images/iptables/Dockerfile b/images/iptables/Dockerfile new file mode 100644 index 000000000..efd91cb80 --- /dev/null +++ b/images/iptables/Dockerfile @@ -0,0 +1,2 @@ +FROM ubuntu +RUN apt update && apt install -y iptables diff --git a/images/packetdrill/Dockerfile b/images/packetdrill/Dockerfile new file mode 100644 index 000000000..7a006c85f --- /dev/null +++ b/images/packetdrill/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic +RUN apt-get update && apt-get install -y net-tools git iptables iputils-ping \ + netcat tcpdump jq tar bison flex make +RUN hash -r +RUN git clone --branch packetdrill-v2.0 \ + https://github.com/google/packetdrill.git +RUN cd packetdrill/gtests/net/packetdrill && ./configure && make +CMD /bin/bash diff --git a/images/packetimpact/Dockerfile b/images/packetimpact/Dockerfile new file mode 100644 index 000000000..87aa99ef2 --- /dev/null +++ b/images/packetimpact/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:bionic +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + # iptables to disable OS native packet processing. + iptables \ + # nc to check that the posix_server is running. + netcat \ + # tcpdump to log brief packet sniffing. + tcpdump \ + # ip link show to display MAC addresses. + iproute2 \ + # tshark to log verbose packet sniffing. + tshark \ + # killall for cleanup. + psmisc +RUN hash -r +CMD /bin/bash diff --git a/images/runtimes/go1.12/Dockerfile b/images/runtimes/go1.12/Dockerfile new file mode 100644 index 000000000..cb2944062 --- /dev/null +++ b/images/runtimes/go1.12/Dockerfile @@ -0,0 +1,4 @@ +# Go is easy, since we already have everything we need to compile the proctor +# binary and run the tests in the golang Docker image. +FROM golang:1.12 +RUN ["go", "tool", "dist", "test", "-compile-only"] diff --git a/images/runtimes/java11/Dockerfile b/images/runtimes/java11/Dockerfile new file mode 100644 index 000000000..03bc8aaf1 --- /dev/null +++ b/images/runtimes/java11/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:bionic +RUN apt-get update && apt-get install -y \ + autoconf \ + build-essential \ + curl \ + make \ + openjdk-11-jdk \ + unzip \ + zip + +# Download the JDK test library. +WORKDIR /root +RUN set -ex \ + && curl -fsSL --retry 10 -o /tmp/jdktests.tar.gz http://hg.openjdk.java.net/jdk/jdk11/archive/76072a077ee1.tar.gz/test \ + && tar -xzf /tmp/jdktests.tar.gz \ + && mv jdk11-76072a077ee1/test test \ + && rm -f /tmp/jdktests.tar.gz + +# Install jtreg and add to PATH. +RUN curl -o jtreg.tar.gz https://ci.adoptopenjdk.net/view/Dependencies/job/jtreg/lastSuccessfulBuild/artifact/jtreg-4.2.0-tip.tar.gz +RUN tar -xzf jtreg.tar.gz +ENV PATH="/root/jtreg/bin:$PATH" diff --git a/images/runtimes/nodejs12.4.0/Dockerfile b/images/runtimes/nodejs12.4.0/Dockerfile new file mode 100644 index 000000000..d17924b62 --- /dev/null +++ b/images/runtimes/nodejs12.4.0/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:bionic +RUN apt-get update && apt-get install -y \ + curl \ + dumb-init \ + g++ \ + make \ + python + +WORKDIR /root +ARG VERSION=v12.4.0 +RUN curl -o node-${VERSION}.tar.gz https://nodejs.org/dist/${VERSION}/node-${VERSION}.tar.gz +RUN tar -zxf node-${VERSION}.tar.gz + +WORKDIR /root/node-${VERSION} +RUN ./configure +RUN make +RUN make test-build + +# Including dumb-init emulates the Linux "init" process, preventing the failure +# of tests involving worker processes. +ENTRYPOINT ["/usr/bin/dumb-init"] diff --git a/images/runtimes/php7.3.6/Dockerfile b/images/runtimes/php7.3.6/Dockerfile new file mode 100644 index 000000000..e5f67f79c --- /dev/null +++ b/images/runtimes/php7.3.6/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:bionic +RUN apt-get update && apt-get install -y \ + autoconf \ + automake \ + bison \ + build-essential \ + curl \ + libtool \ + libxml2-dev \ + re2c + +WORKDIR /root +ARG VERSION=7.3.6 +RUN curl -o php-${VERSION}.tar.gz https://www.php.net/distributions/php-${VERSION}.tar.gz +RUN tar -zxf php-${VERSION}.tar.gz + +WORKDIR /root/php-${VERSION} +RUN ./configure +RUN make diff --git a/images/runtimes/python3.7.3/Dockerfile b/images/runtimes/python3.7.3/Dockerfile new file mode 100644 index 000000000..4d1e1e221 --- /dev/null +++ b/images/runtimes/python3.7.3/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:bionic +RUN apt-get update && apt-get install -y \ + curl \ + gcc \ + libbz2-dev \ + libffi-dev \ + liblzma-dev \ + libreadline-dev \ + libssl-dev \ + make \ + zlib1g-dev + +# Use flags -LJO to follow the html redirect and download .tar.gz. +WORKDIR /root +ARG VERSION=3.7.3 +RUN curl -LJO https://github.com/python/cpython/archive/v${VERSION}.tar.gz +RUN tar -zxf cpython-${VERSION}.tar.gz + +WORKDIR /root/cpython-${VERSION} +RUN ./configure --with-pydebug +RUN make -s -j2 diff --git a/pkg/test/testutil/testutil.go b/pkg/test/testutil/testutil.go index d75ceca3d..ee8c78014 100644 --- a/pkg/test/testutil/testutil.go +++ b/pkg/test/testutil/testutil.go @@ -57,42 +57,10 @@ func IsCheckpointSupported() bool { return *checkpoint } -// nameToActual is used by ImageByName (for now). -var nameToActual = map[string]string{ - "basic/alpine": "alpine", - "basic/busybox": "busybox:1.31.1", - "basic/httpd": "httpd", - "basic/mysql": "mysql", - "basic/nginx": "nginx", - "basic/python": "gcr.io/gvisor-presubmit/python-hello", - "basic/resolv": "k8s.gcr.io/busybox", - "basic/ruby": "ruby", - "basic/tomcat": "tomcat:8.0", - "basic/ubuntu": "ubuntu:trusty", - "iptables": "gcr.io/gvisor-presubmit/iptables-test", - "packetdrill": "gcr.io/gvisor-presubmit/packetdrill", - "packetimpact": "gcr.io/gvisor-presubmit/packetimpact", - "runtimes/go1.12": "gcr.io/gvisor-presubmit/go1.12", - "runtimes/java11": "gcr.io/gvisor-presubmit/java11", - "runtimes/nodejs12.4.0": "gcr.io/gvisor-presubmit/nodejs12.4.0", - "runtimes/php7.3.6": "gcr.io/gvisor-presubmit/php7.3.6", - "runtimes/python3.7.3": "gcr.io/gvisor-presubmit/python3.7.3", -} - -// ImageByName mangles the image name used locally. -// -// For now, this is implemented as a static lookup table. In a subsequent -// change, this will be used to reference a locally-generated image. +// ImageByName mangles the image name used locally. This depends on the image +// build infrastructure in images/ and tools/vm. func ImageByName(name string) string { - actual, ok := nameToActual[name] - if !ok { - panic(fmt.Sprintf("unknown image: %v", name)) - } - // A terrible hack, for now execute a manual pull. - if out, err := exec.Command("docker", "pull", actual).CombinedOutput(); err != nil { - panic(fmt.Sprintf("error pulling image %q -> %q: %v, out: %s", name, actual, err, string(out))) - } - return actual + return fmt.Sprintf("gvisor.dev/images/%s", name) } // ConfigureExePath configures the executable for runsc in the test environment. diff --git a/scripts/build.sh b/scripts/build.sh index 7c9c99800..e821e8624 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -16,9 +16,6 @@ source $(dirname $0)/common.sh -# Install required packages for make_repository.sh et al. -apt_install dpkg-sig coreutils apt-utils xz-utils - # Build runsc. runsc=$(build -c opt //runsc) @@ -45,7 +42,6 @@ if [[ -v KOKORO_REPO_KEY ]]; then repo=$(tools/make_repository.sh \ "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" \ gvisor-bot@google.com \ - main \ "${KOKORO_ARTIFACTS_DIR}" \ ${pkgs}) fi diff --git a/scripts/common.sh b/scripts/common.sh index bc6ba71e8..3ca699e4a 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -84,25 +84,3 @@ function install_runsc() { # Restart docker to pick up the new runtime configuration. sudo systemctl restart docker } - -# Installs the given packages. Note that the package names should be verified to -# be correct, otherwise this may result in a loop that spins until time out. -function apt_install() { - while true; do - sudo apt-get update && - sudo apt-get install -y "$@" && - true - result="${?}" - case $result in - 0) - break - ;; - 100) - # 100 is the error code that apt-get returns. - ;; - *) - exit $result - ;; - esac - done -} diff --git a/scripts/docker_tests.sh b/scripts/docker_tests.sh index 72ba05260..931ce1aa4 100755 --- a/scripts/docker_tests.sh +++ b/scripts/docker_tests.sh @@ -16,5 +16,7 @@ source $(dirname $0)/common.sh +make load-all-images + install_runsc_for_test docker test_runsc //test/image:image_test //test/e2e:integration_test diff --git a/scripts/hostnet_tests.sh b/scripts/hostnet_tests.sh index 41298293d..992db50dd 100755 --- a/scripts/hostnet_tests.sh +++ b/scripts/hostnet_tests.sh @@ -16,6 +16,8 @@ source $(dirname $0)/common.sh +make load-all-images + # Install the runtime and perform basic tests. install_runsc_for_test hostnet --network=host test_runsc --test_arg=-checkpoint=false //test/image:image_test //test/e2e:integration_test diff --git a/scripts/iptables_tests.sh b/scripts/iptables_tests.sh index c8da1f32d..2a8c24907 100755 --- a/scripts/iptables_tests.sh +++ b/scripts/iptables_tests.sh @@ -16,6 +16,8 @@ source $(dirname $0)/common.sh +make load-iptables + install_runsc_for_test iptables --net-raw -test //test/iptables:iptables_test --test_arg=--runtime=runc -test //test/iptables:iptables_test --test_arg=--runtime=${RUNTIME} +test //test/iptables:iptables_test "--test_arg=--runtime=runc" +test //test/iptables:iptables_test "--test_arg=--runtime=${RUNTIME}" diff --git a/scripts/kvm_tests.sh b/scripts/kvm_tests.sh index 5662401df..619571c74 100755 --- a/scripts/kvm_tests.sh +++ b/scripts/kvm_tests.sh @@ -16,6 +16,8 @@ source $(dirname $0)/common.sh +make load-all-images + # Ensure that KVM is loaded, and we can use it. (lsmod | grep -E '^(kvm_intel|kvm_amd)') || sudo modprobe kvm sudo chmod a+rw /dev/kvm diff --git a/scripts/make_tests.sh b/scripts/make_tests.sh index 79426756d..dbf1bba77 100755 --- a/scripts/make_tests.sh +++ b/scripts/make_tests.sh @@ -16,10 +16,5 @@ source $(dirname $0)/common.sh -top_level=$(git rev-parse --show-toplevel 2>/dev/null) -[[ $? -eq 0 ]] && cd "${top_level}" || exit 1 - -make make runsc -make BAZEL_OPTIONS="build //..." bazel make bazel-shutdown diff --git a/scripts/overlay_tests.sh b/scripts/overlay_tests.sh index 2a1f12c0b..448864953 100755 --- a/scripts/overlay_tests.sh +++ b/scripts/overlay_tests.sh @@ -16,6 +16,8 @@ source $(dirname $0)/common.sh +make load-all-images + # Install the runtime and perform basic tests. install_runsc_for_test overlay --overlay test_runsc //test/image:image_test //test/e2e:integration_test diff --git a/scripts/packetdrill_tests.sh b/scripts/packetdrill_tests.sh index fc6bef79c..f0fc444c8 100755 --- a/scripts/packetdrill_tests.sh +++ b/scripts/packetdrill_tests.sh @@ -16,5 +16,7 @@ source $(dirname $0)/common.sh +make load-packetdrill + install_runsc_for_test runsc-d test_runsc $(bazel query "attr(tags, manual, tests(//test/packetdrill/...))") diff --git a/scripts/packetimpact_tests.sh b/scripts/packetimpact_tests.sh index 027d11e64..17fc43f27 100755 --- a/scripts/packetimpact_tests.sh +++ b/scripts/packetimpact_tests.sh @@ -16,5 +16,7 @@ source $(dirname $0)/common.sh +make load-packetimpact + install_runsc_for_test runsc-d test_runsc $(bazel query "attr(tags, packetimpact, tests(//test/packetimpact/...))") diff --git a/scripts/root_tests.sh b/scripts/root_tests.sh index 4e4fcc76b..d629bf2aa 100755 --- a/scripts/root_tests.sh +++ b/scripts/root_tests.sh @@ -16,6 +16,8 @@ source $(dirname $0)/common.sh +make load-all-images + # Reinstall the latest containerd shim. declare -r base="https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim" declare -r latest=$(mktemp --tmpdir gvisor-containerd-shim-latest.XXXXXX) @@ -28,4 +30,3 @@ sudo mv ${shim_path} /usr/local/bin/gvisor-containerd-shim # Run the tests that require root. install_runsc_for_test root run_as_root //test/root:root_test --runtime=${RUNTIME} - diff --git a/scripts/swgso_tests.sh b/scripts/swgso_tests.sh index 0de2df1d2..c67f2fe5c 100755 --- a/scripts/swgso_tests.sh +++ b/scripts/swgso_tests.sh @@ -16,6 +16,8 @@ source $(dirname $0)/common.sh +make load-all-images + # Install the runtime and perform basic tests. install_runsc_for_test swgso --software-gso=true --gso=false test_runsc //test/image:image_test //test/e2e:integration_test diff --git a/test/iptables/runner/Dockerfile b/test/iptables/runner/Dockerfile deleted file mode 100644 index b77db44a1..000000000 --- a/test/iptables/runner/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -# This Dockerfile builds the image hosted at -# gcr.io/gvisor-presubmit/iptables-test. -FROM ubuntu -RUN apt update && apt install -y iptables diff --git a/test/packetdrill/Dockerfile b/test/packetdrill/Dockerfile deleted file mode 100644 index 4b75e9527..000000000 --- a/test/packetdrill/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:bionic - -RUN apt-get update && apt-get install -y net-tools git iptables iputils-ping \ - netcat tcpdump jq tar bison flex make -RUN hash -r -RUN git clone --branch packetdrill-v2.0 \ - https://github.com/google/packetdrill.git -RUN cd packetdrill/gtests/net/packetdrill && ./configure && make -CMD /bin/bash diff --git a/test/packetimpact/tests/Dockerfile b/test/packetimpact/tests/Dockerfile deleted file mode 100644 index 9075bc555..000000000 --- a/test/packetimpact/tests/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:bionic - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - # iptables to disable OS native packet processing. - iptables \ - # nc to check that the posix_server is running. - netcat \ - # tcpdump to log brief packet sniffing. - tcpdump \ - # ip link show to display MAC addresses. - iproute2 \ - # tshark to log verbose packet sniffing. - tshark \ - # killall for cleanup. - psmisc -RUN hash -r -CMD /bin/bash diff --git a/test/root/BUILD b/test/root/BUILD index 17e51e66e..639e293e3 100644 --- a/test/root/BUILD +++ b/test/root/BUILD @@ -48,6 +48,7 @@ go_test( vm_test( name = "root_vm_test", + size = "large", shard_count = 1, targets = [ "//tools/installers:shim", diff --git a/test/runtimes/images/Dockerfile_go1.12 b/test/runtimes/images/Dockerfile_go1.12 deleted file mode 100644 index ab9d6abf3..000000000 --- a/test/runtimes/images/Dockerfile_go1.12 +++ /dev/null @@ -1,10 +0,0 @@ -# Go is easy, since we already have everything we need to compile the proctor -# binary and run the tests in the golang Docker image. -FROM golang:1.12 -ADD ["proctor/", "/go/src/proctor/"] -RUN ["go", "build", "-o", "/proctor", "/go/src/proctor"] - -# Pre-compile the tests so we don't need to do so in each test run. -RUN ["go", "tool", "dist", "test", "-compile-only"] - -ENTRYPOINT ["/proctor", "--runtime=go"] diff --git a/test/runtimes/images/Dockerfile_java11 b/test/runtimes/images/Dockerfile_java11 deleted file mode 100644 index 9b7c3d5a3..000000000 --- a/test/runtimes/images/Dockerfile_java11 +++ /dev/null @@ -1,30 +0,0 @@ -# Compile the proctor binary. -FROM golang:1.12 AS golang -ADD ["proctor/", "/go/src/proctor/"] -RUN ["go", "build", "-o", "/proctor", "/go/src/proctor"] - -FROM ubuntu:bionic -RUN apt-get update && apt-get install -y \ - autoconf \ - build-essential \ - curl \ - make \ - openjdk-11-jdk \ - unzip \ - zip - -# Download the JDK test library. -WORKDIR /root -RUN set -ex \ - && curl -fsSL --retry 10 -o /tmp/jdktests.tar.gz http://hg.openjdk.java.net/jdk/jdk11/archive/76072a077ee1.tar.gz/test \ - && tar -xzf /tmp/jdktests.tar.gz \ - && mv jdk11-76072a077ee1/test test \ - && rm -f /tmp/jdktests.tar.gz - -# Install jtreg and add to PATH. -RUN curl -o jtreg.tar.gz https://ci.adoptopenjdk.net/view/Dependencies/job/jtreg/lastSuccessfulBuild/artifact/jtreg-4.2.0-tip.tar.gz -RUN tar -xzf jtreg.tar.gz -ENV PATH="/root/jtreg/bin:$PATH" - -COPY --from=golang /proctor /proctor -ENTRYPOINT ["/proctor", "--runtime=java"] diff --git a/test/runtimes/images/Dockerfile_nodejs12.4.0 b/test/runtimes/images/Dockerfile_nodejs12.4.0 deleted file mode 100644 index 26f68b487..000000000 --- a/test/runtimes/images/Dockerfile_nodejs12.4.0 +++ /dev/null @@ -1,28 +0,0 @@ -# Compile the proctor binary. -FROM golang:1.12 AS golang -ADD ["proctor/", "/go/src/proctor/"] -RUN ["go", "build", "-o", "/proctor", "/go/src/proctor"] - -FROM ubuntu:bionic -RUN apt-get update && apt-get install -y \ - curl \ - dumb-init \ - g++ \ - make \ - python - -WORKDIR /root -ARG VERSION=v12.4.0 -RUN curl -o node-${VERSION}.tar.gz https://nodejs.org/dist/${VERSION}/node-${VERSION}.tar.gz -RUN tar -zxf node-${VERSION}.tar.gz - -WORKDIR /root/node-${VERSION} -RUN ./configure -RUN make -RUN make test-build - -COPY --from=golang /proctor /proctor - -# Including dumb-init emulates the Linux "init" process, preventing the failure -# of tests involving worker processes. -ENTRYPOINT ["/usr/bin/dumb-init", "/proctor", "--runtime=nodejs"] diff --git a/test/runtimes/images/Dockerfile_php7.3.6 b/test/runtimes/images/Dockerfile_php7.3.6 deleted file mode 100644 index e6b4c6329..000000000 --- a/test/runtimes/images/Dockerfile_php7.3.6 +++ /dev/null @@ -1,27 +0,0 @@ -# Compile the proctor binary. -FROM golang:1.12 AS golang -ADD ["proctor/", "/go/src/proctor/"] -RUN ["go", "build", "-o", "/proctor", "/go/src/proctor"] - -FROM ubuntu:bionic -RUN apt-get update && apt-get install -y \ - autoconf \ - automake \ - bison \ - build-essential \ - curl \ - libtool \ - libxml2-dev \ - re2c - -WORKDIR /root -ARG VERSION=7.3.6 -RUN curl -o php-${VERSION}.tar.gz https://www.php.net/distributions/php-${VERSION}.tar.gz -RUN tar -zxf php-${VERSION}.tar.gz - -WORKDIR /root/php-${VERSION} -RUN ./configure -RUN make - -COPY --from=golang /proctor /proctor -ENTRYPOINT ["/proctor", "--runtime=php"] diff --git a/test/runtimes/images/Dockerfile_python3.7.3 b/test/runtimes/images/Dockerfile_python3.7.3 deleted file mode 100644 index 905cd22d7..000000000 --- a/test/runtimes/images/Dockerfile_python3.7.3 +++ /dev/null @@ -1,30 +0,0 @@ -# Compile the proctor binary. -FROM golang:1.12 AS golang -ADD ["proctor/", "/go/src/proctor/"] -RUN ["go", "build", "-o", "/proctor", "/go/src/proctor"] - -FROM ubuntu:bionic - -RUN apt-get update && apt-get install -y \ - curl \ - gcc \ - libbz2-dev \ - libffi-dev \ - liblzma-dev \ - libreadline-dev \ - libssl-dev \ - make \ - zlib1g-dev - -# Use flags -LJO to follow the html redirect and download .tar.gz. -WORKDIR /root -ARG VERSION=3.7.3 -RUN curl -LJO https://github.com/python/cpython/archive/v${VERSION}.tar.gz -RUN tar -zxf cpython-${VERSION}.tar.gz - -WORKDIR /root/cpython-${VERSION} -RUN ./configure --with-pydebug -RUN make -s -j2 - -COPY --from=golang /proctor /proctor -ENTRYPOINT ["/proctor", "--runtime=python"] diff --git a/tools/bazel.mk b/tools/bazel.mk new file mode 100644 index 000000000..45fbbecca --- /dev/null +++ b/tools/bazel.mk @@ -0,0 +1,106 @@ +#!/usr/bin/make -f + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See base Makefile. +BRANCH_NAME := $(shell (git branch --show-current 2>/dev/null || \ + git rev-parse --abbrev-ref HEAD 2>/dev/null) | \ + xargs -n 1 basename 2>/dev/null) + +# Bazel container configuration (see below). +USER ?= gvisor +DOCKER_NAME ?= gvisor-bazel +DOCKER_RUN_OPTIONS ?= --privileged +BAZEL_CACHE := $(shell readlink -m ~/.cache/bazel/) +GCLOUD_CONFIG := $(shell readlink -m ~/.config/gcloud/) +DOCKER_SOCKET := /var/run/docker.sock + +# Non-configurable. +UID := $(shell id -u ${USER}) +GID := $(shell id -g ${USER}) +FULL_DOCKER_RUN_OPTIONS := $(DOCKER_RUN_OPTIONS) +FULL_DOCKER_RUN_OPTIONS += -v "$(BAZEL_CACHE):$(BAZEL_CACHE)" +FULL_DOCKER_RUN_OPTIONS += -v "$(GCLOUD_CONFIG):$(GCLOUD_CONFIG)" +FULL_DOCKER_RUN_OPTIONS += -v "$(DOCKER_SOCKET):$(DOCKER_SOCKET)" + +## +## Bazel helpers. +## +## This file supports targets that wrap bazel in a running Docker +## container to simplify development. Some options are available to +## control the behavior of this container: +## USER - The in-container user. +## DOCKER_RUN_OPTIONS - Options for the container (default: --privileged, required for tests). +## DOCKER_NAME - The container name (default: gvisor-bazel-HASH). +## BAZEL_CACHE - The bazel cache directory (default: detected). +## GCLOUD_CONFIG - The gcloud config directory (detect: detected). +## DOCKER_SOCKET - The Docker socket (default: detected). +## +bazel-server-start: load-default ## Starts the bazel server. + docker run -d --rm \ + --name $(DOCKER_NAME) \ + --user 0:0 \ + -v "$(CURDIR):$(CURDIR)" \ + --workdir "$(CURDIR)" \ + --tmpfs /tmp:rw,exec \ + --entrypoint "" \ + $(FULL_DOCKER_RUN_OPTIONS) \ + gvisor.dev/images/default \ + sh -c "groupadd --gid $(GID) --non-unique $(USER) && \ + useradd --uid $(UID) --non-unique --no-create-home --gid $(GID) -d $(HOME) $(USER) && \ + bazel version && \ + while :; do sleep 3600; done" + @while :; do if docker logs $(DOCKER_NAME) 2>/dev/null | grep "Build label:" >/dev/null; then break; fi; sleep 1; done +.PHONY: bazel-server-start + +bazel-shutdown: ## Shuts down a running bazel server. + @docker exec --user $(UID):$(GID) $(DOCKER_NAME) bazel shutdown; rc=$$?; docker kill $(DOCKER_NAME) || [[ $$rc -ne 0 ]] +.PHONY: bazel-shutdown + +bazel-alias: ## Emits an alias that can be used within the shell. + @echo "alias bazel='docker exec --user $(UID):$(GID) -i $(DOCKER_NAME) bazel'" +.PHONY: bazel-alias + +bazel-server: ## Ensures that the server exists. Used as an internal target. + @docker exec $(DOCKER_NAME) true || $(MAKE) bazel-server-start +.PHONY: bazel-server + +build_paths = docker exec --user $(UID):$(GID) -i $(DOCKER_NAME) sh -c 'bazel build $(OPTIONS) $(TARGETS) 2>&1 \ + | tee /dev/fd/2 \ + | grep -E "^ bazel-bin/" \ + | awk "{print $$1;}"' \ + | xargs -n 1 -I {} sh -c "$(1)" + +build: bazel-server + @$(call build_paths,echo {}) +.PHONY: build + +copy: bazel-server +ifeq (,$(DESTINATION)) + $(error Destination not provided.) +endif + @$(call build_paths,cp -a {} $(DESTINATION)) + +run: bazel-server + @$(call build_paths,{} $(ARGS)) +.PHONY: run + +sudo: bazel-server + @$(call build_paths,sudo -E {} $(ARGS)) +.PHONY: sudo + +test: bazel-server + @docker exec --user $(UID):$(GID) -i $(DOCKER_NAME) bazel test $(OPTIONS) $(TARGETS) +.PHONY: test diff --git a/tools/installers/BUILD b/tools/installers/BUILD index d78a265ca..caa7b1983 100644 --- a/tools/installers/BUILD +++ b/tools/installers/BUILD @@ -16,6 +16,14 @@ sh_binary( data = [":runsc"], ) +sh_binary( + name = "images", + srcs = ["images.sh"], + data = [ + "//images", + ], +) + sh_binary( name = "master", srcs = ["master.sh"], diff --git a/tools/installers/images.sh b/tools/installers/images.sh new file mode 100755 index 000000000..52e750f57 --- /dev/null +++ b/tools/installers/images.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeuo pipefail + +# Find the images directory. +for images in $(find . -type d -name images); do + if [[ -f "${images}"/Makefile ]]; then + make -C "${images}" load-all-images + fi +done diff --git a/tools/make_repository.sh b/tools/make_repository.sh index 27ffbc9f3..c91fd283c 100755 --- a/tools/make_repository.sh +++ b/tools/make_repository.sh @@ -17,14 +17,37 @@ # Parse arguments. We require more than two arguments, which are the private # keyring, the e-mail associated with the signer, and the list of packages. if [ "$#" -le 3 ]; then - echo "usage: $0 " + echo "usage: $0 " exit 1 fi declare -r private_key=$(readlink -e "$1"); shift declare -r signer="$1"; shift -declare -r component="$1"; shift declare -r root="$1"; shift +# Ensure that we have the correct packages installed. +function apt_install() { + while true; do + sudo apt-get update && + sudo apt-get install -y "$@" && + true + result="${?}" + case $result in + 0) + break + ;; + 100) + # 100 is the error code that apt-get returns. + ;; + *) + exit $result + ;; + esac + done +} +dpkg-sig --help >/dev/null || apt_install dpkg-sig +apt-ftparchive --help >/dev/null || apt_install apt-utils +xz --help >/dev/null || apt_install xz-utils + # Verbose from this point. set -xeo pipefail @@ -78,7 +101,7 @@ for dir in "${root}"/pool/*/binary-*; do name=$(basename "${dir}") arch=${name##binary-} arches+=("${arch}") - repo_packages="${tmpdir}"/"${component}"/"${name}" + repo_packages="${tmpdir}"/main/"${name}" mkdir -p "${repo_packages}" (cd "${root}" && apt-ftparchive --arch "${arch}" packages pool > "${repo_packages}"/Packages) (cd "${repo_packages}" && cat Packages | gzip > Packages.gz) @@ -91,7 +114,7 @@ APT { FTPArchive { Release { Architectures "${arches[@]}"; - Components "${component}"; + Components "main"; }; }; }; diff --git a/tools/vm/defs.bzl b/tools/vm/defs.bzl index 24bf0aabc..61feefcbc 100644 --- a/tools/vm/defs.bzl +++ b/tools/vm/defs.bzl @@ -183,7 +183,10 @@ def vm_test( """ targets = kwargs.pop("targets", []) if installers == None: - installers = ["//tools/installers:head"] + installers = [ + "//tools/installers:head", + "//tools/installers:images", + ] targets = installers + targets if default_installer(): targets = [default_installer()] + targets -- cgit v1.2.3