diff options
| -rw-r--r-- | images/Makefile | 4 | ||||
| -rw-r--r-- | images/packetimpact/Dockerfile | 8 | ||||
| -rw-r--r-- | pkg/test/dockerutil/container.go | 12 | ||||
| -rw-r--r-- | test/packetimpact/dut/BUILD | 10 | ||||
| -rw-r--r-- | test/packetimpact/dut/posix_server.cc | 5 | ||||
| -rw-r--r-- | test/packetimpact/runner/BUILD | 26 | ||||
| -rw-r--r-- | test/packetimpact/runner/defs.bzl | 5 | ||||
| -rw-r--r-- | test/packetimpact/runner/dut.go | 438 | ||||
| -rw-r--r-- | test/packetimpact/runner/packetimpact_test.go | 359 |
9 files changed, 492 insertions, 375 deletions
diff --git a/images/Makefile b/images/Makefile index 278dec02f..d183155a8 100644 --- a/images/Makefile +++ b/images/Makefile @@ -59,9 +59,9 @@ local_image = $(LOCAL_IMAGE_PREFIX)/$(subst _,/,$(1)) # we need to explicitly repull the base layer in order to ensure that the # architecture is correct. Note that we use the term "rebuild" here to avoid # conflicting with the bazel "build" terminology, which is used elsewhere. -rebuild-%: FROM=$(shell grep FROM $(call path,$*)/Dockerfile } cut -d' ' -f2) +rebuild-%: FROM=$(shell grep FROM $(call path,$*)/Dockerfile | cut -d' ' -f2) rebuild-%: register-cross - $(foreach IMAGE,$(FROM),docker $(DOCKER_PLATFORM_ARGS) $(IMAGE); &&) true + $(foreach IMAGE,$(FROM),docker pull $(DOCKER_PLATFORM_ARGS) $(IMAGE) &&) true && \ T=$$(mktemp -d) && cp -a $(call path,$*)/* $$T && \ docker build $(DOCKER_PLATFORM_ARGS) -t $(call remote_image,$*) $$T && \ rm -rf $$T diff --git a/images/packetimpact/Dockerfile b/images/packetimpact/Dockerfile index 87aa99ef2..82b7e8abd 100644 --- a/images/packetimpact/Dockerfile +++ b/images/packetimpact/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:bionic +FROM ubuntu:focal RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ # iptables to disable OS native packet processing. iptables \ @@ -11,6 +11,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ # tshark to log verbose packet sniffing. tshark \ # killall for cleanup. - psmisc + psmisc \ + # qemu-system-x86 to emulate fuchsia. + qemu-system-x86 \ + # sha1sum to generate entropy. + libdigest-sha-perl RUN hash -r CMD /bin/bash diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index 052b6b99d..64d17f661 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -22,6 +22,7 @@ import ( "net" "os" "path" + "path/filepath" "regexp" "strconv" "strings" @@ -403,10 +404,13 @@ func (c *Container) CopyFiles(opts *RunOpts, target string, sources ...string) { return } for _, name := range sources { - src, err := testutil.FindFile(name) - if err != nil { - c.copyErr = fmt.Errorf("testutil.FindFile(%q) failed: %v", name, err) - return + src := name + if !filepath.IsAbs(src) { + src, err = testutil.FindFile(name) + if err != nil { + c.copyErr = fmt.Errorf("testutil.FindFile(%q) failed: %w", name, err) + return + } } dst := path.Join(dir, path.Base(name)) if err := testutil.Copy(src, dst); err != nil { diff --git a/test/packetimpact/dut/BUILD b/test/packetimpact/dut/BUILD index 3ce63c2c6..ccf1c735f 100644 --- a/test/packetimpact/dut/BUILD +++ b/test/packetimpact/dut/BUILD @@ -16,3 +16,13 @@ cc_binary( "//test/packetimpact/proto:posix_server_cc_proto", ], ) + +cc_binary( + name = "posix_server_dynamic", + srcs = ["posix_server.cc"], + deps = [ + grpcpp, + "//test/packetimpact/proto:posix_server_cc_grpc_proto", + "//test/packetimpact/proto:posix_server_cc_proto", + ], +) diff --git a/test/packetimpact/dut/posix_server.cc b/test/packetimpact/dut/posix_server.cc index de5b4be93..2f3becfba 100644 --- a/test/packetimpact/dut/posix_server.cc +++ b/test/packetimpact/dut/posix_server.cc @@ -21,6 +21,7 @@ #include <string.h> #include <sys/socket.h> #include <sys/types.h> +#include <time.h> #include <unistd.h> #include <iostream> @@ -307,9 +308,9 @@ class PosixImpl final : public posix_server::Posix::Service { break; } case ::posix_server::SockOptVal::kTimeval: { - timeval tv = {.tv_sec = static_cast<__time_t>( + timeval tv = {.tv_sec = static_cast<time_t>( request->optval().timeval().seconds()), - .tv_usec = static_cast<__suseconds_t>( + .tv_usec = static_cast<suseconds_t>( request->optval().timeval().microseconds())}; response->set_ret(setsockopt(request->sockfd(), request->level(), request->optname(), &tv, sizeof(tv))); diff --git a/test/packetimpact/runner/BUILD b/test/packetimpact/runner/BUILD index ff2be9b30..605dd4972 100644 --- a/test/packetimpact/runner/BUILD +++ b/test/packetimpact/runner/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "bzl_library", "go_test") +load("//tools:defs.bzl", "bzl_library", "go_library", "go_test") package( default_visibility = ["//test/packetimpact:__subpackages__"], @@ -7,21 +7,31 @@ package( go_test( name = "packetimpact_test", - srcs = ["packetimpact_test.go"], + srcs = [ + "packetimpact_test.go", + ], tags = [ # Not intended to be run directly. "local", "manual", ], - deps = [ - "//pkg/test/dockerutil", - "//test/packetimpact/netdevs", - "@com_github_docker_docker//api/types/mount:go_default_library", - ], + deps = [":runner"], ) bzl_library( name = "defs_bzl", srcs = ["defs.bzl"], - visibility = ["//visibility:private"], + visibility = ["//test/packetimpact:__subpackages__"], +) + +go_library( + name = "runner", + testonly = True, + srcs = ["dut.go"], + visibility = ["//test/packetimpact:__subpackages__"], + deps = [ + "//pkg/test/dockerutil", + "//test/packetimpact/netdevs", + "@com_github_docker_docker//api/types/mount:go_default_library", + ], ) diff --git a/test/packetimpact/runner/defs.bzl b/test/packetimpact/runner/defs.bzl index d72c63fe6..f56d3c42e 100644 --- a/test/packetimpact/runner/defs.bzl +++ b/test/packetimpact/runner/defs.bzl @@ -23,8 +23,9 @@ def _packetimpact_test_impl(ctx): transitive_files = [] if hasattr(ctx.attr._test_runner, "data_runfiles"): transitive_files.append(ctx.attr._test_runner.data_runfiles.files) + files = [test_runner] + ctx.files.testbench_binary + ctx.files._posix_server runfiles = ctx.runfiles( - files = [test_runner] + ctx.files.testbench_binary + ctx.files._posix_server_binary, + files = files, transitive_files = depset(transitive = transitive_files), collect_default = True, collect_data = True, @@ -38,7 +39,7 @@ _packetimpact_test = rule( cfg = "target", default = ":packetimpact_test", ), - "_posix_server_binary": attr.label( + "_posix_server": attr.label( cfg = "target", default = "//test/packetimpact/dut:posix_server", ), diff --git a/test/packetimpact/runner/dut.go b/test/packetimpact/runner/dut.go new file mode 100644 index 000000000..be7b52f18 --- /dev/null +++ b/test/packetimpact/runner/dut.go @@ -0,0 +1,438 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runner starts docker containers and networking for a packetimpact test. +package runner + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "log" + "math/rand" + "net" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types/mount" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/packetimpact/netdevs" +) + +// stringList implements flag.Value. +type stringList []string + +// String implements flag.Value.String. +func (l *stringList) String() string { + return strings.Join(*l, ",") +} + +// Set implements flag.Value.Set. +func (l *stringList) Set(value string) error { + *l = append(*l, value) + return nil +} + +var ( + native = false + testbenchBinary = "" + tshark = false + extraTestArgs = stringList{} + expectFailure = false + + // DutAddr is the IP addres for DUT. + DutAddr = net.IPv4(0, 0, 0, 10) + testbenchAddr = net.IPv4(0, 0, 0, 20) +) + +// RegisterFlags defines flags and associates them with the package-level +// exported variables above. It should be called by tests in their init +// functions. +func RegisterFlags(fs *flag.FlagSet) { + fs.BoolVar(&native, "native", false, "whether the test should be run natively") + fs.StringVar(&testbenchBinary, "testbench_binary", "", "path to the testbench binary") + fs.BoolVar(&tshark, "tshark", false, "use more verbose tshark in logs instead of tcpdump") + flag.Var(&extraTestArgs, "extra_test_arg", "extra arguments to pass to the testbench") + flag.BoolVar(&expectFailure, "expect_failure", false, "expect that the test will fail when run") +} + +// CtrlPort is the port that posix_server listens on. +const CtrlPort = "40000" + +// logger implements testutil.Logger. +// +// Labels logs based on their source and formats multi-line logs. +type logger string + +// Name implements testutil.Logger.Name. +func (l logger) Name() string { + return string(l) +} + +// Logf implements testutil.Logger.Logf. +func (l logger) Logf(format string, args ...interface{}) { + lines := strings.Split(fmt.Sprintf(format, args...), "\n") + log.Printf("%s: %s", l, lines[0]) + for _, line := range lines[1:] { + log.Printf("%*s %s", len(l), "", line) + } +} + +// TestWithDUT runs a packetimpact test with the given information. +func TestWithDUT(ctx context.Context, t *testing.T, mkDevice func(*dockerutil.Container) DUT, containerAddr net.IP) { + if testbenchBinary == "" { + t.Fatal("--testbench_binary is missing") + } + dockerutil.EnsureSupportedDockerVersion() + + // Create the networks needed for the test. One control network is needed for + // the gRPC control packets and one test network on which to transmit the test + // packets. + ctrlNet := dockerutil.NewNetwork(ctx, logger("ctrlNet")) + testNet := dockerutil.NewNetwork(ctx, logger("testNet")) + for _, dn := range []*dockerutil.Network{ctrlNet, testNet} { + for { + if err := createDockerNetwork(ctx, dn); err != nil { + t.Log("creating docker network:", err) + const wait = 100 * time.Millisecond + t.Logf("sleeping %s and will try creating docker network again", wait) + // This can fail if another docker network claimed the same IP so we'll + // just try again. + time.Sleep(wait) + continue + } + break + } + dn := dn + t.Cleanup(func() { + if err := dn.Cleanup(ctx); err != nil { + t.Errorf("unable to cleanup container %s: %s", dn.Name, err) + } + }) + // Sanity check. + if inspect, err := dn.Inspect(ctx); err != nil { + t.Fatalf("failed to inspect network %s: %v", dn.Name, err) + } else if inspect.Name != dn.Name { + t.Fatalf("name mismatch for network want: %s got: %s", dn.Name, inspect.Name) + } + } + + tmpDir, err := ioutil.TempDir("", "container-output") + if err != nil { + t.Fatal("creating temp dir:", err) + } + t.Cleanup(func() { + if err := exec.Command("/bin/cp", "-r", tmpDir, os.Getenv("TEST_UNDECLARED_OUTPUTS_DIR")).Run(); err != nil { + t.Errorf("unable to copy container output files: %s", err) + } + if err := os.RemoveAll(tmpDir); err != nil { + t.Errorf("failed to remove tmpDir %s: %s", tmpDir, err) + } + }) + + const testOutputDir = "/tmp/testoutput" + + // Create the Docker container for the DUT. + var dut *dockerutil.Container + if native { + dut = dockerutil.MakeNativeContainer(ctx, logger("dut")) + } else { + dut = dockerutil.MakeContainer(ctx, logger("dut")) + } + t.Cleanup(func() { + dut.CleanUp(ctx) + }) + + runOpts := dockerutil.RunOpts{ + Image: "packetimpact", + CapAdd: []string{"NET_ADMIN"}, + Mounts: []mount.Mount{{ + Type: mount.TypeBind, + Source: tmpDir, + Target: testOutputDir, + ReadOnly: false, + }}, + } + + // Add ctrlNet as eth1 and testNet as eth2. + const testNetDev = "eth2" + + device := mkDevice(dut) + remoteIPv6, remoteMAC, dutDeviceID := device.Prepare(ctx, t, runOpts, ctrlNet, testNet, containerAddr) + + // Create the Docker container for the testbench. + testbench := dockerutil.MakeNativeContainer(ctx, logger("testbench")) + + tbb := path.Base(testbenchBinary) + containerTestbenchBinary := filepath.Join("/packetimpact", tbb) + testbench.CopyFiles(&runOpts, "/packetimpact", filepath.Join("test/packetimpact/tests", tbb)) + + // Run tcpdump in the test bench unbuffered, without DNS resolution, just on + // the interface with the test packets. + snifferArgs := []string{ + "tcpdump", + "-S", "-vvv", "-U", "-n", + "-i", testNetDev, + "-w", testOutputDir + "/dump.pcap", + } + snifferRegex := "tcpdump: listening.*\n" + if tshark { + // Run tshark in the test bench unbuffered, without DNS resolution, just on + // the interface with the test packets. + snifferArgs = []string{ + "tshark", "-V", "-l", "-n", "-i", testNetDev, + "-o", "tcp.check_checksum:TRUE", + "-o", "udp.check_checksum:TRUE", + } + snifferRegex = "Capturing on.*\n" + } + + if err := StartContainer( + ctx, + runOpts, + testbench, + testbenchAddr, + []*dockerutil.Network{ctrlNet, testNet}, + snifferArgs..., + ); err != nil { + t.Fatalf("failed to start docker container for testbench sniffer: %s", err) + } + // Kill so that it will flush output. + t.Cleanup(func() { + time.Sleep(1 * time.Second) + testbench.Exec(ctx, dockerutil.ExecOpts{}, "killall", snifferArgs[0]) + }) + + if _, err := testbench.WaitForOutput(ctx, snifferRegex, 60*time.Second); err != nil { + t.Fatalf("sniffer on %s never listened: %s", dut.Name, err) + } + + // When the Linux kernel receives a SYN-ACK for a SYN it didn't send, it + // will respond with an RST. In most packetimpact tests, the SYN is sent + // by the raw socket and the kernel knows nothing about the connection, this + // behavior will break lots of TCP related packetimpact tests. To prevent + // this, we can install the following iptables rules. The raw socket that + // packetimpact tests use will still be able to see everything. + for _, bin := range []string{"iptables", "ip6tables"} { + if logs, err := testbench.Exec(ctx, dockerutil.ExecOpts{}, bin, "-A", "INPUT", "-i", testNetDev, "-p", "tcp", "-j", "DROP"); err != nil { + t.Fatalf("unable to Exec %s on container %s: %s, logs from testbench:\n%s", bin, testbench.Name, err, logs) + } + } + + // FIXME(b/156449515): Some piece of the system has a race. The old + // bash script version had a sleep, so we have one too. The race should + // be fixed and this sleep removed. + time.Sleep(time.Second) + + // Start a packetimpact test on the test bench. The packetimpact test sends + // and receives packets and also sends POSIX socket commands to the + // posix_server to be executed on the DUT. + testArgs := []string{containerTestbenchBinary} + testArgs = append(testArgs, extraTestArgs...) + testArgs = append(testArgs, + "--posix_server_ip", AddressInSubnet(DutAddr, *ctrlNet.Subnet).String(), + "--posix_server_port", CtrlPort, + "--remote_ipv4", AddressInSubnet(DutAddr, *testNet.Subnet).String(), + "--local_ipv4", AddressInSubnet(testbenchAddr, *testNet.Subnet).String(), + "--remote_ipv6", remoteIPv6.String(), + "--remote_mac", remoteMAC.String(), + "--remote_interface_id", fmt.Sprintf("%d", dutDeviceID), + "--device", testNetDev, + fmt.Sprintf("--native=%t", native), + ) + testbenchLogs, err := testbench.Exec(ctx, dockerutil.ExecOpts{}, testArgs...) + if (err != nil) != expectFailure { + var dutLogs string + if logs, err := device.Logs(ctx); err != nil { + dutLogs = fmt.Sprintf("failed to fetch DUT logs: %s", err) + } else { + dutLogs = logs + } + + t.Errorf(`test error: %v, expect failure: %t + +%s + +====== Begin of Testbench Logs ====== + +%s + +====== End of Testbench Logs ======`, + err, expectFailure, dutLogs, testbenchLogs) + } +} + +// DUT describes how to setup/teardown the dut for packetimpact tests. +type DUT interface { + // Prepare prepares the dut, starts posix_server and returns the IPv6, MAC + // address and the interface ID for the testNet on DUT. + Prepare(ctx context.Context, t *testing.T, runOpts dockerutil.RunOpts, ctrlNet, testNet *dockerutil.Network, containerAddr net.IP) (net.IP, net.HardwareAddr, uint32) + // Logs retrieves the logs from the dut. + Logs(ctx context.Context) (string, error) +} + +// DockerDUT describes a docker based DUT. +type DockerDUT struct { + c *dockerutil.Container +} + +// NewDockerDUT creates a docker based DUT. +func NewDockerDUT(c *dockerutil.Container) DUT { + return &DockerDUT{ + c: c, + } +} + +// Prepare implements DUT.Prepare. +func (dut *DockerDUT) Prepare(ctx context.Context, t *testing.T, runOpts dockerutil.RunOpts, ctrlNet, testNet *dockerutil.Network, containerAddr net.IP) (net.IP, net.HardwareAddr, uint32) { + const containerPosixServerBinary = "/packetimpact/posix_server" + dut.c.CopyFiles(&runOpts, "/packetimpact", "test/packetimpact/dut/posix_server") + + if err := StartContainer( + ctx, + runOpts, + dut.c, + containerAddr, + []*dockerutil.Network{ctrlNet, testNet}, + containerPosixServerBinary, + "--ip=0.0.0.0", + "--port="+CtrlPort, + ); err != nil { + t.Fatalf("failed to start docker container for DUT: %s", err) + } + + if _, err := dut.c.WaitForOutput(ctx, "Server listening.*\n", 60*time.Second); err != nil { + t.Fatalf("%s on container %s never listened: %s", containerPosixServerBinary, dut.c.Name, err) + } + + dutTestDevice, dutDeviceInfo, err := deviceByIP(ctx, dut.c, AddressInSubnet(containerAddr, *testNet.Subnet)) + if err != nil { + t.Fatal(err) + } + + remoteMAC := dutDeviceInfo.MAC + remoteIPv6 := dutDeviceInfo.IPv6Addr + // Netstack as DUT doesn't assign IPv6 addresses automatically so do it if + // needed. + if remoteIPv6 == nil { + if _, err := dut.c.Exec(ctx, dockerutil.ExecOpts{}, "ip", "addr", "add", netdevs.MACToIP(remoteMAC).String(), "scope", "link", "dev", dutTestDevice); err != nil { + t.Fatalf("unable to ip addr add on container %s: %s", dut.c.Name, err) + } + // Now try again, to make sure that it worked. + _, dutDeviceInfo, err = deviceByIP(ctx, dut.c, AddressInSubnet(containerAddr, *testNet.Subnet)) + if err != nil { + t.Fatal(err) + } + remoteIPv6 = dutDeviceInfo.IPv6Addr + if remoteIPv6 == nil { + t.Fatalf("unable to set IPv6 address on container %s", dut.c.Name) + } + } + return remoteIPv6, dutDeviceInfo.MAC, dutDeviceInfo.ID +} + +// Logs implements DUT.Logs. +func (dut *DockerDUT) Logs(ctx context.Context) (string, error) { + logs, err := dut.c.Logs(ctx) + if err != nil { + return "", err + } + return fmt.Sprintf(`====== Begin of DUT Logs ====== + +%s + +====== End of DUT Logs ======`, logs), nil +} + +// AddNetworks connects docker network with the container and assigns the specific IP. +func AddNetworks(ctx context.Context, d *dockerutil.Container, addr net.IP, networks []*dockerutil.Network) error { + for _, dn := range networks { + ip := AddressInSubnet(addr, *dn.Subnet) + // Connect to the network with the specified IP address. + if err := dn.Connect(ctx, d, ip.String(), ""); err != nil { + return fmt.Errorf("unable to connect container %s to network %s: %w", d.Name, dn.Name, err) + } + } + return nil +} + +// AddressInSubnet combines the subnet provided with the address and returns a +// new address. The return address bits come from the subnet where the mask is 1 +// and from the ip address where the mask is 0. +func AddressInSubnet(addr net.IP, subnet net.IPNet) net.IP { + var octets []byte + for i := 0; i < 4; i++ { + octets = append(octets, (subnet.IP.To4()[i]&subnet.Mask[i])+(addr.To4()[i]&(^subnet.Mask[i]))) + } + return net.IP(octets) +} + +// deviceByIP finds a deviceInfo and device name from an IP address. +func deviceByIP(ctx context.Context, d *dockerutil.Container, ip net.IP) (string, netdevs.DeviceInfo, error) { + out, err := d.Exec(ctx, dockerutil.ExecOpts{}, "ip", "addr", "show") + if err != nil { + return "", netdevs.DeviceInfo{}, fmt.Errorf("listing devices on %s container: %w\n%s", d.Name, err, out) + } + devs, err := netdevs.ParseDevices(out) + if err != nil { + return "", netdevs.DeviceInfo{}, fmt.Errorf("parsing devices from %s container: %w\n%s", d.Name, err, out) + } + testDevice, deviceInfo, err := netdevs.FindDeviceByIP(ip, devs) + if err != nil { + return "", netdevs.DeviceInfo{}, fmt.Errorf("can't find deviceInfo for container %s: %w", d.Name, err) + } + return testDevice, deviceInfo, nil +} + +// createDockerNetwork makes a randomly-named network that will start with the +// namePrefix. The network will be a random /24 subnet. +func createDockerNetwork(ctx context.Context, n *dockerutil.Network) error { + randSource := rand.NewSource(time.Now().UnixNano()) + r1 := rand.New(randSource) + // Class C, 192.0.0.0 to 223.255.255.255, transitionally has mask 24. + ip := net.IPv4(byte(r1.Intn(224-192)+192), byte(r1.Intn(256)), byte(r1.Intn(256)), 0) + n.Subnet = &net.IPNet{ + IP: ip, + Mask: ip.DefaultMask(), + } + return n.Create(ctx) +} + +// StartContainer will create a container instance from runOpts, connect it +// with the specified docker networks and start executing the specified cmd. +func StartContainer(ctx context.Context, runOpts dockerutil.RunOpts, c *dockerutil.Container, containerAddr net.IP, ns []*dockerutil.Network, cmd ...string) error { + conf, hostconf, netconf := c.ConfigsFrom(runOpts, cmd...) + _ = netconf + hostconf.AutoRemove = true + hostconf.Sysctls = map[string]string{"net.ipv6.conf.all.disable_ipv6": "0"} + + if err := c.CreateFrom(ctx, conf, hostconf, nil); err != nil { + return fmt.Errorf("unable to create container %s: %w", c.Name, err) + } + + if err := AddNetworks(ctx, c, containerAddr, ns); err != nil { + return fmt.Errorf("unable to connect the container with the networks: %w", err) + } + + if err := c.Start(ctx); err != nil { + return fmt.Errorf("unable to start container %s: %w", c.Name, err) + } + return nil +} diff --git a/test/packetimpact/runner/packetimpact_test.go b/test/packetimpact/runner/packetimpact_test.go index cb9bfd5b7..c598bfc29 100644 --- a/test/packetimpact/runner/packetimpact_test.go +++ b/test/packetimpact/runner/packetimpact_test.go @@ -18,366 +18,15 @@ package packetimpact_test import ( "context" "flag" - "fmt" - "io/ioutil" - "log" - "math/rand" - "net" - "os" - "os/exec" - "path" - "strings" "testing" - "time" - "github.com/docker/docker/api/types/mount" - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/packetimpact/netdevs" + "gvisor.dev/gvisor/test/packetimpact/runner" ) -// stringList implements flag.Value. -type stringList []string - -// String implements flag.Value.String. -func (l *stringList) String() string { - return strings.Join(*l, ",") -} - -// Set implements flag.Value.Set. -func (l *stringList) Set(value string) error { - *l = append(*l, value) - return nil -} - -var ( - native = flag.Bool("native", false, "whether the test should be run natively") - testbenchBinary = flag.String("testbench_binary", "", "path to the testbench binary") - tshark = flag.Bool("tshark", false, "use more verbose tshark in logs instead of tcpdump") - extraTestArgs = stringList{} - expectFailure = flag.Bool("expect_failure", false, "expect that the test will fail when run") - - dutAddr = net.IPv4(0, 0, 0, 10) - testbenchAddr = net.IPv4(0, 0, 0, 20) -) - -const ctrlPort = "40000" - -// logger implements testutil.Logger. -// -// Labels logs based on their source and formats multi-line logs. -type logger string - -// Name implements testutil.Logger.Name. -func (l logger) Name() string { - return string(l) -} - -// Logf implements testutil.Logger.Logf. -func (l logger) Logf(format string, args ...interface{}) { - lines := strings.Split(fmt.Sprintf(format, args...), "\n") - log.Printf("%s: %s", l, lines[0]) - for _, line := range lines[1:] { - log.Printf("%*s %s", len(l), "", line) - } +func init() { + runner.RegisterFlags(flag.CommandLine) } func TestOne(t *testing.T) { - flag.Var(&extraTestArgs, "extra_test_arg", "extra arguments to pass to the testbench") - flag.Parse() - if *testbenchBinary == "" { - t.Fatal("--testbench_binary is missing") - } - dockerutil.EnsureSupportedDockerVersion() - ctx := context.Background() - - // Create the networks needed for the test. One control network is needed for - // the gRPC control packets and one test network on which to transmit the test - // packets. - ctrlNet := dockerutil.NewNetwork(ctx, logger("ctrlNet")) - testNet := dockerutil.NewNetwork(ctx, logger("testNet")) - for _, dn := range []*dockerutil.Network{ctrlNet, testNet} { - for { - if err := createDockerNetwork(ctx, dn); err != nil { - t.Log("creating docker network:", err) - const wait = 100 * time.Millisecond - t.Logf("sleeping %s and will try creating docker network again", wait) - // This can fail if another docker network claimed the same IP so we'll - // just try again. - time.Sleep(wait) - continue - } - break - } - defer func(dn *dockerutil.Network) { - if err := dn.Cleanup(ctx); err != nil { - t.Errorf("unable to cleanup container %s: %s", dn.Name, err) - } - }(dn) - // Sanity check. - inspect, err := dn.Inspect(ctx) - if err != nil { - t.Fatalf("failed to inspect network %s: %v", dn.Name, err) - } else if inspect.Name != dn.Name { - t.Fatalf("name mismatch for network want: %s got: %s", dn.Name, inspect.Name) - } - - } - - tmpDir, err := ioutil.TempDir("", "container-output") - if err != nil { - t.Fatal("creating temp dir:", err) - } - defer os.RemoveAll(tmpDir) - - const testOutputDir = "/tmp/testoutput" - - // Create the Docker container for the DUT. - var dut *dockerutil.Container - if *native { - dut = dockerutil.MakeNativeContainer(ctx, logger("dut")) - } else { - dut = dockerutil.MakeContainer(ctx, logger("dut")) - } - - runOpts := dockerutil.RunOpts{ - Image: "packetimpact", - CapAdd: []string{"NET_ADMIN"}, - Mounts: []mount.Mount{mount.Mount{ - Type: mount.TypeBind, - Source: tmpDir, - Target: testOutputDir, - ReadOnly: false, - }}, - } - - const containerPosixServerBinary = "/packetimpact/posix_server" - dut.CopyFiles(&runOpts, "/packetimpact", "/test/packetimpact/dut/posix_server") - - conf, hostconf, _ := dut.ConfigsFrom(runOpts, containerPosixServerBinary, "--ip=0.0.0.0", "--port="+ctrlPort) - hostconf.AutoRemove = true - hostconf.Sysctls = map[string]string{"net.ipv6.conf.all.disable_ipv6": "0"} - - if err := dut.CreateFrom(ctx, conf, hostconf, nil); err != nil { - t.Fatalf("unable to create container %s: %v", dut.Name, err) - } - - defer dut.CleanUp(ctx) - - // Add ctrlNet as eth1 and testNet as eth2. - const testNetDev = "eth2" - if err := addNetworks(ctx, dut, dutAddr, []*dockerutil.Network{ctrlNet, testNet}); err != nil { - t.Fatal(err) - } - - if err := dut.Start(ctx); err != nil { - t.Fatalf("unable to start container %s: %s", dut.Name, err) - } - - if _, err := dut.WaitForOutput(ctx, "Server listening.*\n", 60*time.Second); err != nil { - t.Fatalf("%s on container %s never listened: %s", containerPosixServerBinary, dut.Name, err) - } - - dutTestDevice, dutDeviceInfo, err := deviceByIP(ctx, dut, addressInSubnet(dutAddr, *testNet.Subnet)) - if err != nil { - t.Fatal(err) - } - - remoteMAC := dutDeviceInfo.MAC - remoteIPv6 := dutDeviceInfo.IPv6Addr - // Netstack as DUT doesn't assign IPv6 addresses automatically so do it if - // needed. - if remoteIPv6 == nil { - if _, err := dut.Exec(ctx, dockerutil.ExecOpts{}, "ip", "addr", "add", netdevs.MACToIP(remoteMAC).String(), "scope", "link", "dev", dutTestDevice); err != nil { - t.Fatalf("unable to ip addr add on container %s: %s", dut.Name, err) - } - // Now try again, to make sure that it worked. - _, dutDeviceInfo, err = deviceByIP(ctx, dut, addressInSubnet(dutAddr, *testNet.Subnet)) - if err != nil { - t.Fatal(err) - } - remoteIPv6 = dutDeviceInfo.IPv6Addr - if remoteIPv6 == nil { - t.Fatal("unable to set IPv6 address on container", dut.Name) - } - } - - // Create the Docker container for the testbench. - testbench := dockerutil.MakeNativeContainer(ctx, logger("testbench")) - - tbb := path.Base(*testbenchBinary) - containerTestbenchBinary := "/packetimpact/" + tbb - runOpts = dockerutil.RunOpts{ - Image: "packetimpact", - CapAdd: []string{"NET_ADMIN"}, - Mounts: []mount.Mount{mount.Mount{ - Type: mount.TypeBind, - Source: tmpDir, - Target: testOutputDir, - ReadOnly: false, - }}, - } - testbench.CopyFiles(&runOpts, "/packetimpact", "/test/packetimpact/tests/"+tbb) - - // Run tcpdump in the test bench unbuffered, without DNS resolution, just on - // the interface with the test packets. - snifferArgs := []string{ - "tcpdump", - "-S", "-vvv", "-U", "-n", - "-i", testNetDev, - "-w", testOutputDir + "/dump.pcap", - } - snifferRegex := "tcpdump: listening.*\n" - if *tshark { - // Run tshark in the test bench unbuffered, without DNS resolution, just on - // the interface with the test packets. - snifferArgs = []string{ - "tshark", "-V", "-l", "-n", "-i", testNetDev, - "-o", "tcp.check_checksum:TRUE", - "-o", "udp.check_checksum:TRUE", - } - snifferRegex = "Capturing on.*\n" - } - - defer func() { - if err := exec.Command("/bin/cp", "-r", tmpDir, os.Getenv("TEST_UNDECLARED_OUTPUTS_DIR")).Run(); err != nil { - t.Error("unable to copy container output files:", err) - } - }() - - conf, hostconf, _ = testbench.ConfigsFrom(runOpts, snifferArgs...) - hostconf.AutoRemove = true - hostconf.Sysctls = map[string]string{"net.ipv6.conf.all.disable_ipv6": "0"} - - if err := testbench.CreateFrom(ctx, conf, hostconf, nil); err != nil { - t.Fatalf("unable to create container %s: %s", testbench.Name, err) - } - defer testbench.CleanUp(ctx) - - // Add ctrlNet as eth1 and testNet as eth2. - if err := addNetworks(ctx, testbench, testbenchAddr, []*dockerutil.Network{ctrlNet, testNet}); err != nil { - t.Fatal(err) - } - - if err := testbench.Start(ctx); err != nil { - t.Fatalf("unable to start container %s: %s", testbench.Name, err) - } - - // Kill so that it will flush output. - defer func() { - time.Sleep(1 * time.Second) - testbench.Exec(ctx, dockerutil.ExecOpts{}, "killall", snifferArgs[0]) - }() - - if _, err := testbench.WaitForOutput(ctx, snifferRegex, 60*time.Second); err != nil { - t.Fatalf("sniffer on %s never listened: %s", dut.Name, err) - } - - // Because the Linux kernel receives the SYN-ACK but didn't send the SYN it - // will issue an RST. To prevent this IPtables can be used to filter out all - // incoming packets. The raw socket that packetimpact tests use will still see - // everything. - for _, bin := range []string{"iptables", "ip6tables"} { - if logs, err := testbench.Exec(ctx, dockerutil.ExecOpts{}, bin, "-A", "INPUT", "-i", testNetDev, "-p", "tcp", "-j", "DROP"); err != nil { - t.Fatalf("unable to Exec %s on container %s: %s, logs from testbench:\n%s", bin, testbench.Name, err, logs) - } - } - - // FIXME(b/156449515): Some piece of the system has a race. The old - // bash script version had a sleep, so we have one too. The race should - // be fixed and this sleep removed. - time.Sleep(time.Second) - - // Start a packetimpact test on the test bench. The packetimpact test sends - // and receives packets and also sends POSIX socket commands to the - // posix_server to be executed on the DUT. - testArgs := []string{containerTestbenchBinary} - testArgs = append(testArgs, extraTestArgs...) - testArgs = append(testArgs, - "--posix_server_ip", addressInSubnet(dutAddr, *ctrlNet.Subnet).String(), - "--posix_server_port", ctrlPort, - "--remote_ipv4", addressInSubnet(dutAddr, *testNet.Subnet).String(), - "--local_ipv4", addressInSubnet(testbenchAddr, *testNet.Subnet).String(), - "--remote_ipv6", remoteIPv6.String(), - "--remote_mac", remoteMAC.String(), - "--remote_interface_id", fmt.Sprintf("%d", dutDeviceInfo.ID), - "--device", testNetDev, - fmt.Sprintf("--native=%t", *native), - ) - testbenchLogs, err := testbench.Exec(ctx, dockerutil.ExecOpts{}, testArgs...) - if (err != nil) != *expectFailure { - var dutLogs string - if logs, err := dut.Logs(ctx); err != nil { - dutLogs = fmt.Sprintf("failed to fetch DUT logs: %s", err) - } else { - dutLogs = logs - } - - t.Errorf(`test error: %v, expect failure: %t - -====== Begin of DUT Logs ====== - -%s - -====== End of DUT Logs ====== - -====== Begin of Testbench Logs ====== - -%s - -====== End of Testbench Logs ======`, - err, *expectFailure, dutLogs, testbenchLogs) - } -} - -func addNetworks(ctx context.Context, d *dockerutil.Container, addr net.IP, networks []*dockerutil.Network) error { - for _, dn := range networks { - ip := addressInSubnet(addr, *dn.Subnet) - // Connect to the network with the specified IP address. - if err := dn.Connect(ctx, d, ip.String(), ""); err != nil { - return fmt.Errorf("unable to connect container %s to network %s: %w", d.Name, dn.Name, err) - } - } - return nil -} - -// addressInSubnet combines the subnet provided with the address and returns a -// new address. The return address bits come from the subnet where the mask is 1 -// and from the ip address where the mask is 0. -func addressInSubnet(addr net.IP, subnet net.IPNet) net.IP { - var octets []byte - for i := 0; i < 4; i++ { - octets = append(octets, (subnet.IP.To4()[i]&subnet.Mask[i])+(addr.To4()[i]&(^subnet.Mask[i]))) - } - return net.IP(octets) -} - -// createDockerNetwork makes a randomly-named network that will start with the -// namePrefix. The network will be a random /24 subnet. -func createDockerNetwork(ctx context.Context, n *dockerutil.Network) error { - randSource := rand.NewSource(time.Now().UnixNano()) - r1 := rand.New(randSource) - // Class C, 192.0.0.0 to 223.255.255.255, transitionally has mask 24. - ip := net.IPv4(byte(r1.Intn(224-192)+192), byte(r1.Intn(256)), byte(r1.Intn(256)), 0) - n.Subnet = &net.IPNet{ - IP: ip, - Mask: ip.DefaultMask(), - } - return n.Create(ctx) -} - -// deviceByIP finds a deviceInfo and device name from an IP address. -func deviceByIP(ctx context.Context, d *dockerutil.Container, ip net.IP) (string, netdevs.DeviceInfo, error) { - out, err := d.Exec(ctx, dockerutil.ExecOpts{}, "ip", "addr", "show") - if err != nil { - return "", netdevs.DeviceInfo{}, fmt.Errorf("listing devices on %s container: %w\n%s", d.Name, err, out) - } - devs, err := netdevs.ParseDevices(out) - if err != nil { - return "", netdevs.DeviceInfo{}, fmt.Errorf("parsing devices from %s container: %w\n%s", d.Name, err, out) - } - testDevice, deviceInfo, err := netdevs.FindDeviceByIP(ip, devs) - if err != nil { - return "", netdevs.DeviceInfo{}, fmt.Errorf("can't find deviceInfo for container %s: %w", d.Name, err) - } - return testDevice, deviceInfo, nil + runner.TestWithDUT(context.Background(), t, runner.NewDockerDUT, runner.DutAddr) } |