diff options
Diffstat (limited to 'benchmarks')
37 files changed, 587 insertions, 343 deletions
diff --git a/benchmarks/README.md b/benchmarks/README.md index ad44cd6ac..ff21614c5 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -6,66 +6,55 @@ These scripts are tools for collecting performance data for Docker-based tests. The scripts assume the following: -* You have a local machine with bazel installed. -* You have some machine(s) with docker installed. These machines will be - refered to as the "Environment". -* Environment machines have the runtime(s) under test installed, such that you - can run docker with a command like: `docker run --runtime=$RUNTIME - your/image`. -* You are able to login to machines in the environment with the local machine - via ssh and the user for ssh can run docker commands without using `sudo`. +* There are two sets of machines: one where the scripts will be run + (controller) and one or more machines on which docker containers will be run + (environment). +* The controller machine must have bazel installed along with this source + code. You should be able to run a command like `bazel run :benchmarks -- + --list` +* Environment machines must have docker and the required runtimes installed. + More specifically, you should be able to run a command like: `docker run + --runtime=$RUNTIME your/image`. +* The controller has ssh private key which can be used to login to environment + machines and run docker commands without using `sudo`. This is not required + if running locally via the `run-local` command. * The docker daemon on each of your environment machines is listening on `unix:///var/run/docker.sock` (docker's default). For configuring the environment manually, consult the [dockerd documentation][dockerd]. -## Environment - -All benchmarks require a user defined yaml file describe the environment. These -files are of the form: - -```yaml -machine1: local -machine2: - hostname: 100.100.100.100 - username: username - key_path: ~/private_keyfile - key_password: passphrase -machine3: - hostname: 100.100.100.101 - username: username - key_path: ~/private_keyfile - key_password: passphrase -``` +## Running benchmarks -The yaml file defines an environment with three machines named `machine1`, -`machine2` and `machine3`. `machine1` is the local machine, `machine2` and -`machine3` are remote machines. Both `machine2` and `machine3` should be -reachable by `ssh`. For example, the command `ssh -i ~/private_keyfile -username@100.100.100.100` (using the passphrase `passphrase`) should connect to -`machine2`. +Run the following from the benchmarks directory: -The above is an example only. Machines should be uniform, since they are treated -as such by the tests. Machines must also be accessible to each other via their -default routes. Furthermore, some benchmarks will meaningless if running on the -local machine, such as density. +```bash +bazel run :benchmarks -- run-local startup -For remote machines, `hostname`, `key_path`, and `username` are required and -others are optional. In addition key files must be generated -[using the instrcutions below](#generating-ssh-keys). +... +method,metric,result +startup.empty,startup_time_ms,652.5772 +startup.node,startup_time_ms,1654.4042000000002 +startup.ruby,startup_time_ms,1429.835 +``` -The above yaml file can be checked for correctness with the `validate` command -in the top level perf.py script: +The above command ran the startup benchmark locally, which consists of three +benchmarks (empty, node, and ruby). Benchmark tools ran it on the default +runtime, runc. Running on another installed runtime, like say runsc, is as +simple as: -`bazel run :benchmarks -- validate $PWD/examples/localhost.yaml` +```bash +bazel run :benchmakrs -- run-local startup --runtime=runsc +``` -## Running benchmarks +There is help: ``bash bash bazel run :benchmarks -- --help bazel +run :benchmarks -- run-local --help` `` To list available benchmarks, use the `list` commmand: ```bash bazel run :benchmarks -- list +ls ... Benchmark: sysbench.cpu @@ -75,24 +64,44 @@ Metrics: events_per_second :param max_prime: The maximum prime number to search. ``` -To run benchmarks, use the `run` command. For example, to run the sysbench -benchmark above: +You can choose benchmarks by name or regex like: ```bash -bazel run :benchmarks -- run --env $PWD/examples/localhost.yaml sysbench.cpu +bazel run :benchmarks -- run-local startup.node +... +metric,result +startup_time_ms,1671.7178000000001 + +``` + +or + +```bash +bazel run :benchmarks -- run-local s +... +method,metric,result +startup.empty,startup_time_ms,1792.8292 +startup.node,startup_time_ms,3113.5274 +startup.ruby,startup_time_ms,3025.2424 +sysbench.cpu,cpu_events_per_second,12661.47 +sysbench.memory,memory_ops_per_second,7228268.44 +sysbench.mutex,mutex_time,17.4835 +sysbench.mutex,mutex_latency,3496.7 +sysbench.mutex,mutex_deviation,0.04 +syscall.syscall,syscall_time_ns,2065.0 ``` You can run parameterized benchmarks, for example to run with different runtimes: ```bash -bazel run :benchmarks -- run --env $PWD/examples/localhost.yaml --runtime=runc --runtime=runsc sysbench.cpu +bazel run :benchmarks -- run-local --runtime=runc --runtime=runsc sysbench.cpu ``` Or with different parameters: ```bash -bazel run :benchmarks -- run --env $PWD/examples/localhost.yaml --max_prime=10 --max_prime=100 sysbench.cpu +bazel run :benchmarks -- run-local --max_prime=10 --max_prime=100 sysbench.cpu ``` ## Writing benchmarks @@ -121,7 +130,7 @@ The harness requires workloads to run. These are all available in the In general, a workload consists of a Dockerfile to build it (while these are not hermetic, in general they should be as fixed and isolated as possible), some -parses for output if required, parser tests and sample data. Provided the test +parsers for output if required, parser tests and sample data. Provided the test is named after the workload package and contains a function named `sample`, this variable will be used to automatically mock workload output when the `--mock` flag is provided to the main tool. @@ -149,24 +158,5 @@ To write a new benchmark, open a module in the `suites` directory and use the above signature. You should add a descriptive doc string to describe what your benchmark is and any test centric arguments. -## Generating SSH Keys - -The scripts only support RSA Keys, and ssh library used in paramiko. Paramiko -only supports RSA keys that look like the following (PEM format): - -```bash -$ cat /path/to/ssh/key - ------BEGIN RSA PRIVATE KEY----- -...private key text... ------END RSA PRIVATE KEY----- - -``` - -To generate ssh keys in PEM format, use the [`-t rsa -m PEM -b 4096`][RSA-keys]. -option. - [dockerd]: https://docs.docker.com/engine/reference/commandline/dockerd/ [docker-py]: https://docker-py.readthedocs.io/en/stable/ -[paramiko]: http://docs.paramiko.org/en/2.4/api/client.html -[RSA-keys]: https://serverfault.com/questions/939909/ssh-keygen-does-not-create-rsa-private-key diff --git a/benchmarks/defs.bzl b/benchmarks/defs.bzl deleted file mode 100644 index 79e6cdbc8..000000000 --- a/benchmarks/defs.bzl +++ /dev/null @@ -1,18 +0,0 @@ -"""Provides python helper functions.""" - -load("@pydeps//:requirements.bzl", _requirement = "requirement") - -def filter_deps(deps = None): - if deps == None: - deps = [] - return [dep for dep in deps if dep] - -def py_library(deps = None, **kwargs): - return native.py_library(deps = filter_deps(deps), **kwargs) - -def py_test(deps = None, **kwargs): - return native.py_test(deps = filter_deps(deps), **kwargs) - -def requirement(name, direct = True): - """ requirement returns the required dependency. """ - return _requirement(name) diff --git a/benchmarks/harness/BUILD b/benchmarks/harness/BUILD index 081a74243..52d4e42f8 100644 --- a/benchmarks/harness/BUILD +++ b/benchmarks/harness/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "requirement") +load("//tools:defs.bzl", "py_library", "py_requirement") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -25,16 +25,16 @@ py_library( srcs = ["container.py"], deps = [ "//benchmarks/workloads", - requirement("asn1crypto", False), - requirement("chardet", False), - requirement("certifi", False), - requirement("docker", True), - requirement("docker-pycreds", False), - requirement("idna", False), - requirement("ptyprocess", False), - requirement("requests", False), - requirement("urllib3", False), - requirement("websocket-client", False), + py_requirement("asn1crypto", False), + py_requirement("chardet", False), + py_requirement("certifi", False), + py_requirement("docker", True), + py_requirement("docker-pycreds", False), + py_requirement("idna", False), + py_requirement("ptyprocess", False), + py_requirement("requests", False), + py_requirement("urllib3", False), + py_requirement("websocket-client", False), ], ) @@ -47,17 +47,17 @@ py_library( "//benchmarks/harness:ssh_connection", "//benchmarks/harness:tunnel_dispatcher", "//benchmarks/harness/machine_mocks", - requirement("asn1crypto", False), - requirement("chardet", False), - requirement("certifi", False), - requirement("docker", True), - requirement("docker-pycreds", False), - requirement("idna", False), - requirement("ptyprocess", False), - requirement("requests", False), - requirement("six", False), - requirement("urllib3", False), - requirement("websocket-client", False), + py_requirement("asn1crypto", False), + py_requirement("chardet", False), + py_requirement("certifi", False), + py_requirement("docker", True), + py_requirement("docker-pycreds", False), + py_requirement("idna", False), + py_requirement("ptyprocess", False), + py_requirement("requests", False), + py_requirement("six", False), + py_requirement("urllib3", False), + py_requirement("websocket-client", False), ], ) @@ -66,10 +66,10 @@ py_library( srcs = ["ssh_connection.py"], deps = [ "//benchmarks/harness", - requirement("bcrypt", False), - requirement("cffi", True), - requirement("paramiko", True), - requirement("cryptography", False), + py_requirement("bcrypt", False), + py_requirement("cffi", True), + py_requirement("paramiko", True), + py_requirement("cryptography", False), ], ) @@ -77,16 +77,16 @@ py_library( name = "tunnel_dispatcher", srcs = ["tunnel_dispatcher.py"], deps = [ - requirement("asn1crypto", False), - requirement("chardet", False), - requirement("certifi", False), - requirement("docker", True), - requirement("docker-pycreds", False), - requirement("idna", False), - requirement("pexpect", True), - requirement("ptyprocess", False), - requirement("requests", False), - requirement("urllib3", False), - requirement("websocket-client", False), + py_requirement("asn1crypto", False), + py_requirement("chardet", False), + py_requirement("certifi", False), + py_requirement("docker", True), + py_requirement("docker-pycreds", False), + py_requirement("idna", False), + py_requirement("pexpect", True), + py_requirement("ptyprocess", False), + py_requirement("requests", False), + py_requirement("urllib3", False), + py_requirement("websocket-client", False), ], ) diff --git a/benchmarks/harness/__init__.py b/benchmarks/harness/__init__.py index a7f34da9e..61fd25f73 100644 --- a/benchmarks/harness/__init__.py +++ b/benchmarks/harness/__init__.py @@ -13,13 +13,20 @@ # limitations under the License. """Core benchmark utilities.""" +import getpass import os # LOCAL_WORKLOADS_PATH defines the path to use for local workloads. This is a # format string that accepts a single string parameter. LOCAL_WORKLOADS_PATH = os.path.join( - os.path.dirname(__file__), "../workloads/{}") + os.path.dirname(__file__), "../workloads/{}/tar.tar") # REMOTE_WORKLOADS_PATH defines the path to use for storing the workloads on the # remote host. This is a format string that accepts a single string parameter. REMOTE_WORKLOADS_PATH = "workloads/{}" + +# DEFAULT_USER is the default user running this script. +DEFAULT_USER = getpass.getuser() + +# DEFAULT_USER_HOME is the home directory of the user running the script. +DEFAULT_USER_HOME = os.environ["HOME"] if "HOME" in os.environ else "" diff --git a/benchmarks/harness/machine.py b/benchmarks/harness/machine.py index 66b719b63..2df4c9e31 100644 --- a/benchmarks/harness/machine.py +++ b/benchmarks/harness/machine.py @@ -160,15 +160,17 @@ class LocalMachine(Machine): stdout, stderr = process.communicate() return stdout.decode("utf-8"), stderr.decode("utf-8") - def read(self, path: str) -> str: + def read(self, path: str) -> bytes: # Read the exact path locally. return open(path, "r").read() def pull(self, workload: str) -> str: # Run the docker build command locally. logging.info("Building %s@%s locally...", workload, self._name) - self.run("docker build --tag={} {}".format( - workload, harness.LOCAL_WORKLOADS_PATH.format(workload))) + with open(harness.LOCAL_WORKLOADS_PATH.format(workload), + "rb") as dockerfile: + self._docker_client.images.build( + fileobj=dockerfile, tag=workload, custom_context=True) return workload # Workload is the tag. def container(self, image: str, **kwargs) -> container.Container: @@ -212,6 +214,9 @@ class RemoteMachine(Machine): # Push to the remote machine and build. logging.info("Building %s@%s remotely...", workload, self._name) remote_path = self._ssh_connection.send_workload(workload) + # Workloads are all tarballs. + self.run("tar -xvf {remote_path}/tar.tar -C {remote_path}".format( + remote_path=remote_path)) self.run("docker build --tag={} {}".format(workload, remote_path)) return workload # Workload is the tag. diff --git a/benchmarks/harness/machine_producers/BUILD b/benchmarks/harness/machine_producers/BUILD index c4e943882..48ea0ef39 100644 --- a/benchmarks/harness/machine_producers/BUILD +++ b/benchmarks/harness/machine_producers/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "requirement") +load("//tools:defs.bzl", "py_library", "py_requirement") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -31,7 +31,7 @@ py_library( deps = [ "//benchmarks/harness:machine", "//benchmarks/harness/machine_producers:machine_producer", - requirement("PyYAML", False), + py_requirement("PyYAML", False), ], ) diff --git a/benchmarks/harness/machine_producers/gcloud_producer.py b/benchmarks/harness/machine_producers/gcloud_producer.py index 4693dd8a2..e0b77d52b 100644 --- a/benchmarks/harness/machine_producers/gcloud_producer.py +++ b/benchmarks/harness/machine_producers/gcloud_producer.py @@ -29,7 +29,6 @@ collisions with user instances shouldn't happen. producer.release_machines(NUM_MACHINES) """ import datetime -import getpass import json import subprocess import threading @@ -40,8 +39,6 @@ from benchmarks.harness import machine from benchmarks.harness.machine_producers import gcloud_mock_recorder from benchmarks.harness.machine_producers import machine_producer -DEFAULT_USER = getpass.getuser() - class GCloudProducer(machine_producer.MachineProducer): """Implementation of MachineProducer backed by GCP. @@ -50,9 +47,10 @@ class GCloudProducer(machine_producer.MachineProducer): Attributes: project: The GCP project name under which to create the machines. - ssh_key_path: path to a valid ssh key. See README on vaild ssh keys. + ssh_key_file: path to a valid ssh private key. See README on vaild ssh keys. image: image name as a string. image_project: image project as a string. + machine_type: type of GCP to create. e.g. n1-standard-4 zone: string to a valid GCP zone. ssh_user: string of user name for ssh_key ssh_password: string of password for ssh key @@ -63,18 +61,22 @@ class GCloudProducer(machine_producer.MachineProducer): def __init__(self, project: str, - ssh_key_path: str, + ssh_key_file: str, image: str, image_project: str, + machine_type: str, zone: str, ssh_user: str, + ssh_password: str, mock: gcloud_mock_recorder.MockPrinter = None): self.project = project - self.ssh_key_path = ssh_key_path + self.ssh_key_file = ssh_key_file self.image = image self.image_project = image_project + self.machine_type = machine_type self.zone = zone - self.ssh_user = ssh_user if ssh_user else DEFAULT_USER + self.ssh_user = ssh_user + self.ssh_password = ssh_password self.mock = mock self.condition = threading.Condition() @@ -86,20 +88,19 @@ class GCloudProducer(machine_producer.MachineProducer): with self.condition: names = self._get_unique_names(num_machines) self._build_instances(names) - instances = self._start_command(names) - self._add_ssh_key_to_instances(names) - return self._machines_from_instances(instances) + instances = self._start_command(names) + self._add_ssh_key_to_instances(names) + return self._machines_from_instances(instances) def release_machines(self, machine_list: List[machine.Machine]): """Releases the requested number of machines, deleting the instances.""" if not machine_list: return - with self.condition: - cmd = "gcloud compute instances delete --quiet".split(" ") - names = [str(m) for m in machine_list] - cmd.extend(names) - cmd.append("--zone={zone}".format(zone=self.zone)) - self._run_command(cmd) + cmd = "gcloud compute instances delete --quiet".split(" ") + names = [str(m) for m in machine_list] + cmd.extend(names) + cmd.append("--zone={zone}".format(zone=self.zone)) + self._run_command(cmd, detach=True) def _machines_from_instances( self, instances: List[Dict[str, Any]]) -> List[machine.Machine]: @@ -111,9 +112,11 @@ class GCloudProducer(machine_producer.MachineProducer): "hostname": instance["networkInterfaces"][0]["accessConfigs"][0]["natIP"], "key_path": - self.ssh_key_path, + self.ssh_key_file, "username": - self.ssh_user + self.ssh_user, + "key_password": + self.ssh_password } machines.append(machine.RemoteMachine(name=name, **kwargs)) return machines @@ -148,12 +151,15 @@ class GCloudProducer(machine_producer.MachineProducer): "_build_instances cannot create instances without names.") cmd = "gcloud compute instances create".split(" ") cmd.extend(names) - cmd.extend("--preemptible --image={image} --zone={zone}".format( - image=self.image, zone=self.zone).split(" ")) + cmd.extend( + "--preemptible --image={image} --zone={zone} --machine-type={machine_type}" + .format( + image=self.image, zone=self.zone, + machine_type=self.machine_type).split(" ")) if self.image_project: cmd.append("--image-project={project}".format(project=self.image_project)) - res = self._run_command(cmd) - return json.loads(res.stdout) + res = self._run_command(cmd) + return json.loads(res.stdout) def _start_command(self, names): """Starts instances using gcloud command. @@ -184,7 +190,7 @@ class GCloudProducer(machine_producer.MachineProducer): Args: names: list of machine names to which to add the ssh-key - self.ssh_key_path. + self.ssh_key_file. Raises: subprocess.CalledProcessError: when underlying subprocess call returns an @@ -193,7 +199,7 @@ class GCloudProducer(machine_producer.MachineProducer): """ for name in names: cmd = "gcloud compute ssh {name}".format(name=name).split(" ") - cmd.append("--ssh-key-file={key}".format(key=self.ssh_key_path)) + cmd.append("--ssh-key-file={key}".format(key=self.ssh_key_file)) cmd.append("--zone={zone}".format(zone=self.zone)) cmd.append("--command=uname") timeout = datetime.timedelta(seconds=5 * 60) @@ -221,7 +227,9 @@ class GCloudProducer(machine_producer.MachineProducer): res = self._run_command(cmd) return json.loads(res.stdout) - def _run_command(self, cmd: List[str]) -> subprocess.CompletedProcess: + def _run_command(self, + cmd: List[str], + detach: bool = False) -> [None, subprocess.CompletedProcess]: """Runs command as a subprocess. Runs command as subprocess and returns the result. @@ -230,14 +238,24 @@ class GCloudProducer(machine_producer.MachineProducer): Args: cmd: command to be run as a list of strings. + detach: if True, run the child process and don't wait for it to return. Returns: - Completed process object to be parsed by caller. + Completed process object to be parsed by caller or None if detach=True. Raises: CalledProcessError: if subprocess.run returns an error. """ cmd = cmd + ["--format=json"] + if detach: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if self.mock: + out, _ = p.communicate() + self.mock.record( + subprocess.CompletedProcess( + returncode=p.returncode, stdout=out, args=p.args)) + return + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if self.mock: self.mock.record(res) diff --git a/benchmarks/harness/machine_producers/machine_producer.py b/benchmarks/harness/machine_producers/machine_producer.py index 124ee14cc..f5591c026 100644 --- a/benchmarks/harness/machine_producers/machine_producer.py +++ b/benchmarks/harness/machine_producers/machine_producer.py @@ -13,6 +13,7 @@ # limitations under the License. """Abstract types.""" +import threading from typing import List from benchmarks.harness import machine @@ -28,3 +29,23 @@ class MachineProducer: def release_machines(self, machine_list: List[machine.Machine]): """Releases the given set of machines.""" raise NotImplementedError + + +class LocalMachineProducer(MachineProducer): + """Produces Local Machines.""" + + def __init__(self, limit: int): + self.limit_sem = threading.Semaphore(value=limit) + + def get_machines(self, num_machines: int) -> List[machine.Machine]: + """Returns the request number of MockMachines.""" + + self.limit_sem.acquire() + return [machine.LocalMachine("local") for _ in range(num_machines)] + + def release_machines(self, machine_list: List[machine.MockMachine]): + """No-op.""" + if not machine_list: + raise ValueError("Cannot release an empty list!") + self.limit_sem.release() + machine_list.clear() diff --git a/benchmarks/harness/ssh_connection.py b/benchmarks/harness/ssh_connection.py index fcbfbcdb2..e0bf258f1 100644 --- a/benchmarks/harness/ssh_connection.py +++ b/benchmarks/harness/ssh_connection.py @@ -94,7 +94,7 @@ class SSHConnection: return stdout, stderr def send_workload(self, name: str) -> str: - """Sends a workload to the remote machine. + """Sends a workload tarball to the remote machine. Args: name: The workload name. @@ -103,9 +103,6 @@ class SSHConnection: The remote path. """ with self._client() as client: - for dirpath, _, filenames in os.walk( - harness.LOCAL_WORKLOADS_PATH.format(name)): - for filename in filenames: - send_one_file(client, os.path.join(dirpath, filename), - harness.REMOTE_WORKLOADS_PATH.format(name)) + send_one_file(client, harness.LOCAL_WORKLOADS_PATH.format(name), + harness.REMOTE_WORKLOADS_PATH.format(name)) return harness.REMOTE_WORKLOADS_PATH.format(name) diff --git a/benchmarks/runner/BUILD b/benchmarks/runner/BUILD index de24824cc..fae0ca800 100644 --- a/benchmarks/runner/BUILD +++ b/benchmarks/runner/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "py_library", "py_requirement", "py_test") package(licenses = ["notice"]) @@ -10,7 +10,9 @@ py_library( ], visibility = ["//benchmarks:__pkg__"], deps = [ + ":commands", "//benchmarks/harness:benchmark_driver", + "//benchmarks/harness/machine_producers:machine_producer", "//benchmarks/harness/machine_producers:mock_producer", "//benchmarks/harness/machine_producers:yaml_producer", "//benchmarks/suites", @@ -26,7 +28,15 @@ py_library( "//benchmarks/suites:startup", "//benchmarks/suites:sysbench", "//benchmarks/suites:syscall", - requirement("click", True), + py_requirement("click", True), + ], +) + +py_library( + name = "commands", + srcs = ["commands.py"], + deps = [ + py_requirement("click", True), ], ) @@ -40,14 +50,14 @@ py_test( ], deps = [ ":runner", - requirement("click", True), - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("click", True), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) diff --git a/benchmarks/runner/__init__.py b/benchmarks/runner/__init__.py index 9bf9cfd65..ba80d83d7 100644 --- a/benchmarks/runner/__init__.py +++ b/benchmarks/runner/__init__.py @@ -15,10 +15,13 @@ import copy import csv +import json import logging +import os import pkgutil import pydoc import re +import subprocess import sys import types from typing import List @@ -26,10 +29,14 @@ from typing import Tuple import click +from benchmarks import harness from benchmarks import suites from benchmarks.harness import benchmark_driver +from benchmarks.harness.machine_producers import gcloud_producer +from benchmarks.harness.machine_producers import machine_producer from benchmarks.harness.machine_producers import mock_producer from benchmarks.harness.machine_producers import yaml_producer +from benchmarks.runner import commands @click.group() @@ -100,30 +107,77 @@ def list_all(method): print("\n") -# pylint: disable=too-many-arguments -# pylint: disable=too-many-branches -# pylint: disable=too-many-locals -@runner.command( - context_settings=dict(ignore_unknown_options=True, allow_extra_args=True)) +@runner.command("run-local", commands.LocalCommand) @click.pass_context -@click.argument("method") -@click.option("--mock/--no-mock", default=False, help="Mock the machines.") -@click.option("--env", default=None, help="Specify a yaml file with machines.") -@click.option( - "--runtime", default=["runc"], help="The runtime to use.", multiple=True) -@click.option("--metric", help="The metric to extract.", multiple=True) -@click.option( - "--runs", default=1, help="The number of times to run each benchmark.") -@click.option( - "--stat", - default="median", - help="How to aggregate the data from all runs." - "\nmedian - returns the median of all runs (default)" - "\nall - returns all results comma separated" - "\nmeanstd - returns result as mean,std") -# pylint: disable=too-many-statements -def run(ctx, method: str, runs: int, env: str, mock: bool, runtime: List[str], - metric: List[str], stat: str, **kwargs): +def run_local(ctx, limit: float, **kwargs): + """Runs benchmarks locally.""" + run(ctx, machine_producer.LocalMachineProducer(limit=limit), **kwargs) + + +@runner.command("run-mock", commands.RunCommand) +@click.pass_context +def run_mock(ctx, **kwargs): + """Runs benchmarks on Mock machines. Used for testing.""" + run(ctx, mock_producer.MockMachineProducer(), **kwargs) + + +@runner.command("run-gcp", commands.GCPCommand) +@click.pass_context +def run_gcp(ctx, project: str, ssh_key_file: str, image: str, + image_project: str, machine_type: str, zone: str, ssh_user: str, + ssh_password: str, **kwargs): + """Runs all benchmarks on GCP instances.""" + + if not ssh_user: + ssh_user = harness.DEFAULT_USER + + # Get the default project if one was not provided. + if not project: + sub = subprocess.run( + "gcloud config get-value project".split(" "), stdout=subprocess.PIPE) + if sub.returncode: + raise ValueError( + "Cannot get default project from gcloud. Is it configured>") + project = sub.stdout.decode("utf-8").strip("\n") + + if not image_project: + image_project = project + + # Check that the ssh-key exists and is readable. + if not os.access(ssh_key_file, os.R_OK): + raise ValueError( + "ssh key given `{ssh_key}` is does not exist or is not readable." + .format(ssh_key=ssh_key_file)) + + # Check that the image exists. + sub = subprocess.run( + "gcloud compute images describe {image} --project {image_project} --format=json" + .format(image=image, image_project=image_project).split(" "), + stdout=subprocess.PIPE) + if sub.returncode or "READY" not in json.loads(sub.stdout)["status"]: + raise ValueError( + "given image was not found or is not ready: {image} {image_project}." + .format(image=image, image_project=image_project)) + + # Check and set zone to default. + if not zone: + sub = subprocess.run( + "gcloud config get-value compute/zone".split(" "), + stdout=subprocess.PIPE) + if sub.returncode: + raise ValueError( + "Default zone is not set in gcloud. Set one or pass a zone with the --zone flag." + ) + zone = sub.stdout.decode("utf-8").strip("\n") + + producer = gcloud_producer.GCloudProducer(project, ssh_key_file, image, + image_project, machine_type, zone, + ssh_user, ssh_password) + run(ctx, producer, **kwargs) + + +def run(ctx, producer: machine_producer.MachineProducer, method: str, runs: int, + runtime: List[str], metric: List[str], stat: str, **kwargs): """Runs arbitrary benchmarks. All unknown command line flags are passed through to the underlying benchmark @@ -139,16 +193,13 @@ def run(ctx, method: str, runs: int, env: str, mock: bool, runtime: List[str], All benchmarks are run in parallel where possible, but have exclusive ownership over the individual machines. - Exactly one of the --mock and --env flag must be specified. - Every benchmark method will be run the times indicated by --runs. Args: ctx: Click context. + producer: A Machine Producer from which to get Machines. method: A regular expression for methods to be run. runs: Number of runs. - env: Environment to use. - mock: If true, use mocked environment (supercedes env). runtime: A list of runtimes to test. metric: A list of metrics to extract. stat: The class of statistics to extract. @@ -218,20 +269,6 @@ def run(ctx, method: str, runs: int, env: str, mock: bool, runtime: List[str], sys.exit(1) fold("method", list(methods.keys()), allow_flatten=True) - # Construct the environment. - if mock and env: - # You can't provide both. - logging.error("both --mock and --env are set: which one is it?") - sys.exit(1) - elif mock: - producer = mock_producer.MockMachineProducer() - elif env: - producer = yaml_producer.YamlMachineProducer(env) - else: - # You must provide one of mock or env. - logging.error("no enviroment provided: use --mock or --env.") - sys.exit(1) - # Spin up the drivers. # # We ensure that metric is the last entry, because we have special behavior. diff --git a/benchmarks/runner/commands.py b/benchmarks/runner/commands.py new file mode 100644 index 000000000..7ab12fac6 --- /dev/null +++ b/benchmarks/runner/commands.py @@ -0,0 +1,135 @@ +# python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module with the guts of `click` commands. + +Overrides of the click.core.Command. This is done so flags are inherited between +similar commands (the run command). The classes below are meant to be used in +click templates like so. + +@runner.command("run-mock", RunCommand) +def run_mock(**kwargs): + # mock implementation + +""" +import click + +from benchmarks import harness + + +class RunCommand(click.core.Command): + """Base Run Command with flags. + + Attributes: + method: regex of which suite to choose (e.g. sysbench would run + sysbench.cpu, sysbench.memory, and sysbench.mutex) See list command for + details. + metric: metric(s) to extract. See list command for details. + runtime: the runtime(s) on which to run. + runs: the number of runs to do of each method. + stat: how to compile results in the case of multiple run (e.g. median). + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + method = click.core.Argument(("method",)) + + metric = click.core.Option(("--metric",), + help="The metric to extract.", + multiple=True) + + runtime = click.core.Option(("--runtime",), + default=["runc"], + help="The runtime to use.", + multiple=True) + runs = click.core.Option(("--runs",), + default=1, + help="The number of times to run each benchmark.") + stat = click.core.Option( + ("--stat",), + default="median", + help="How to aggregate the data from all runs." + "\nmedian - returns the median of all runs (default)" + "\nall - returns all results comma separated" + "\nmeanstd - returns result as mean,std") + self.params.extend([method, runtime, runs, stat, metric]) + self.ignore_unknown_options = True + self.allow_extra_args = True + + +class LocalCommand(RunCommand): + """LocalCommand inherits all flags from RunCommand. + + Attributes: + limit: limits the number of machines on which to run benchmarks. This limits + for local how many benchmarks may run at a time. e.g. "startup" requires + one machine -- passing two machines would limit two startup jobs at a + time. Default is infinity. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.params.append( + click.core.Option( + ("--limit",), + default=1, + help="Limit of number of benchmarks that can run at a given time.")) + + +class GCPCommand(RunCommand): + """GCPCommand inherits all flags from RunCommand and adds flags for run_gcp method. + + Attributes: + project: GCP project + ssh_key_path: path to the ssh-key to use for the run + image: name of the image to build machines from + image_project: GCP project under which to find image + zone: a GCP zone (e.g. us-west1-b) + ssh_user: username to use for the ssh-key + ssh_password: password to use for the ssh-key + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + project = click.core.Option( + ("--project",), + help="Project to run on if not default value given by 'gcloud config get-value project'." + ) + ssh_key_path = click.core.Option( + ("--ssh-key-file",), + help="Path to a valid ssh private key to use. See README on generating a valid ssh key. Set to ~/.ssh/benchmark-tools by default.", + default=harness.DEFAULT_USER_HOME + "/.ssh/benchmark-tools") + image = click.core.Option(("--image",), + help="The image on which to build VMs.", + default="bm-tools-testing") + image_project = click.core.Option( + ("--image_project",), + help="The project under which the image to be used is listed.", + default="") + machine_type = click.core.Option(("--machine_type",), + help="Type to make all machines.", + default="n1-standard-4") + zone = click.core.Option(("--zone",), + help="The GCP zone to run on.", + default="") + ssh_user = click.core.Option(("--ssh-user",), + help="User for the ssh key.", + default=harness.DEFAULT_USER) + ssh_password = click.core.Option(("--ssh-password",), + help="Password for the ssh key.", + default="") + self.params.extend([ + project, ssh_key_path, image, image_project, machine_type, zone, + ssh_user, ssh_password + ]) diff --git a/benchmarks/runner/runner_test.py b/benchmarks/runner/runner_test.py index 5719c2838..7818d631a 100644 --- a/benchmarks/runner/runner_test.py +++ b/benchmarks/runner/runner_test.py @@ -49,7 +49,7 @@ def test_list(): def test_run(): cli_runner = testing.CliRunner() - result = cli_runner.invoke(runner.runner, ["run", "--mock", "."]) + result = cli_runner.invoke(runner.runner, ["run-mock", "."]) print(result.output) assert result.exit_code == 0 diff --git a/benchmarks/suites/http.py b/benchmarks/suites/http.py index ea9024e43..6efea938c 100644 --- a/benchmarks/suites/http.py +++ b/benchmarks/suites/http.py @@ -92,7 +92,7 @@ def http_app(server: machine.Machine, redis = server.pull("redis") image = server.pull(workload) redis_port = 6379 - redis_name = "redis_server" + redis_name = "{workload}_redis_server".format(workload=workload) with server.container(redis, name=redis_name).detach(): server.container(server_netcat, links={redis_name: redis_name})\ diff --git a/benchmarks/tcp/BUILD b/benchmarks/tcp/BUILD index 735d7127f..d5e401acc 100644 --- a/benchmarks/tcp/BUILD +++ b/benchmarks/tcp/BUILD @@ -1,5 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary") -load("@rules_cc//cc:defs.bzl", "cc_binary") +load("//tools:defs.bzl", "cc_binary", "go_binary") package(licenses = ["notice"]) diff --git a/benchmarks/tcp/tcp_proxy.go b/benchmarks/tcp/tcp_proxy.go index be0d7bdd6..72ada5700 100644 --- a/benchmarks/tcp/tcp_proxy.go +++ b/benchmarks/tcp/tcp_proxy.go @@ -84,8 +84,8 @@ func (netImpl) printStats() { } const ( - nicID = 1 // Fixed. - rcvBufSize = 1 << 20 // 1MB. + nicID = 1 // Fixed. + bufSize = 4 << 20 // 4MB. ) type netstackImpl struct { @@ -125,9 +125,13 @@ func setupNetwork(ifaceName string, numChannels int) (fds []int, err error) { } // RAW Sockets by default have a very small SO_RCVBUF of 256KB, - // up it to at least 1MB to reduce packet drops. - if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, rcvBufSize); err != nil { - return nil, fmt.Errorf("setsockopt(..., SO_RCVBUF, %v,..) = %v", rcvBufSize, err) + // up it to at least 4MB to reduce packet drops. + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bufSize); err != nil { + return nil, fmt.Errorf("setsockopt(..., SO_RCVBUF, %v,..) = %v", bufSize, err) + } + + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, bufSize); err != nil { + return nil, fmt.Errorf("setsockopt(..., SO_SNDBUF, %v,..) = %v", bufSize, err) } if !*swgso && *gso != 0 { diff --git a/benchmarks/workloads/BUILD b/benchmarks/workloads/BUILD index 643806105..ccb86af5b 100644 --- a/benchmarks/workloads/BUILD +++ b/benchmarks/workloads/BUILD @@ -11,25 +11,25 @@ py_library( filegroup( name = "files", srcs = [ - "//benchmarks/workloads/ab:files", - "//benchmarks/workloads/absl:files", - "//benchmarks/workloads/curl:files", - "//benchmarks/workloads/ffmpeg:files", - "//benchmarks/workloads/fio:files", - "//benchmarks/workloads/httpd:files", - "//benchmarks/workloads/iperf:files", - "//benchmarks/workloads/netcat:files", - "//benchmarks/workloads/nginx:files", - "//benchmarks/workloads/node:files", - "//benchmarks/workloads/node_template:files", - "//benchmarks/workloads/redis:files", - "//benchmarks/workloads/redisbenchmark:files", - "//benchmarks/workloads/ruby:files", - "//benchmarks/workloads/ruby_template:files", - "//benchmarks/workloads/sleep:files", - "//benchmarks/workloads/sysbench:files", - "//benchmarks/workloads/syscall:files", - "//benchmarks/workloads/tensorflow:files", - "//benchmarks/workloads/true:files", + "//benchmarks/workloads/ab:tar", + "//benchmarks/workloads/absl:tar", + "//benchmarks/workloads/curl:tar", + "//benchmarks/workloads/ffmpeg:tar", + "//benchmarks/workloads/fio:tar", + "//benchmarks/workloads/httpd:tar", + "//benchmarks/workloads/iperf:tar", + "//benchmarks/workloads/netcat:tar", + "//benchmarks/workloads/nginx:tar", + "//benchmarks/workloads/node:tar", + "//benchmarks/workloads/node_template:tar", + "//benchmarks/workloads/redis:tar", + "//benchmarks/workloads/redisbenchmark:tar", + "//benchmarks/workloads/ruby:tar", + "//benchmarks/workloads/ruby_template:tar", + "//benchmarks/workloads/sleep:tar", + "//benchmarks/workloads/sysbench:tar", + "//benchmarks/workloads/syscall:tar", + "//benchmarks/workloads/tensorflow:tar", + "//benchmarks/workloads/true:tar", ], ) diff --git a/benchmarks/workloads/ab/BUILD b/benchmarks/workloads/ab/BUILD index e99a8d674..4dd91ceb3 100644 --- a/benchmarks/workloads/ab/BUILD +++ b/benchmarks/workloads/ab/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -16,19 +16,19 @@ py_test( python_version = "PY3", deps = [ ":ab", - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/absl/BUILD b/benchmarks/workloads/absl/BUILD index bb499620e..55dae3baa 100644 --- a/benchmarks/workloads/absl/BUILD +++ b/benchmarks/workloads/absl/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -16,19 +16,19 @@ py_test( python_version = "PY3", deps = [ ":absl", - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/curl/BUILD b/benchmarks/workloads/curl/BUILD index 83f3c71a0..a70873065 100644 --- a/benchmarks/workloads/curl/BUILD +++ b/benchmarks/workloads/curl/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/ffmpeg/BUILD b/benchmarks/workloads/ffmpeg/BUILD index c1f2afc40..7c41ba631 100644 --- a/benchmarks/workloads/ffmpeg/BUILD +++ b/benchmarks/workloads/ffmpeg/BUILD @@ -1,3 +1,5 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], @@ -8,8 +10,8 @@ py_library( srcs = ["__init__.py"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/fio/BUILD b/benchmarks/workloads/fio/BUILD index 7fc96cfa5..7b78e8e75 100644 --- a/benchmarks/workloads/fio/BUILD +++ b/benchmarks/workloads/fio/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -16,19 +16,19 @@ py_test( python_version = "PY3", deps = [ ":fio", - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/httpd/BUILD b/benchmarks/workloads/httpd/BUILD index 83f3c71a0..a70873065 100644 --- a/benchmarks/workloads/httpd/BUILD +++ b/benchmarks/workloads/httpd/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/iperf/BUILD b/benchmarks/workloads/iperf/BUILD index fe0acbfce..570f40148 100644 --- a/benchmarks/workloads/iperf/BUILD +++ b/benchmarks/workloads/iperf/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -16,19 +16,19 @@ py_test( python_version = "PY3", deps = [ ":iperf", - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/netcat/BUILD b/benchmarks/workloads/netcat/BUILD index 83f3c71a0..a70873065 100644 --- a/benchmarks/workloads/netcat/BUILD +++ b/benchmarks/workloads/netcat/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/nginx/BUILD b/benchmarks/workloads/nginx/BUILD index 83f3c71a0..a70873065 100644 --- a/benchmarks/workloads/nginx/BUILD +++ b/benchmarks/workloads/nginx/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/node/BUILD b/benchmarks/workloads/node/BUILD index 59460d02f..bfcf78cf9 100644 --- a/benchmarks/workloads/node/BUILD +++ b/benchmarks/workloads/node/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", "index.js", diff --git a/benchmarks/workloads/node_template/BUILD b/benchmarks/workloads/node_template/BUILD index ae7f121d3..e142f082a 100644 --- a/benchmarks/workloads/node_template/BUILD +++ b/benchmarks/workloads/node_template/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", "index.hbs", diff --git a/benchmarks/workloads/redis/BUILD b/benchmarks/workloads/redis/BUILD index 83f3c71a0..a70873065 100644 --- a/benchmarks/workloads/redis/BUILD +++ b/benchmarks/workloads/redis/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/redisbenchmark/BUILD b/benchmarks/workloads/redisbenchmark/BUILD index d40e75a3a..f472a4443 100644 --- a/benchmarks/workloads/redisbenchmark/BUILD +++ b/benchmarks/workloads/redisbenchmark/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -16,19 +16,19 @@ py_test( python_version = "PY3", deps = [ ":redisbenchmark", - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/ruby/BUILD b/benchmarks/workloads/ruby/BUILD index 9846c7e70..a3be4fe92 100644 --- a/benchmarks/workloads/ruby/BUILD +++ b/benchmarks/workloads/ruby/BUILD @@ -1,3 +1,5 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], @@ -13,3 +15,14 @@ filegroup( "index.rb", ], ) + +pkg_tar( + name = "tar", + srcs = [ + "Dockerfile", + "Gemfile", + "Gemfile.lock", + "config.ru", + "index.rb", + ], +) diff --git a/benchmarks/workloads/ruby_template/BUILD b/benchmarks/workloads/ruby_template/BUILD index 2b99892af..59443b14a 100644 --- a/benchmarks/workloads/ruby_template/BUILD +++ b/benchmarks/workloads/ruby_template/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", "Gemfile", @@ -13,4 +15,5 @@ filegroup( "index.erb", "main.rb", ], + strip_prefix = "third_party/gvisor/benchmarks/workloads/ruby_template", ) diff --git a/benchmarks/workloads/sleep/BUILD b/benchmarks/workloads/sleep/BUILD index 83f3c71a0..a70873065 100644 --- a/benchmarks/workloads/sleep/BUILD +++ b/benchmarks/workloads/sleep/BUILD @@ -1,10 +1,12 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/sysbench/BUILD b/benchmarks/workloads/sysbench/BUILD index 35f4d460b..3834af7ed 100644 --- a/benchmarks/workloads/sysbench/BUILD +++ b/benchmarks/workloads/sysbench/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -16,19 +16,19 @@ py_test( python_version = "PY3", deps = [ ":sysbench", - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/syscall/BUILD b/benchmarks/workloads/syscall/BUILD index e1ff3059b..dba4bb1e7 100644 --- a/benchmarks/workloads/syscall/BUILD +++ b/benchmarks/workloads/syscall/BUILD @@ -1,4 +1,4 @@ -load("//benchmarks:defs.bzl", "py_library", "py_test", "requirement") +load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test") package( default_visibility = ["//benchmarks:__subpackages__"], @@ -16,19 +16,19 @@ py_test( python_version = "PY3", deps = [ ":syscall", - requirement("attrs", False), - requirement("atomicwrites", False), - requirement("more-itertools", False), - requirement("pathlib2", False), - requirement("pluggy", False), - requirement("py", False), - requirement("pytest", True), - requirement("six", False), + py_requirement("attrs", False), + py_requirement("atomicwrites", False), + py_requirement("more-itertools", False), + py_requirement("pathlib2", False), + py_requirement("pluggy", False), + py_requirement("py", False), + py_requirement("pytest", True), + py_requirement("six", False), ], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", "syscall.c", diff --git a/benchmarks/workloads/tensorflow/BUILD b/benchmarks/workloads/tensorflow/BUILD index 17f1f8ebb..a7b7742f4 100644 --- a/benchmarks/workloads/tensorflow/BUILD +++ b/benchmarks/workloads/tensorflow/BUILD @@ -1,3 +1,5 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], @@ -8,8 +10,8 @@ py_library( srcs = ["__init__.py"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], diff --git a/benchmarks/workloads/true/BUILD b/benchmarks/workloads/true/BUILD index 83f3c71a0..eba23d325 100644 --- a/benchmarks/workloads/true/BUILD +++ b/benchmarks/workloads/true/BUILD @@ -1,11 +1,14 @@ +load("//tools:defs.bzl", "pkg_tar") + package( default_visibility = ["//benchmarks:__subpackages__"], licenses = ["notice"], ) -filegroup( - name = "files", +pkg_tar( + name = "tar", srcs = [ "Dockerfile", ], + extension = "tar", ) |