summaryrefslogtreecommitdiffhomepage
path: root/benchmarks
diff options
context:
space:
mode:
Diffstat (limited to 'benchmarks')
-rw-r--r--benchmarks/BUILD18
-rw-r--r--benchmarks/README.md23
-rw-r--r--benchmarks/defs.bzl14
-rw-r--r--benchmarks/harness/BUILD165
-rw-r--r--benchmarks/harness/machine.py9
-rw-r--r--benchmarks/harness/machine_producers/BUILD5
-rw-r--r--benchmarks/harness/ssh_connection.py9
-rw-r--r--benchmarks/runner/BUILD17
-rw-r--r--benchmarks/workloads/ab/BUILD13
-rw-r--r--benchmarks/workloads/absl/BUILD13
-rw-r--r--benchmarks/workloads/fio/BUILD13
-rw-r--r--benchmarks/workloads/iperf/BUILD13
-rw-r--r--benchmarks/workloads/redisbenchmark/BUILD13
-rw-r--r--benchmarks/workloads/sysbench/BUILD13
-rw-r--r--benchmarks/workloads/syscall/BUILD13
15 files changed, 210 insertions, 141 deletions
diff --git a/benchmarks/BUILD b/benchmarks/BUILD
index 43614cf5d..2a2d15d7e 100644
--- a/benchmarks/BUILD
+++ b/benchmarks/BUILD
@@ -1,12 +1,22 @@
package(licenses = ["notice"])
+config_setting(
+ name = "gcloud_rule",
+ values = {
+ "define": "gcloud=off",
+ },
+)
+
py_binary(
name = "benchmarks",
srcs = ["run.py"],
- data = [
- "//tools/images:ubuntu1604",
- "//tools/images:zone",
- ],
+ data = select({
+ ":gcloud_rule": [],
+ "//conditions:default": [
+ "//tools/images:ubuntu1604",
+ "//tools/images:zone",
+ ],
+ }),
main = "run.py",
python_version = "PY3",
srcs_version = "PY3",
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 975321c99..6d1ea3ae2 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -28,10 +28,12 @@ For configuring the environment manually, consult the
### Locally
-Run the following from the benchmarks directory:
+The tool is built to, by default, use Google Cloud Platform to run benchmarks,
+but it does support GCP workflows. To run locally, run the following from the
+benchmarks directory:
```bash
-bazel run :benchmarks -- run-local startup
+bazel run --define gcloud=off :benchmarks -- run-local startup
...
method,metric,result
@@ -46,17 +48,16 @@ runtime, runc. Running on another installed runtime, like say runsc, is as
simple as:
```bash
-bazel run :benchmarks -- run-local startup --runtime=runsc
+bazel run --define gcloud=off :benchmarks -- run-local startup --runtime=runsc
```
-There is help: ``bash bash bazel run :benchmarks -- --help bazel
-run :benchmarks -- run-local --help` ``
+There is help: `bash bazel run --define gcloud=off :benchmarks -- --help bazel
+run --define gcloud=off :benchmarks -- run-local --help`
To list available benchmarks, use the `list` commmand:
```bash
-bazel run :benchmarks -- list
-ls
+bazel --define gcloud=off run :benchmarks -- list
...
Benchmark: sysbench.cpu
@@ -69,7 +70,7 @@ Metrics: events_per_second
You can choose benchmarks by name or regex like:
```bash
-bazel run :benchmarks -- run-local startup.node
+bazel run --define gcloud=off :benchmarks -- run-local startup.node
...
metric,result
startup_time_ms,1671.7178000000001
@@ -79,7 +80,7 @@ startup_time_ms,1671.7178000000001
or
```bash
-bazel run :benchmarks -- run-local s
+bazel run --define gcloud=off :benchmarks -- run-local s
...
method,metric,result
startup.empty,startup_time_ms,1792.8292
@@ -97,13 +98,13 @@ You can run parameterized benchmarks, for example to run with different
runtimes:
```bash
-bazel run :benchmarks -- run-local --runtime=runc --runtime=runsc sysbench.cpu
+bazel run --define gcloud=off :benchmarks -- run-local --runtime=runc --runtime=runsc sysbench.cpu
```
Or with different parameters:
```bash
-bazel run :benchmarks -- run-local --max_prime=10 --max_prime=100 sysbench.cpu
+bazel run --define gcloud=off :benchmarks -- run-local --max_prime=10 --max_prime=100 sysbench.cpu
```
### On Google Compute Engine (GCE)
diff --git a/benchmarks/defs.bzl b/benchmarks/defs.bzl
new file mode 100644
index 000000000..56d28223e
--- /dev/null
+++ b/benchmarks/defs.bzl
@@ -0,0 +1,14 @@
+"""Provides attributes common to many workload tests."""
+
+load("//tools:defs.bzl", "py_requirement")
+
+test_deps = [
+ py_requirement("attrs", direct = False),
+ py_requirement("atomicwrites", direct = False),
+ py_requirement("more-itertools", direct = False),
+ py_requirement("pathlib2", direct = False),
+ py_requirement("pluggy", direct = False),
+ py_requirement("py", direct = False),
+ py_requirement("pytest"),
+ py_requirement("six", direct = False),
+]
diff --git a/benchmarks/harness/BUILD b/benchmarks/harness/BUILD
index 4d03e3a06..48c548d59 100644
--- a/benchmarks/harness/BUILD
+++ b/benchmarks/harness/BUILD
@@ -1,5 +1,4 @@
-load("//tools:defs.bzl", "pkg_tar")
-load("//tools:defs.bzl", "py_library", "py_requirement")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -46,16 +45,43 @@ py_library(
srcs = ["container.py"],
deps = [
"//benchmarks/workloads",
- py_requirement("asn1crypto", False),
- py_requirement("chardet", False),
- py_requirement("certifi", False),
- py_requirement("docker", True),
- py_requirement("docker-pycreds", False),
- py_requirement("idna", False),
- py_requirement("ptyprocess", False),
- py_requirement("requests", False),
- py_requirement("urllib3", False),
- py_requirement("websocket-client", False),
+ py_requirement(
+ "asn1crypto",
+ direct = False,
+ ),
+ py_requirement(
+ "chardet",
+ direct = False,
+ ),
+ py_requirement(
+ "certifi",
+ direct = False,
+ ),
+ py_requirement("docker"),
+ py_requirement(
+ "docker-pycreds",
+ direct = False,
+ ),
+ py_requirement(
+ "idna",
+ direct = False,
+ ),
+ py_requirement(
+ "ptyprocess",
+ direct = False,
+ ),
+ py_requirement(
+ "requests",
+ direct = False,
+ ),
+ py_requirement(
+ "urllib3",
+ direct = False,
+ ),
+ py_requirement(
+ "websocket-client",
+ direct = False,
+ ),
],
)
@@ -68,17 +94,47 @@ py_library(
"//benchmarks/harness:ssh_connection",
"//benchmarks/harness:tunnel_dispatcher",
"//benchmarks/harness/machine_mocks",
- py_requirement("asn1crypto", False),
- py_requirement("chardet", False),
- py_requirement("certifi", False),
- py_requirement("docker", True),
- py_requirement("docker-pycreds", False),
- py_requirement("idna", False),
- py_requirement("ptyprocess", False),
- py_requirement("requests", False),
- py_requirement("six", False),
- py_requirement("urllib3", False),
- py_requirement("websocket-client", False),
+ py_requirement(
+ "asn1crypto",
+ direct = False,
+ ),
+ py_requirement(
+ "chardet",
+ direct = False,
+ ),
+ py_requirement(
+ "certifi",
+ direct = False,
+ ),
+ py_requirement("docker"),
+ py_requirement(
+ "docker-pycreds",
+ direct = False,
+ ),
+ py_requirement(
+ "idna",
+ direct = False,
+ ),
+ py_requirement(
+ "ptyprocess",
+ direct = False,
+ ),
+ py_requirement(
+ "requests",
+ direct = False,
+ ),
+ py_requirement(
+ "six",
+ direct = False,
+ ),
+ py_requirement(
+ "urllib3",
+ direct = False,
+ ),
+ py_requirement(
+ "websocket-client",
+ direct = False,
+ ),
],
)
@@ -87,10 +143,16 @@ py_library(
srcs = ["ssh_connection.py"],
deps = [
"//benchmarks/harness",
- py_requirement("bcrypt", False),
- py_requirement("cffi", True),
- py_requirement("paramiko", True),
- py_requirement("cryptography", False),
+ py_requirement(
+ "bcrypt",
+ direct = False,
+ ),
+ py_requirement("cffi"),
+ py_requirement("paramiko"),
+ py_requirement(
+ "cryptography",
+ direct = False,
+ ),
],
)
@@ -98,16 +160,43 @@ py_library(
name = "tunnel_dispatcher",
srcs = ["tunnel_dispatcher.py"],
deps = [
- py_requirement("asn1crypto", False),
- py_requirement("chardet", False),
- py_requirement("certifi", False),
- py_requirement("docker", True),
- py_requirement("docker-pycreds", False),
- py_requirement("idna", False),
- py_requirement("pexpect", True),
- py_requirement("ptyprocess", False),
- py_requirement("requests", False),
- py_requirement("urllib3", False),
- py_requirement("websocket-client", False),
+ py_requirement(
+ "asn1crypto",
+ direct = False,
+ ),
+ py_requirement(
+ "chardet",
+ direct = False,
+ ),
+ py_requirement(
+ "certifi",
+ direct = False,
+ ),
+ py_requirement("docker"),
+ py_requirement(
+ "docker-pycreds",
+ direct = False,
+ ),
+ py_requirement(
+ "idna",
+ direct = False,
+ ),
+ py_requirement("pexpect"),
+ py_requirement(
+ "ptyprocess",
+ direct = False,
+ ),
+ py_requirement(
+ "requests",
+ direct = False,
+ ),
+ py_requirement(
+ "urllib3",
+ direct = False,
+ ),
+ py_requirement(
+ "websocket-client",
+ direct = False,
+ ),
],
)
diff --git a/benchmarks/harness/machine.py b/benchmarks/harness/machine.py
index 3d32d3dda..5bdc4aa85 100644
--- a/benchmarks/harness/machine.py
+++ b/benchmarks/harness/machine.py
@@ -43,6 +43,8 @@ from benchmarks.harness import machine_mocks
from benchmarks.harness import ssh_connection
from benchmarks.harness import tunnel_dispatcher
+log = logging.getLogger(__name__)
+
class Machine(object):
"""The machine object is the primary object for benchmarks.
@@ -236,9 +238,10 @@ class RemoteMachine(Machine):
archive=archive, dir=harness.REMOTE_INSTALLERS_PATH))
self._has_installers = True
- # Execute the remote installer.
- self.run("sudo {dir}/{file}".format(
- dir=harness.REMOTE_INSTALLERS_PATH, file=installer))
+ # Execute the remote installer.
+ self.run("sudo {dir}/{file}".format(
+ dir=harness.REMOTE_INSTALLERS_PATH, file=installer))
+
if results:
results[index] = True
diff --git a/benchmarks/harness/machine_producers/BUILD b/benchmarks/harness/machine_producers/BUILD
index 3711a397f..81f19bd08 100644
--- a/benchmarks/harness/machine_producers/BUILD
+++ b/benchmarks/harness/machine_producers/BUILD
@@ -31,7 +31,10 @@ py_library(
deps = [
"//benchmarks/harness:machine",
"//benchmarks/harness/machine_producers:machine_producer",
- py_requirement("PyYAML", False),
+ py_requirement(
+ "PyYAML",
+ direct = False,
+ ),
],
)
diff --git a/benchmarks/harness/ssh_connection.py b/benchmarks/harness/ssh_connection.py
index a50e34293..b8c8e42d4 100644
--- a/benchmarks/harness/ssh_connection.py
+++ b/benchmarks/harness/ssh_connection.py
@@ -13,7 +13,7 @@
# limitations under the License.
"""SSHConnection handles the details of SSH connections."""
-
+import logging
import os
import warnings
@@ -24,6 +24,8 @@ from benchmarks import harness
# Get rid of paramiko Cryptography Warnings.
warnings.filterwarnings(action="ignore", module=".*paramiko.*")
+log = logging.getLogger(__name__)
+
def send_one_file(client: paramiko.SSHClient, path: str,
remote_dir: str) -> str:
@@ -94,10 +96,13 @@ class SSHConnection:
The contents of stdout and stderr.
"""
with self._client() as client:
+ log.info("running command: %s", cmd)
_, stdout, stderr = client.exec_command(command=cmd)
- stdout.channel.recv_exit_status()
+ log.info("returned status: %d", stdout.channel.recv_exit_status())
stdout = stdout.read().decode("utf-8")
stderr = stderr.read().decode("utf-8")
+ log.info("stdout: %s", stdout)
+ log.info("stderr: %s", stderr)
return stdout, stderr
def send_workload(self, name: str) -> str:
diff --git a/benchmarks/runner/BUILD b/benchmarks/runner/BUILD
index fae0ca800..471debfdf 100644
--- a/benchmarks/runner/BUILD
+++ b/benchmarks/runner/BUILD
@@ -1,4 +1,5 @@
load("//tools:defs.bzl", "py_library", "py_requirement", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(licenses = ["notice"])
@@ -28,7 +29,7 @@ py_library(
"//benchmarks/suites:startup",
"//benchmarks/suites:sysbench",
"//benchmarks/suites:syscall",
- py_requirement("click", True),
+ py_requirement("click"),
],
)
@@ -36,7 +37,7 @@ py_library(
name = "commands",
srcs = ["commands.py"],
deps = [
- py_requirement("click", True),
+ py_requirement("click"),
],
)
@@ -48,16 +49,8 @@ py_test(
"local",
"manual",
],
- deps = [
+ deps = test_deps + [
":runner",
- py_requirement("click", True),
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
+ py_requirement("click"),
],
)
diff --git a/benchmarks/workloads/ab/BUILD b/benchmarks/workloads/ab/BUILD
index 4dd91ceb3..945ac7026 100644
--- a/benchmarks/workloads/ab/BUILD
+++ b/benchmarks/workloads/ab/BUILD
@@ -1,4 +1,5 @@
-load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -14,16 +15,8 @@ py_test(
name = "ab_test",
srcs = ["ab_test.py"],
python_version = "PY3",
- deps = [
+ deps = test_deps + [
":ab",
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
],
)
diff --git a/benchmarks/workloads/absl/BUILD b/benchmarks/workloads/absl/BUILD
index 55dae3baa..bb1a308bf 100644
--- a/benchmarks/workloads/absl/BUILD
+++ b/benchmarks/workloads/absl/BUILD
@@ -1,4 +1,5 @@
-load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -14,16 +15,8 @@ py_test(
name = "absl_test",
srcs = ["absl_test.py"],
python_version = "PY3",
- deps = [
+ deps = test_deps + [
":absl",
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
],
)
diff --git a/benchmarks/workloads/fio/BUILD b/benchmarks/workloads/fio/BUILD
index 7b78e8e75..24d909c53 100644
--- a/benchmarks/workloads/fio/BUILD
+++ b/benchmarks/workloads/fio/BUILD
@@ -1,4 +1,5 @@
-load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -14,16 +15,8 @@ py_test(
name = "fio_test",
srcs = ["fio_test.py"],
python_version = "PY3",
- deps = [
+ deps = test_deps + [
":fio",
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
],
)
diff --git a/benchmarks/workloads/iperf/BUILD b/benchmarks/workloads/iperf/BUILD
index 570f40148..91b953718 100644
--- a/benchmarks/workloads/iperf/BUILD
+++ b/benchmarks/workloads/iperf/BUILD
@@ -1,4 +1,5 @@
-load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -14,16 +15,8 @@ py_test(
name = "iperf_test",
srcs = ["iperf_test.py"],
python_version = "PY3",
- deps = [
+ deps = test_deps + [
":iperf",
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
],
)
diff --git a/benchmarks/workloads/redisbenchmark/BUILD b/benchmarks/workloads/redisbenchmark/BUILD
index f472a4443..147cfedd2 100644
--- a/benchmarks/workloads/redisbenchmark/BUILD
+++ b/benchmarks/workloads/redisbenchmark/BUILD
@@ -1,4 +1,5 @@
-load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -14,16 +15,8 @@ py_test(
name = "redisbenchmark_test",
srcs = ["redisbenchmark_test.py"],
python_version = "PY3",
- deps = [
+ deps = test_deps + [
":redisbenchmark",
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
],
)
diff --git a/benchmarks/workloads/sysbench/BUILD b/benchmarks/workloads/sysbench/BUILD
index 3834af7ed..ab2556064 100644
--- a/benchmarks/workloads/sysbench/BUILD
+++ b/benchmarks/workloads/sysbench/BUILD
@@ -1,4 +1,5 @@
-load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -14,16 +15,8 @@ py_test(
name = "sysbench_test",
srcs = ["sysbench_test.py"],
python_version = "PY3",
- deps = [
+ deps = test_deps + [
":sysbench",
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
],
)
diff --git a/benchmarks/workloads/syscall/BUILD b/benchmarks/workloads/syscall/BUILD
index dba4bb1e7..f8c43bca1 100644
--- a/benchmarks/workloads/syscall/BUILD
+++ b/benchmarks/workloads/syscall/BUILD
@@ -1,4 +1,5 @@
-load("//tools:defs.bzl", "pkg_tar", "py_library", "py_requirement", "py_test")
+load("//tools:defs.bzl", "pkg_tar", "py_library", "py_test")
+load("//benchmarks:defs.bzl", "test_deps")
package(
default_visibility = ["//benchmarks:__subpackages__"],
@@ -14,16 +15,8 @@ py_test(
name = "syscall_test",
srcs = ["syscall_test.py"],
python_version = "PY3",
- deps = [
+ deps = test_deps + [
":syscall",
- py_requirement("attrs", False),
- py_requirement("atomicwrites", False),
- py_requirement("more-itertools", False),
- py_requirement("pathlib2", False),
- py_requirement("pluggy", False),
- py_requirement("py", False),
- py_requirement("pytest", True),
- py_requirement("six", False),
],
)
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-27 12:23:23 +0000