Remove old benchmark tools.

Remove the old benchmark-tools directory, including
imports in the WORKSPACE file and associated bazel rules.

The new Golang benchmark-tools can be found at //test/benchmarks
and it is functionally equivalent, excepting syscall_test
which can be found in //test/perf/linux.

PiperOrigin-RevId: 325529075

10 files changed, 0 insertions, 1057 deletions

diff --git a/benchmarks/harness/machine_producers/BUILD b/benchmarks/harness/machine_producers/BUILD
deleted file mode 100644
index 81f19bd08..000000000
--- a/benchmarks/harness/machine_producers/BUILD
+++ /dev/null
@@ -1,84 +0,0 @@
-load("//tools:defs.bzl", "py_library", "py_requirement")
-
-package(
-    default_visibility = ["//benchmarks:__subpackages__"],
-    licenses = ["notice"],
-)
-
-py_library(
-    name = "harness",
-    srcs = ["__init__.py"],
-)
-
-py_library(
-    name = "machine_producer",
-    srcs = ["machine_producer.py"],
-)
-
-py_library(
-    name = "mock_producer",
-    srcs = ["mock_producer.py"],
-    deps = [
-        "//benchmarks/harness:machine",
-        "//benchmarks/harness/machine_producers:gcloud_producer",
-        "//benchmarks/harness/machine_producers:machine_producer",
-    ],
-)
-
-py_library(
-    name = "yaml_producer",
-    srcs = ["yaml_producer.py"],
-    deps = [
-        "//benchmarks/harness:machine",
-        "//benchmarks/harness/machine_producers:machine_producer",
-        py_requirement(
-            "PyYAML",
-            direct = False,
-        ),
-    ],
-)
-
-py_library(
-    name = "gcloud_mock_recorder",
-    srcs = ["gcloud_mock_recorder.py"],
-)
-
-py_library(
-    name = "gcloud_producer",
-    srcs = ["gcloud_producer.py"],
-    deps = [
-        "//benchmarks/harness:machine",
-        "//benchmarks/harness/machine_producers:gcloud_mock_recorder",
-        "//benchmarks/harness/machine_producers:machine_producer",
-    ],
-)
-
-filegroup(
-    name = "test_data",
-    srcs = [
-        "testdata/get_five.json",
-        "testdata/get_one.json",
-    ],
-)
-
-py_library(
-    name = "gcloud_producer_test_lib",
-    srcs = ["gcloud_producer_test.py"],
-    deps = [
-        "//benchmarks/harness/machine_producers:machine_producer",
-        "//benchmarks/harness/machine_producers:mock_producer",
-    ],
-)
-
-py_test(
-    name = "gcloud_producer_test",
-    srcs = [":gcloud_producer_test_lib"],
-    data = [
-        ":test_data",
-    ],
-    python_version = "PY3",
-    tags = [
-        "local",
-        "manual",
-    ],
-)
diff --git a/benchmarks/harness/machine_producers/__init__.py b/benchmarks/harness/machine_producers/__init__.py
deleted file mode 100644
index 634ef4843..000000000
--- a/benchmarks/harness/machine_producers/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# python3
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/benchmarks/harness/machine_producers/gcloud_mock_recorder.py b/benchmarks/harness/machine_producers/gcloud_mock_recorder.py
deleted file mode 100644
index fd9837a37..000000000
--- a/benchmarks/harness/machine_producers/gcloud_mock_recorder.py
+++ /dev/null
@@ -1,97 +0,0 @@
-# python3
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""A recorder and replay for testing the GCloudProducer.
-
-MockPrinter and MockReader handle printing and reading mock data for the
-purposes of testing. MockPrinter is passed to GCloudProducer objects. The user
-can then run scenarios and record them for playback in tests later.
-
-MockReader is passed to MockGcloudProducer objects and handles reading the
-previously recorded mock data.
-
-It is left to the user to check if data printed is properly redacted for their
-own use. The intended usecase for this class is data coming from gcloud
-commands, which will contain public IPs and other instance data.
-
-The data format is json and printed/read from the ./test_data directory. The
-data is the output of subprocess.CompletedProcess objects in json format.
-
-  Typical usage example:
-
-  recorder = MockPrinter()
-  producer = GCloudProducer(args, recorder)
-  machines = producer.get_machines(1)
-  with open("my_file.json") as fd:
-    recorder.write_out(fd)
-
-  reader = MockReader(filename)
-  producer = MockGcloudProducer(args, mock)
-  machines = producer.get_machines(1)
-  assert len(machines) == 1
-"""
-
-import io
-import json
-import subprocess
-
-
-class MockPrinter(object):
-  """Handles printing Mock data for MockGcloudProducer.
-
-  Attributes:
-    _records: list of json object records for printing
-  """
-
-  def __init__(self):
-    self._records = []
-
-  def record(self, entry: subprocess.CompletedProcess):
-    """Records data and strips out ip addresses."""
-
-    record = {
-        "args": entry.args,
-        "stdout": entry.stdout.decode("utf-8"),
-        "returncode": str(entry.returncode)
-    }
-    self._records.append(record)
-
-  def write_out(self, fd: io.FileIO):
-    """Prints out the data into the given filepath."""
-    fd.write(json.dumps(self._records, indent=4))
-
-
-class MockReader(object):
-  """Handles reading Mock data for MockGcloudProducer.
-
-  Attributes:
-    _records: List[json] records read from the passed in file.
-  """
-
-  def __init__(self, filepath: str):
-    with open(filepath, "rb") as file:
-      self._records = json.loads(file.read())
-      self._i = 0
-
-  def __iter__(self):
-    return self
-
-  def __next__(self, args) -> subprocess.CompletedProcess:
-    """Returns the next record as a CompletedProcess."""
-    if self._i < len(self._records):
-      record = self._records[self._i]
-      stdout = record["stdout"].encode("ascii")
-      returncode = int(record["returncode"])
-      return subprocess.CompletedProcess(
-          args=args, returncode=returncode, stdout=stdout, stderr=b"")
-    raise StopIteration()
diff --git a/benchmarks/harness/machine_producers/gcloud_producer.py b/benchmarks/harness/machine_producers/gcloud_producer.py
deleted file mode 100644
index 44d72f575..000000000
--- a/benchmarks/harness/machine_producers/gcloud_producer.py
+++ /dev/null
@@ -1,250 +0,0 @@
-# python3
-# Copyright 2019 The gVisor Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""A machine producer which produces machine objects using `gcloud`.
-
-Machine producers produce valid harness.Machine objects which are backed by
-real machines. This producer produces those machines on the given user's GCP
-account using the `gcloud` tool.
-
-GCloudProducer creates instances on the given GCP account named like:
-`machine-XXXXXXX-XXXX-XXXX-XXXXXXXXXXXX` in a randomized fashion such that name
-collisions with user instances shouldn't happen.
-
-  Typical usage example:
-
-  producer = GCloudProducer(args)
-  machines = producer.get_machines(NUM_MACHINES)
-  # run stuff on machines with machines[i].run(CMD)
-  producer.release_machines(NUM_MACHINES)
-"""
-import datetime
-import json
-import subprocess
-import threading
-from typing import List, Dict, Any
-import uuid
-
-from benchmarks.harness import machine
-from benchmarks.harness.machine_producers import gcloud_mock_recorder
-from benchmarks.harness.machine_producers import machine_producer
-
-
-class GCloudProducer(machine_producer.MachineProducer):
-  """Implementation of MachineProducer backed by GCP.
-
-  Produces Machine objects backed by GCP instances.
-
-  Attributes:
-    image: image name as a string.
-    zone: string to a valid GCP zone.
-    machine_type: type of GCP to create (e.g. n1-standard-4).
-    installers: list of installers post-boot.
-    ssh_key_file: path to a valid ssh private key. See README on vaild ssh keys.
-    ssh_user: string of user name for ssh_key
-    ssh_password: string of password for ssh key
-    internal: if true, use internal IPs of instances. Used if bm-tools is
-    running on a GCP vm when a firewall is set for external IPs.
-    mock: a mock printer which will print mock data if required. Mock data is
-      recorded output from subprocess calls (returncode, stdout, args).
-    condition: mutex for this class around machine creation and deleteion.
-  """
-
-  def __init__(self,
-               image: str,
-               zone: str,
-               machine_type: str,
-               installers: List[str],
-               ssh_key_file: str,
-               ssh_user: str,
-               ssh_password: str,
-               internal: bool,
-               mock: gcloud_mock_recorder.MockPrinter = None):
-    self.image = image
-    self.zone = zone
-    self.machine_type = machine_type
-    self.installers = installers
-    self.ssh_key_file = ssh_key_file
-    self.ssh_user = ssh_user
-    self.ssh_password = ssh_password
-    self.internal = internal
-    self.mock = mock
-    self.condition = threading.Condition()
-
-  def get_machines(self, num_machines: int) -> List[machine.Machine]:
-    """Returns requested number of machines backed by GCP instances."""
-    if num_machines <= 0:
-      raise ValueError(
-          "Cannot ask for {num} machines!".format(num=num_machines))
-    with self.condition:
-      names = self._get_unique_names(num_machines)
-      instances = self._build_instances(names)
-    self._add_ssh_key_to_instances(names)
-    machines = self._machines_from_instances(instances)
-
-    # Install all bits in lock-step.
-    #
-    # This will perform paralell installations for however many machines we
-    # have, but it's easy to track errors because if installing (a, b, c), we
-    # won't install "c" until "b" is installed on all machines.
-    for installer in self.installers:
-      threads = [None] * len(machines)
-      results = [False] * len(machines)
-      for i in range(len(machines)):
-        threads[i] = threading.Thread(
-            target=machines[i].install, args=(installer, results, i))
-        threads[i].start()
-      for thread in threads:
-        thread.join()
-      for result in results:
-        if not result:
-          raise NotImplementedError(
-              "Installers failed on at least one machine!")
-
-    # Add this user to each machine's docker group.
-    for m in machines:
-      m.run("sudo setfacl -m user:$USER:rw /var/run/docker.sock")
-
-    return machines
-
-  def release_machines(self, machine_list: List[machine.Machine]):
-    """Releases the requested number of machines, deleting the instances."""
-    if not machine_list:
-      return
-    cmd = "gcloud compute instances delete --quiet".split(" ")
-    names = [str(m) for m in machine_list]
-    cmd.extend(names)
-    cmd.append("--zone={zone}".format(zone=self.zone))
-    self._run_command(cmd, detach=True)
-
-  def _machines_from_instances(
-      self, instances: List[Dict[str, Any]]) -> List[machine.Machine]:
-    """Creates Machine Objects from json data describing created instances."""
-    machines = []
-    for instance in instances:
-      name = instance["name"]
-      external = instance["networkInterfaces"][0]["accessConfigs"][0]["natIP"]
-      internal = instance["networkInterfaces"][0]["networkIP"]
-      kwargs = {
-          "hostname": internal if self.internal else external,
-          "key_path": self.ssh_key_file,
-          "username": self.ssh_user,
-          "key_password": self.ssh_password
-      }
-      machines.append(machine.RemoteMachine(name=name, **kwargs))
-    return machines
-
-  def _get_unique_names(self, num_names) -> List[str]:
-    """Returns num_names unique names based on data from the GCP project."""
-    return ["machine-" + str(uuid.uuid4()) for _ in range(0, num_names)]
-
-  def _build_instances(self, names: List[str]) -> List[Dict[str, Any]]:
-    """Creates instances using gcloud command.
-
-    Runs the command `gcloud compute instances create` and returns json data
-    on created instances on success. Creates len(names) instances, one for each
-    name.
-
-    Args:
-      names: list of names of instances to create.
-
-    Returns:
-      List of json data describing created machines.
-    """
-    if not names:
-      raise ValueError(
-          "_build_instances cannot create instances without names.")
-    cmd = "gcloud compute instances create".split(" ")
-    cmd.extend(names)
-    cmd.append("--image=" + self.image)
-    cmd.append("--zone=" + self.zone)
-    cmd.append("--machine-type=" + self.machine_type)
-    res = self._run_command(cmd)
-    data = res.stdout
-    data = str(data, "utf-8") if isinstance(data, (bytes, bytearray)) else data
-    return json.loads(data)
-
-  def _add_ssh_key_to_instances(self, names: List[str]) -> None:
-    """Adds ssh key to instances by calling gcloud ssh command.
-
-    Runs the command `gcloud compute ssh instance_name` on list of images by
-    name. Tries to ssh into given instance.
-
-    Args:
-      names: list of machine names to which to add the ssh-key
-        self.ssh_key_file.
-
-    Raises:
-      subprocess.CalledProcessError: when underlying subprocess call returns an
-      error other than 255 (Connection closed by remote host).
-      TimeoutError: when 3 unsuccessful tries to ssh into the host return 255.
-    """
-    for name in names:
-      cmd = "gcloud compute ssh {user}@{name}".format(
-          user=self.ssh_user, name=name).split(" ")
-      if self.internal:
-        cmd.append("--internal-ip")
-      cmd.append("--ssh-key-file={key}".format(key=self.ssh_key_file))
-      cmd.append("--zone={zone}".format(zone=self.zone))
-      cmd.append("--command=uname")
-      timeout = datetime.timedelta(seconds=5 * 60)
-      start = datetime.datetime.now()
-      while datetime.datetime.now() <= timeout + start:
-        try:
-          self._run_command(cmd)
-          break
-        except subprocess.CalledProcessError:
-          if datetime.datetime.now() > timeout + start:
-            raise TimeoutError(
-                "Could not SSH into instance after 5 min: {name}".format(
-                    name=name))
-
-  def _run_command(self,
-                   cmd: List[str],
-                   detach: bool = False) -> [None, subprocess.CompletedProcess]:
-    """Runs command as a subprocess.
-
-    Runs command as subprocess and returns the result.
-    If this has a mock recorder, use the record method to record the subprocess
-    call.
-
-    Args:
-      cmd: command to be run as a list of strings.
-      detach: if True, run the child process and don't wait for it to return.
-
-    Returns:
-      Completed process object to be parsed by caller or None if detach=True.
-
-    Raises:
-      CalledProcessError: if subprocess.run returns an error.
-    """
-    cmd = cmd + ["--format=json"]
-    if detach:
-      p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      if self.mock:
-        out, _ = p.communicate()
-        self.mock.record(
-            subprocess.CompletedProcess(
-                returncode=p.returncode, stdout=out, args=p.args))
-      return
-
-    res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    if self.mock:
-      self.mock.record(res)
-    if res.returncode != 0:
-      raise subprocess.CalledProcessError(
-          cmd=" ".join(res.args),
-          output=res.stdout,
-          stderr=res.stderr,
-          returncode=res.returncode)
-    return res
diff --git a/benchmarks/harness/machine_producers/gcloud_producer_test.py b/benchmarks/harness/machine_producers/gcloud_producer_test.py
deleted file mode 100644
index c8adb2bdc..000000000
--- a/benchmarks/harness/machine_producers/gcloud_producer_test.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# python3
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tests GCloudProducer using mock data.
-
-GCloudProducer produces machines using 'get_machines' and 'release_machines'
-methods. The tests check recorded data (jsonified subprocess.CompletedProcess
-objects) of the producer producing one and five machines.
-"""
-import os
-import types
-
-from benchmarks.harness.machine_producers import machine_producer
-from benchmarks.harness.machine_producers import mock_producer
-
-TEST_DIR = os.path.dirname(__file__)
-
-
-def run_get_release(producer: machine_producer.MachineProducer,
-                    num_machines: int,
-                    validator: types.FunctionType = None):
-  machines = producer.get_machines(num_machines)
-  assert len(machines) == num_machines
-  if validator:
-    validator(machines=machines, cmd="uname -a", workload=None)
-  producer.release_machines(machines)
-
-
-def test_run_one():
-  mock = mock_producer.MockReader(TEST_DIR + "get_one.json")
-  producer = mock_producer.MockGCloudProducer(mock)
-  run_get_release(producer, 1)
-
-
-def test_run_five():
-  mock = mock_producer.MockReader(TEST_DIR + "get_five.json")
-  producer = mock_producer.MockGCloudProducer(mock)
-  run_get_release(producer, 5)
diff --git a/benchmarks/harness/machine_producers/machine_producer.py b/benchmarks/harness/machine_producers/machine_producer.py
deleted file mode 100644
index f5591c026..000000000
--- a/benchmarks/harness/machine_producers/machine_producer.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# python3
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Abstract types."""
-
-import threading
-from typing import List
-
-from benchmarks.harness import machine
-
-
-class MachineProducer:
-  """Abstract Machine producer."""
-
-  def get_machines(self, num_machines: int) -> List[machine.Machine]:
-    """Returns the requested number of machines."""
-    raise NotImplementedError
-
-  def release_machines(self, machine_list: List[machine.Machine]):
-    """Releases the given set of machines."""
-    raise NotImplementedError
-
-
-class LocalMachineProducer(MachineProducer):
-  """Produces Local Machines."""
-
-  def __init__(self, limit: int):
-    self.limit_sem = threading.Semaphore(value=limit)
-
-  def get_machines(self, num_machines: int) -> List[machine.Machine]:
-    """Returns the request number of MockMachines."""
-
-    self.limit_sem.acquire()
-    return [machine.LocalMachine("local") for _ in range(num_machines)]
-
-  def release_machines(self, machine_list: List[machine.MockMachine]):
-    """No-op."""
-    if not machine_list:
-      raise ValueError("Cannot release an empty list!")
-    self.limit_sem.release()
-    machine_list.clear()
diff --git a/benchmarks/harness/machine_producers/mock_producer.py b/benchmarks/harness/machine_producers/mock_producer.py
deleted file mode 100644
index 37e9cb4b7..000000000
--- a/benchmarks/harness/machine_producers/mock_producer.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# python3
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Producers of mocks."""
-
-from typing import List, Any
-
-from benchmarks.harness import machine
-from benchmarks.harness.machine_producers import gcloud_mock_recorder
-from benchmarks.harness.machine_producers import gcloud_producer
-from benchmarks.harness.machine_producers import machine_producer
-
-
-class MockMachineProducer(machine_producer.MachineProducer):
-  """Produces MockMachine objects."""
-
-  def get_machines(self, num_machines: int) -> List[machine.MockMachine]:
-    """Returns the request number of MockMachines."""
-    return [machine.MockMachine() for i in range(num_machines)]
-
-  def release_machines(self, machine_list: List[machine.MockMachine]):
-    """No-op."""
-    return
-
-
-class MockGCloudProducer(gcloud_producer.GCloudProducer):
-  """Mocks GCloudProducer for testing purposes."""
-
-  def __init__(self, mock: gcloud_mock_recorder.MockReader, **kwargs):
-    gcloud_producer.GCloudProducer.__init__(
-        self, project="mock", ssh_private_key_path="mock", **kwargs)
-    self.mock = mock
-
-  def _validate_ssh_file(self):
-    pass
-
-  def _run_command(self, cmd):
-    return self.mock.pop(cmd)
-
-  def _machines_from_instances(
-      self, instances: List[Any]) -> List[machine.MockMachine]:
-    return [machine.MockMachine() for _ in instances]
diff --git a/benchmarks/harness/machine_producers/testdata/get_five.json b/benchmarks/harness/machine_producers/testdata/get_five.json
deleted file mode 100644
index 32bad1b06..000000000
--- a/benchmarks/harness/machine_producers/testdata/get_five.json
+++ /dev/null
@@ -1,211 +0,0 @@
-[
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "list",
-            "--project",
-            "project",
-            "--format=json"
-        ],
-        "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":{\"natIP\":\"0.0.0.0\"}]}]}]",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "create",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92",
-            "machine-da5859b5-bae6-435d-8005-0202d6f6e065",
-            "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05",
-            "machine-1149147d-71e2-43ea-8fe1-49256e5c441c",
-            "--preemptible",
-            "--image=ubuntu-1910-eoan-v20191204",
-            "--zone=us-west1-b",
-            "--image-project=ubuntu-os-cloud",
-            "--format=json"
-        ],
-        "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "start",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92",
-            "machine-da5859b5-bae6-435d-8005-0202d6f6e065",
-            "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05",
-            "machine-1149147d-71e2-43ea-8fe1-49256e5c441c",
-            "--zone=us-west1-b",
-            "--project=project",
-            "--format=json"
-        ],
-        "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "Linux\n[]\n",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "Linux\n[]\n",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-da5859b5-bae6-435d-8005-0202d6f6e065",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "Linux\n[]\n",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "Linux\n[]\n",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-1149147d-71e2-43ea-8fe1-49256e5c441c",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "Linux\n[]\n",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "delete",
-            "--quiet",
-            "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc",
-            "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92",
-            "machine-da5859b5-bae6-435d-8005-0202d6f6e065",
-            "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05",
-            "machine-1149147d-71e2-43ea-8fe1-49256e5c441c",
-            "--zone=us-west1-b",
-            "--format=json"
-        ],
-        "stdout": "[]\n",
-        "returncode": "0"
-    }
-]
diff --git a/benchmarks/harness/machine_producers/testdata/get_one.json b/benchmarks/harness/machine_producers/testdata/get_one.json
deleted file mode 100644
index c359c19c8..000000000
--- a/benchmarks/harness/machine_producers/testdata/get_one.json
+++ /dev/null
@@ -1,145 +0,0 @@
-[
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "list",
-            "--project",
-            "linux-testing-user",
-            "--format=json"
-        ],
-        "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]",
-
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "create",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--preemptible",
-            "--image=ubuntu-1910-eoan-v20191204",
-            "--zone=us-west1-b",
-            "--image-project=ubuntu-os-cloud",
-            "--format=json"
-        ],
-        "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "start",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--zone=us-west1-b",
-            "--project=linux-testing-user",
-            "--format=json"
-        ],
-        "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]",
-        
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "",
-        "returncode": "255"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "ssh",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools",
-            "--zone=us-west1-b",
-            "--command=uname",
-            "--format=json"
-        ],
-        "stdout": "Linux\n[]\n",
-        "returncode": "0"
-    },
-    {
-        "args": [
-            "gcloud",
-            "compute",
-            "instances",
-            "delete",
-            "--quiet",
-            "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc",
-            "--zone=us-west1-b",
-            "--format=json"
-        ],
-        "stdout": "[]\n",
-        "returncode": "0"
-    }
-]
diff --git a/benchmarks/harness/machine_producers/yaml_producer.py b/benchmarks/harness/machine_producers/yaml_producer.py
deleted file mode 100644
index 5d334e480..000000000
--- a/benchmarks/harness/machine_producers/yaml_producer.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# python3
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Producers based on yaml files."""
-
-import os
-import threading
-from typing import Dict
-from typing import List
-
-import yaml
-
-from benchmarks.harness import machine
-from benchmarks.harness.machine_producers import machine_producer
-
-
-class YamlMachineProducer(machine_producer.MachineProducer):
-  """Loads machines from a yaml file."""
-
-  def __init__(self, path: str):
-    self.machines = build_machines(path)
-    self.max_machines = len(self.machines)
-    self.machine_condition = threading.Condition()
-
-  def get_machines(self, num_machines: int) -> List[machine.Machine]:
-    if num_machines > self.max_machines:
-      raise ValueError(
-          "Insufficient Ammount of Machines. {ask} asked for and have {max_num} max."
-          .format(ask=num_machines, max_num=self.max_machines))
-
-    with self.machine_condition:
-      while not self._enough_machines(num_machines):
-        self.machine_condition.wait(timeout=1)
-      return [self.machines.pop(0) for _ in range(num_machines)]
-
-  def release_machines(self, machine_list: List[machine.Machine]):
-    with self.machine_condition:
-      while machine_list:
-        next_machine = machine_list.pop()
-        self.machines.append(next_machine)
-      self.machine_condition.notify()
-
-  def _enough_machines(self, ask: int):
-    return ask <= len(self.machines)
-
-
-def build_machines(path: str, num_machines: str = -1) -> List[machine.Machine]:
-  """Builds machine objects defined by the yaml file "path".
-
-  Args:
-    path: The path to a yaml file which defines machines.
-    num_machines: Optional limit on how many machine objects to build.
-
-  Returns:
-    Machine objects in a list.
-
-    If num_machines is set, len(machines) <= num_machines.
-  """
-  data = parse_yaml(path)
-  machines = []
-  for key, value in data.items():
-    if len(machines) == num_machines:
-      return machines
-    if isinstance(value, dict):
-      machines.append(machine.RemoteMachine(key, **value))
-    else:
-      machines.append(machine.LocalMachine(key))
-  return machines
-
-
-def parse_yaml(path: str) -> Dict[str, Dict[str, str]]:
-  """Parse the yaml file pointed by path.
-
-  Args:
-    path: The path to yaml file.
-
-  Returns:
-    The contents of the yaml file as a dictionary.
-  """
-  data = get_file_contents(path)
-  return yaml.load(data, Loader=yaml.Loader)
-
-
-def get_file_contents(path: str) -> str:
-  """Dumps the file contents to a string and returns them.
-
-  Args:
-    path: The path to dump.
-
-  Returns:
-    The file contents as a string.
-  """
-  if not os.path.isabs(path):
-    path = os.path.abspath(path)
-  with open(path) as input_file:
-    return input_file.read()