diff options
Diffstat (limited to 'benchmarks/harness/machine_producers')
10 files changed, 1057 insertions, 0 deletions
diff --git a/benchmarks/harness/machine_producers/BUILD b/benchmarks/harness/machine_producers/BUILD new file mode 100644 index 000000000..81f19bd08 --- /dev/null +++ b/benchmarks/harness/machine_producers/BUILD @@ -0,0 +1,84 @@ +load("//tools:defs.bzl", "py_library", "py_requirement") + +package( + default_visibility = ["//benchmarks:__subpackages__"], + licenses = ["notice"], +) + +py_library( + name = "harness", + srcs = ["__init__.py"], +) + +py_library( + name = "machine_producer", + srcs = ["machine_producer.py"], +) + +py_library( + name = "mock_producer", + srcs = ["mock_producer.py"], + deps = [ + "//benchmarks/harness:machine", + "//benchmarks/harness/machine_producers:gcloud_producer", + "//benchmarks/harness/machine_producers:machine_producer", + ], +) + +py_library( + name = "yaml_producer", + srcs = ["yaml_producer.py"], + deps = [ + "//benchmarks/harness:machine", + "//benchmarks/harness/machine_producers:machine_producer", + py_requirement( + "PyYAML", + direct = False, + ), + ], +) + +py_library( + name = "gcloud_mock_recorder", + srcs = ["gcloud_mock_recorder.py"], +) + +py_library( + name = "gcloud_producer", + srcs = ["gcloud_producer.py"], + deps = [ + "//benchmarks/harness:machine", + "//benchmarks/harness/machine_producers:gcloud_mock_recorder", + "//benchmarks/harness/machine_producers:machine_producer", + ], +) + +filegroup( + name = "test_data", + srcs = [ + "testdata/get_five.json", + "testdata/get_one.json", + ], +) + +py_library( + name = "gcloud_producer_test_lib", + srcs = ["gcloud_producer_test.py"], + deps = [ + "//benchmarks/harness/machine_producers:machine_producer", + "//benchmarks/harness/machine_producers:mock_producer", + ], +) + +py_test( + name = "gcloud_producer_test", + srcs = [":gcloud_producer_test_lib"], + data = [ + ":test_data", + ], + python_version = "PY3", + tags = [ + "local", + "manual", + ], +) diff --git a/benchmarks/harness/machine_producers/__init__.py b/benchmarks/harness/machine_producers/__init__.py new file mode 100644 index 000000000..634ef4843 --- /dev/null +++ b/benchmarks/harness/machine_producers/__init__.py @@ -0,0 +1,13 @@ +# python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/benchmarks/harness/machine_producers/gcloud_mock_recorder.py b/benchmarks/harness/machine_producers/gcloud_mock_recorder.py new file mode 100644 index 000000000..fd9837a37 --- /dev/null +++ b/benchmarks/harness/machine_producers/gcloud_mock_recorder.py @@ -0,0 +1,97 @@ +# python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A recorder and replay for testing the GCloudProducer. + +MockPrinter and MockReader handle printing and reading mock data for the +purposes of testing. MockPrinter is passed to GCloudProducer objects. The user +can then run scenarios and record them for playback in tests later. + +MockReader is passed to MockGcloudProducer objects and handles reading the +previously recorded mock data. + +It is left to the user to check if data printed is properly redacted for their +own use. The intended usecase for this class is data coming from gcloud +commands, which will contain public IPs and other instance data. + +The data format is json and printed/read from the ./test_data directory. The +data is the output of subprocess.CompletedProcess objects in json format. + + Typical usage example: + + recorder = MockPrinter() + producer = GCloudProducer(args, recorder) + machines = producer.get_machines(1) + with open("my_file.json") as fd: + recorder.write_out(fd) + + reader = MockReader(filename) + producer = MockGcloudProducer(args, mock) + machines = producer.get_machines(1) + assert len(machines) == 1 +""" + +import io +import json +import subprocess + + +class MockPrinter(object): + """Handles printing Mock data for MockGcloudProducer. + + Attributes: + _records: list of json object records for printing + """ + + def __init__(self): + self._records = [] + + def record(self, entry: subprocess.CompletedProcess): + """Records data and strips out ip addresses.""" + + record = { + "args": entry.args, + "stdout": entry.stdout.decode("utf-8"), + "returncode": str(entry.returncode) + } + self._records.append(record) + + def write_out(self, fd: io.FileIO): + """Prints out the data into the given filepath.""" + fd.write(json.dumps(self._records, indent=4)) + + +class MockReader(object): + """Handles reading Mock data for MockGcloudProducer. + + Attributes: + _records: List[json] records read from the passed in file. + """ + + def __init__(self, filepath: str): + with open(filepath, "rb") as file: + self._records = json.loads(file.read()) + self._i = 0 + + def __iter__(self): + return self + + def __next__(self, args) -> subprocess.CompletedProcess: + """Returns the next record as a CompletedProcess.""" + if self._i < len(self._records): + record = self._records[self._i] + stdout = record["stdout"].encode("ascii") + returncode = int(record["returncode"]) + return subprocess.CompletedProcess( + args=args, returncode=returncode, stdout=stdout, stderr=b"") + raise StopIteration() diff --git a/benchmarks/harness/machine_producers/gcloud_producer.py b/benchmarks/harness/machine_producers/gcloud_producer.py new file mode 100644 index 000000000..44d72f575 --- /dev/null +++ b/benchmarks/harness/machine_producers/gcloud_producer.py @@ -0,0 +1,250 @@ +# python3 +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A machine producer which produces machine objects using `gcloud`. + +Machine producers produce valid harness.Machine objects which are backed by +real machines. This producer produces those machines on the given user's GCP +account using the `gcloud` tool. + +GCloudProducer creates instances on the given GCP account named like: +`machine-XXXXXXX-XXXX-XXXX-XXXXXXXXXXXX` in a randomized fashion such that name +collisions with user instances shouldn't happen. + + Typical usage example: + + producer = GCloudProducer(args) + machines = producer.get_machines(NUM_MACHINES) + # run stuff on machines with machines[i].run(CMD) + producer.release_machines(NUM_MACHINES) +""" +import datetime +import json +import subprocess +import threading +from typing import List, Dict, Any +import uuid + +from benchmarks.harness import machine +from benchmarks.harness.machine_producers import gcloud_mock_recorder +from benchmarks.harness.machine_producers import machine_producer + + +class GCloudProducer(machine_producer.MachineProducer): + """Implementation of MachineProducer backed by GCP. + + Produces Machine objects backed by GCP instances. + + Attributes: + image: image name as a string. + zone: string to a valid GCP zone. + machine_type: type of GCP to create (e.g. n1-standard-4). + installers: list of installers post-boot. + ssh_key_file: path to a valid ssh private key. See README on vaild ssh keys. + ssh_user: string of user name for ssh_key + ssh_password: string of password for ssh key + internal: if true, use internal IPs of instances. Used if bm-tools is + running on a GCP vm when a firewall is set for external IPs. + mock: a mock printer which will print mock data if required. Mock data is + recorded output from subprocess calls (returncode, stdout, args). + condition: mutex for this class around machine creation and deleteion. + """ + + def __init__(self, + image: str, + zone: str, + machine_type: str, + installers: List[str], + ssh_key_file: str, + ssh_user: str, + ssh_password: str, + internal: bool, + mock: gcloud_mock_recorder.MockPrinter = None): + self.image = image + self.zone = zone + self.machine_type = machine_type + self.installers = installers + self.ssh_key_file = ssh_key_file + self.ssh_user = ssh_user + self.ssh_password = ssh_password + self.internal = internal + self.mock = mock + self.condition = threading.Condition() + + def get_machines(self, num_machines: int) -> List[machine.Machine]: + """Returns requested number of machines backed by GCP instances.""" + if num_machines <= 0: + raise ValueError( + "Cannot ask for {num} machines!".format(num=num_machines)) + with self.condition: + names = self._get_unique_names(num_machines) + instances = self._build_instances(names) + self._add_ssh_key_to_instances(names) + machines = self._machines_from_instances(instances) + + # Install all bits in lock-step. + # + # This will perform paralell installations for however many machines we + # have, but it's easy to track errors because if installing (a, b, c), we + # won't install "c" until "b" is installed on all machines. + for installer in self.installers: + threads = [None] * len(machines) + results = [False] * len(machines) + for i in range(len(machines)): + threads[i] = threading.Thread( + target=machines[i].install, args=(installer, results, i)) + threads[i].start() + for thread in threads: + thread.join() + for result in results: + if not result: + raise NotImplementedError( + "Installers failed on at least one machine!") + + # Add this user to each machine's docker group. + for m in machines: + m.run("sudo setfacl -m user:$USER:rw /var/run/docker.sock") + + return machines + + def release_machines(self, machine_list: List[machine.Machine]): + """Releases the requested number of machines, deleting the instances.""" + if not machine_list: + return + cmd = "gcloud compute instances delete --quiet".split(" ") + names = [str(m) for m in machine_list] + cmd.extend(names) + cmd.append("--zone={zone}".format(zone=self.zone)) + self._run_command(cmd, detach=True) + + def _machines_from_instances( + self, instances: List[Dict[str, Any]]) -> List[machine.Machine]: + """Creates Machine Objects from json data describing created instances.""" + machines = [] + for instance in instances: + name = instance["name"] + external = instance["networkInterfaces"][0]["accessConfigs"][0]["natIP"] + internal = instance["networkInterfaces"][0]["networkIP"] + kwargs = { + "hostname": internal if self.internal else external, + "key_path": self.ssh_key_file, + "username": self.ssh_user, + "key_password": self.ssh_password + } + machines.append(machine.RemoteMachine(name=name, **kwargs)) + return machines + + def _get_unique_names(self, num_names) -> List[str]: + """Returns num_names unique names based on data from the GCP project.""" + return ["machine-" + str(uuid.uuid4()) for _ in range(0, num_names)] + + def _build_instances(self, names: List[str]) -> List[Dict[str, Any]]: + """Creates instances using gcloud command. + + Runs the command `gcloud compute instances create` and returns json data + on created instances on success. Creates len(names) instances, one for each + name. + + Args: + names: list of names of instances to create. + + Returns: + List of json data describing created machines. + """ + if not names: + raise ValueError( + "_build_instances cannot create instances without names.") + cmd = "gcloud compute instances create".split(" ") + cmd.extend(names) + cmd.append("--image=" + self.image) + cmd.append("--zone=" + self.zone) + cmd.append("--machine-type=" + self.machine_type) + res = self._run_command(cmd) + data = res.stdout + data = str(data, "utf-8") if isinstance(data, (bytes, bytearray)) else data + return json.loads(data) + + def _add_ssh_key_to_instances(self, names: List[str]) -> None: + """Adds ssh key to instances by calling gcloud ssh command. + + Runs the command `gcloud compute ssh instance_name` on list of images by + name. Tries to ssh into given instance. + + Args: + names: list of machine names to which to add the ssh-key + self.ssh_key_file. + + Raises: + subprocess.CalledProcessError: when underlying subprocess call returns an + error other than 255 (Connection closed by remote host). + TimeoutError: when 3 unsuccessful tries to ssh into the host return 255. + """ + for name in names: + cmd = "gcloud compute ssh {user}@{name}".format( + user=self.ssh_user, name=name).split(" ") + if self.internal: + cmd.append("--internal-ip") + cmd.append("--ssh-key-file={key}".format(key=self.ssh_key_file)) + cmd.append("--zone={zone}".format(zone=self.zone)) + cmd.append("--command=uname") + timeout = datetime.timedelta(seconds=5 * 60) + start = datetime.datetime.now() + while datetime.datetime.now() <= timeout + start: + try: + self._run_command(cmd) + break + except subprocess.CalledProcessError: + if datetime.datetime.now() > timeout + start: + raise TimeoutError( + "Could not SSH into instance after 5 min: {name}".format( + name=name)) + + def _run_command(self, + cmd: List[str], + detach: bool = False) -> [None, subprocess.CompletedProcess]: + """Runs command as a subprocess. + + Runs command as subprocess and returns the result. + If this has a mock recorder, use the record method to record the subprocess + call. + + Args: + cmd: command to be run as a list of strings. + detach: if True, run the child process and don't wait for it to return. + + Returns: + Completed process object to be parsed by caller or None if detach=True. + + Raises: + CalledProcessError: if subprocess.run returns an error. + """ + cmd = cmd + ["--format=json"] + if detach: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if self.mock: + out, _ = p.communicate() + self.mock.record( + subprocess.CompletedProcess( + returncode=p.returncode, stdout=out, args=p.args)) + return + + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if self.mock: + self.mock.record(res) + if res.returncode != 0: + raise subprocess.CalledProcessError( + cmd=" ".join(res.args), + output=res.stdout, + stderr=res.stderr, + returncode=res.returncode) + return res diff --git a/benchmarks/harness/machine_producers/gcloud_producer_test.py b/benchmarks/harness/machine_producers/gcloud_producer_test.py new file mode 100644 index 000000000..c8adb2bdc --- /dev/null +++ b/benchmarks/harness/machine_producers/gcloud_producer_test.py @@ -0,0 +1,48 @@ +# python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests GCloudProducer using mock data. + +GCloudProducer produces machines using 'get_machines' and 'release_machines' +methods. The tests check recorded data (jsonified subprocess.CompletedProcess +objects) of the producer producing one and five machines. +""" +import os +import types + +from benchmarks.harness.machine_producers import machine_producer +from benchmarks.harness.machine_producers import mock_producer + +TEST_DIR = os.path.dirname(__file__) + + +def run_get_release(producer: machine_producer.MachineProducer, + num_machines: int, + validator: types.FunctionType = None): + machines = producer.get_machines(num_machines) + assert len(machines) == num_machines + if validator: + validator(machines=machines, cmd="uname -a", workload=None) + producer.release_machines(machines) + + +def test_run_one(): + mock = mock_producer.MockReader(TEST_DIR + "get_one.json") + producer = mock_producer.MockGCloudProducer(mock) + run_get_release(producer, 1) + + +def test_run_five(): + mock = mock_producer.MockReader(TEST_DIR + "get_five.json") + producer = mock_producer.MockGCloudProducer(mock) + run_get_release(producer, 5) diff --git a/benchmarks/harness/machine_producers/machine_producer.py b/benchmarks/harness/machine_producers/machine_producer.py new file mode 100644 index 000000000..f5591c026 --- /dev/null +++ b/benchmarks/harness/machine_producers/machine_producer.py @@ -0,0 +1,51 @@ +# python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Abstract types.""" + +import threading +from typing import List + +from benchmarks.harness import machine + + +class MachineProducer: + """Abstract Machine producer.""" + + def get_machines(self, num_machines: int) -> List[machine.Machine]: + """Returns the requested number of machines.""" + raise NotImplementedError + + def release_machines(self, machine_list: List[machine.Machine]): + """Releases the given set of machines.""" + raise NotImplementedError + + +class LocalMachineProducer(MachineProducer): + """Produces Local Machines.""" + + def __init__(self, limit: int): + self.limit_sem = threading.Semaphore(value=limit) + + def get_machines(self, num_machines: int) -> List[machine.Machine]: + """Returns the request number of MockMachines.""" + + self.limit_sem.acquire() + return [machine.LocalMachine("local") for _ in range(num_machines)] + + def release_machines(self, machine_list: List[machine.MockMachine]): + """No-op.""" + if not machine_list: + raise ValueError("Cannot release an empty list!") + self.limit_sem.release() + machine_list.clear() diff --git a/benchmarks/harness/machine_producers/mock_producer.py b/benchmarks/harness/machine_producers/mock_producer.py new file mode 100644 index 000000000..37e9cb4b7 --- /dev/null +++ b/benchmarks/harness/machine_producers/mock_producer.py @@ -0,0 +1,52 @@ +# python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Producers of mocks.""" + +from typing import List, Any + +from benchmarks.harness import machine +from benchmarks.harness.machine_producers import gcloud_mock_recorder +from benchmarks.harness.machine_producers import gcloud_producer +from benchmarks.harness.machine_producers import machine_producer + + +class MockMachineProducer(machine_producer.MachineProducer): + """Produces MockMachine objects.""" + + def get_machines(self, num_machines: int) -> List[machine.MockMachine]: + """Returns the request number of MockMachines.""" + return [machine.MockMachine() for i in range(num_machines)] + + def release_machines(self, machine_list: List[machine.MockMachine]): + """No-op.""" + return + + +class MockGCloudProducer(gcloud_producer.GCloudProducer): + """Mocks GCloudProducer for testing purposes.""" + + def __init__(self, mock: gcloud_mock_recorder.MockReader, **kwargs): + gcloud_producer.GCloudProducer.__init__( + self, project="mock", ssh_private_key_path="mock", **kwargs) + self.mock = mock + + def _validate_ssh_file(self): + pass + + def _run_command(self, cmd): + return self.mock.pop(cmd) + + def _machines_from_instances( + self, instances: List[Any]) -> List[machine.MockMachine]: + return [machine.MockMachine() for _ in instances] diff --git a/benchmarks/harness/machine_producers/testdata/get_five.json b/benchmarks/harness/machine_producers/testdata/get_five.json new file mode 100644 index 000000000..32bad1b06 --- /dev/null +++ b/benchmarks/harness/machine_producers/testdata/get_five.json @@ -0,0 +1,211 @@ +[ + { + "args": [ + "gcloud", + "compute", + "instances", + "list", + "--project", + "project", + "--format=json" + ], + "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":{\"natIP\":\"0.0.0.0\"}]}]}]", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "instances", + "create", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92", + "machine-da5859b5-bae6-435d-8005-0202d6f6e065", + "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05", + "machine-1149147d-71e2-43ea-8fe1-49256e5c441c", + "--preemptible", + "--image=ubuntu-1910-eoan-v20191204", + "--zone=us-west1-b", + "--image-project=ubuntu-os-cloud", + "--format=json" + ], + "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "instances", + "start", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92", + "machine-da5859b5-bae6-435d-8005-0202d6f6e065", + "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05", + "machine-1149147d-71e2-43ea-8fe1-49256e5c441c", + "--zone=us-west1-b", + "--project=project", + "--format=json" + ], + "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]},{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "Linux\n[]\n", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "Linux\n[]\n", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-da5859b5-bae6-435d-8005-0202d6f6e065", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "Linux\n[]\n", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "Linux\n[]\n", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-1149147d-71e2-43ea-8fe1-49256e5c441c", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "Linux\n[]\n", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "instances", + "delete", + "--quiet", + "machine-42c9bf6e-8d45-4c37-b1c0-7e4fdcf530fc", + "machine-5f28f145-cc2d-427d-9cbf-428d164cdb92", + "machine-da5859b5-bae6-435d-8005-0202d6f6e065", + "machine-880a8a2f-918c-4f9e-a43c-ed3c8e02ea05", + "machine-1149147d-71e2-43ea-8fe1-49256e5c441c", + "--zone=us-west1-b", + "--format=json" + ], + "stdout": "[]\n", + "returncode": "0" + } +] diff --git a/benchmarks/harness/machine_producers/testdata/get_one.json b/benchmarks/harness/machine_producers/testdata/get_one.json new file mode 100644 index 000000000..c359c19c8 --- /dev/null +++ b/benchmarks/harness/machine_producers/testdata/get_one.json @@ -0,0 +1,145 @@ +[ + { + "args": [ + "gcloud", + "compute", + "instances", + "list", + "--project", + "linux-testing-user", + "--format=json" + ], + "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]", + + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "instances", + "create", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--preemptible", + "--image=ubuntu-1910-eoan-v20191204", + "--zone=us-west1-b", + "--image-project=ubuntu-os-cloud", + "--format=json" + ], + "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "instances", + "start", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--zone=us-west1-b", + "--project=linux-testing-user", + "--format=json" + ], + "stdout": "[{\"name\":\"name\", \"networkInterfaces\":[{\"accessConfigs\":[{\"natIP\":\"0.0.0.0\"}]}]}]", + + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "", + "returncode": "255" + }, + { + "args": [ + "gcloud", + "compute", + "ssh", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--ssh-key-file=/usr/local/google/home/user/.ssh/benchmark-tools", + "--zone=us-west1-b", + "--command=uname", + "--format=json" + ], + "stdout": "Linux\n[]\n", + "returncode": "0" + }, + { + "args": [ + "gcloud", + "compute", + "instances", + "delete", + "--quiet", + "machine-129dfcf9-b05b-4c16-a4cd-21353b570ddc", + "--zone=us-west1-b", + "--format=json" + ], + "stdout": "[]\n", + "returncode": "0" + } +] diff --git a/benchmarks/harness/machine_producers/yaml_producer.py b/benchmarks/harness/machine_producers/yaml_producer.py new file mode 100644 index 000000000..5d334e480 --- /dev/null +++ b/benchmarks/harness/machine_producers/yaml_producer.py @@ -0,0 +1,106 @@ +# python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Producers based on yaml files.""" + +import os +import threading +from typing import Dict +from typing import List + +import yaml + +from benchmarks.harness import machine +from benchmarks.harness.machine_producers import machine_producer + + +class YamlMachineProducer(machine_producer.MachineProducer): + """Loads machines from a yaml file.""" + + def __init__(self, path: str): + self.machines = build_machines(path) + self.max_machines = len(self.machines) + self.machine_condition = threading.Condition() + + def get_machines(self, num_machines: int) -> List[machine.Machine]: + if num_machines > self.max_machines: + raise ValueError( + "Insufficient Ammount of Machines. {ask} asked for and have {max_num} max." + .format(ask=num_machines, max_num=self.max_machines)) + + with self.machine_condition: + while not self._enough_machines(num_machines): + self.machine_condition.wait(timeout=1) + return [self.machines.pop(0) for _ in range(num_machines)] + + def release_machines(self, machine_list: List[machine.Machine]): + with self.machine_condition: + while machine_list: + next_machine = machine_list.pop() + self.machines.append(next_machine) + self.machine_condition.notify() + + def _enough_machines(self, ask: int): + return ask <= len(self.machines) + + +def build_machines(path: str, num_machines: str = -1) -> List[machine.Machine]: + """Builds machine objects defined by the yaml file "path". + + Args: + path: The path to a yaml file which defines machines. + num_machines: Optional limit on how many machine objects to build. + + Returns: + Machine objects in a list. + + If num_machines is set, len(machines) <= num_machines. + """ + data = parse_yaml(path) + machines = [] + for key, value in data.items(): + if len(machines) == num_machines: + return machines + if isinstance(value, dict): + machines.append(machine.RemoteMachine(key, **value)) + else: + machines.append(machine.LocalMachine(key)) + return machines + + +def parse_yaml(path: str) -> Dict[str, Dict[str, str]]: + """Parse the yaml file pointed by path. + + Args: + path: The path to yaml file. + + Returns: + The contents of the yaml file as a dictionary. + """ + data = get_file_contents(path) + return yaml.load(data, Loader=yaml.Loader) + + +def get_file_contents(path: str) -> str: + """Dumps the file contents to a string and returns them. + + Args: + path: The path to dump. + + Returns: + The file contents as a string. + """ + if not os.path.isabs(path): + path = os.path.abspath(path) + with open(path) as input_file: + return input_file.read() |