From 77552f1c770da7413c3c8212443328c9081901c0 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Mon, 27 Jul 2020 10:06:07 -0700 Subject: Port ffmpeg benchmark PiperOrigin-RevId: 323383320 --- test/benchmarks/media/BUILD | 21 +++++++++++++++ test/benchmarks/media/ffmpeg_test.go | 52 ++++++++++++++++++++++++++++++++++++ test/benchmarks/media/media.go | 31 +++++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 test/benchmarks/media/BUILD create mode 100644 test/benchmarks/media/ffmpeg_test.go create mode 100644 test/benchmarks/media/media.go (limited to 'test/benchmarks/media') diff --git a/test/benchmarks/media/BUILD b/test/benchmarks/media/BUILD new file mode 100644 index 000000000..6c41fc4f6 --- /dev/null +++ b/test/benchmarks/media/BUILD @@ -0,0 +1,21 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "media", + testonly = 1, + srcs = ["media.go"], + deps = ["//test/benchmarks/harness"], +) + +go_test( + name = "media_test", + size = "large", + srcs = ["ffmpeg_test.go"], + library = ":media", + deps = [ + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + ], +) diff --git a/test/benchmarks/media/ffmpeg_test.go b/test/benchmarks/media/ffmpeg_test.go new file mode 100644 index 000000000..bfcfbab80 --- /dev/null +++ b/test/benchmarks/media/ffmpeg_test.go @@ -0,0 +1,52 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package media + +import ( + "context" + "strings" + "testing" + + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/harness" +) + +// BenchmarkFfmpeg runs ffmpeg in a container and records runtime. +// BenchmarkFfmpeg should run as root to drop caches. +func BenchmarkFfmpeg(b *testing.B) { + machine, err := h.GetMachine() + if err != nil { + b.Fatalf("failed to get machine: %v", err) + } + defer machine.CleanUp() + + ctx := context.Background() + container := machine.GetContainer(ctx, b) + cmd := strings.Split("ffmpeg -i video.mp4 -c:v libx264 -preset veryslow output.mp4", " ") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + if err := harness.DropCaches(machine); err != nil { + b.Skipf("failed to drop caches: %v. You probably need root.", err) + } + b.StartTimer() + + if _, err := container.Run(ctx, dockerutil.RunOpts{ + Image: "benchmarks/ffmpeg", + }, cmd...); err != nil { + b.Fatalf("failed to run container: %v", err) + } + } +} diff --git a/test/benchmarks/media/media.go b/test/benchmarks/media/media.go new file mode 100644 index 000000000..c7b35b758 --- /dev/null +++ b/test/benchmarks/media/media.go @@ -0,0 +1,31 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package media holds benchmarks around media processing applications. +package media + +import ( + "os" + "testing" + + "gvisor.dev/gvisor/test/benchmarks/harness" +) + +var h harness.Harness + +// TestMain is the main method for package media. +func TestMain(m *testing.M) { + h.Init() + os.Exit(m.Run()) +} -- cgit v1.2.3 From 5873b0f43f5ff4d811f951baaacb7bbe8b1a486c Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Tue, 28 Jul 2020 12:51:54 -0700 Subject: Port tensorflow benchmark. PiperOrigin-RevId: 323633737 --- images/benchmarks/tensorflow/Dockerfile | 7 ++++ test/benchmarks/media/ffmpeg_test.go | 1 + test/benchmarks/ml/BUILD | 21 ++++++++++ test/benchmarks/ml/ml.go | 31 +++++++++++++++ test/benchmarks/ml/tensorflow_test.go | 69 +++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+) create mode 100644 images/benchmarks/tensorflow/Dockerfile create mode 100644 test/benchmarks/ml/BUILD create mode 100644 test/benchmarks/ml/ml.go create mode 100644 test/benchmarks/ml/tensorflow_test.go (limited to 'test/benchmarks/media') diff --git a/images/benchmarks/tensorflow/Dockerfile b/images/benchmarks/tensorflow/Dockerfile new file mode 100644 index 000000000..7564a4ee5 --- /dev/null +++ b/images/benchmarks/tensorflow/Dockerfile @@ -0,0 +1,7 @@ +FROM tensorflow/tensorflow:1.13.2 + +RUN apt-get update \ + && apt-get install -y git +RUN git clone --depth 1 https://github.com/aymericdamien/TensorFlow-Examples.git +RUN python -m pip install -U pip setuptools +RUN python -m pip install matplotlib diff --git a/test/benchmarks/media/ffmpeg_test.go b/test/benchmarks/media/ffmpeg_test.go index bfcfbab80..7822dfad7 100644 --- a/test/benchmarks/media/ffmpeg_test.go +++ b/test/benchmarks/media/ffmpeg_test.go @@ -33,6 +33,7 @@ func BenchmarkFfmpeg(b *testing.B) { ctx := context.Background() container := machine.GetContainer(ctx, b) + defer container.CleanUp(ctx) cmd := strings.Split("ffmpeg -i video.mp4 -c:v libx264 -preset veryslow output.mp4", " ") b.ResetTimer() diff --git a/test/benchmarks/ml/BUILD b/test/benchmarks/ml/BUILD new file mode 100644 index 000000000..2430b60a7 --- /dev/null +++ b/test/benchmarks/ml/BUILD @@ -0,0 +1,21 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "ml", + testonly = 1, + srcs = ["ml.go"], + deps = ["//test/benchmarks/harness"], +) + +go_test( + name = "ml_test", + size = "large", + srcs = ["tensorflow_test.go"], + library = ":ml", + deps = [ + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + ], +) diff --git a/test/benchmarks/ml/ml.go b/test/benchmarks/ml/ml.go new file mode 100644 index 000000000..13282d7bb --- /dev/null +++ b/test/benchmarks/ml/ml.go @@ -0,0 +1,31 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ml holds benchmarks around machine learning performance. +package ml + +import ( + "os" + "testing" + + "gvisor.dev/gvisor/test/benchmarks/harness" +) + +var h harness.Harness + +// TestMain is the main method for package ml. +func TestMain(m *testing.M) { + h.Init() + os.Exit(m.Run()) +} diff --git a/test/benchmarks/ml/tensorflow_test.go b/test/benchmarks/ml/tensorflow_test.go new file mode 100644 index 000000000..f7746897d --- /dev/null +++ b/test/benchmarks/ml/tensorflow_test.go @@ -0,0 +1,69 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ml + +import ( + "context" + "testing" + + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/harness" +) + +// BenchmarkTensorflow runs workloads from a TensorFlow tutorial. +// See: https://github.com/aymericdamien/TensorFlow-Examples +func BenchmarkTensorflow(b *testing.B) { + workloads := map[string]string{ + "GradientDecisionTree": "2_BasicModels/gradient_boosted_decision_tree.py", + "Kmeans": "2_BasicModels/kmeans.py", + "LogisticRegression": "2_BasicModels/logistic_regression.py", + "NearestNeighbor": "2_BasicModels/nearest_neighbor.py", + "RandomForest": "2_BasicModels/random_forest.py", + "ConvolutionalNetwork": "3_NeuralNetworks/convolutional_network.py", + "MultilayerPerceptron": "3_NeuralNetworks/multilayer_perceptron.py", + "NeuralNetwork": "3_NeuralNetworks/neural_network.py", + } + + machine, err := h.GetMachine() + if err != nil { + b.Fatalf("failed to get machine: %v", err) + } + defer machine.CleanUp() + + for name, workload := range workloads { + b.Run(name, func(b *testing.B) { + ctx := context.Background() + container := machine.GetContainer(ctx, b) + defer container.CleanUp(ctx) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + if err := harness.DropCaches(machine); err != nil { + b.Skipf("failed to drop caches: %v. You probably need root.", err) + } + b.StartTimer() + + if out, err := container.Run(ctx, dockerutil.RunOpts{ + Image: "benchmarks/tensorflow", + Env: []string{"PYTHONPATH=$PYTHONPATH:/TensorFlow-Examples/examples"}, + WorkDir: "/TensorFlow-Examples/examples", + }, "python", workload); err != nil { + b.Fatalf("failed to run container: %v logs: %s", err, out) + } + } + }) + } + +} -- cgit v1.2.3 From a88cf5a2e18869acb636f696700ecce2781be818 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Mon, 10 Aug 2020 14:51:07 -0700 Subject: Add benchmarks to continuous build. PiperOrigin-RevId: 325892974 --- BUILD | 6 ++++++ test/benchmarks/base/BUILD | 1 + test/benchmarks/database/BUILD | 1 + test/benchmarks/fs/BUILD | 1 + test/benchmarks/media/BUILD | 1 + test/benchmarks/ml/BUILD | 1 + test/benchmarks/network/BUILD | 1 + 7 files changed, 12 insertions(+) (limited to 'test/benchmarks/media') diff --git a/BUILD b/BUILD index f3d0eec4e..2639f8169 100644 --- a/BUILD +++ b/BUILD @@ -57,6 +57,12 @@ build_test( "//test/e2e:integration_test", "//test/image:image_test", "//test/root:root_test", + "//test/benchmarks/base:base_test", + "//test/benchmarks/database:database_test", + "//test/benchmarks/fs:fs_test", + "//test/benchmarks/media:media_test", + "//test/benchmarks/ml:ml_test", + "//test/benchmarks/network:network_test", ], ) diff --git a/test/benchmarks/base/BUILD b/test/benchmarks/base/BUILD index 5e099d0f9..32c139204 100644 --- a/test/benchmarks/base/BUILD +++ b/test/benchmarks/base/BUILD @@ -25,6 +25,7 @@ go_test( "manual", "local", ], + visibility = ["//:sandbox"], deps = [ "//pkg/test/dockerutil", "//test/benchmarks/harness", diff --git a/test/benchmarks/database/BUILD b/test/benchmarks/database/BUILD index 6139f6e8a..93b380e8a 100644 --- a/test/benchmarks/database/BUILD +++ b/test/benchmarks/database/BUILD @@ -19,6 +19,7 @@ go_test( "manual", "local", ], + visibility = ["//:sandbox"], deps = [ "//pkg/test/dockerutil", "//test/benchmarks/harness", diff --git a/test/benchmarks/fs/BUILD b/test/benchmarks/fs/BUILD index 20654d88f..45f11372b 100644 --- a/test/benchmarks/fs/BUILD +++ b/test/benchmarks/fs/BUILD @@ -22,6 +22,7 @@ go_test( "local", "manual", ], + visibility = ["//:sandbox"], deps = [ "//pkg/test/dockerutil", "//test/benchmarks/harness", diff --git a/test/benchmarks/media/BUILD b/test/benchmarks/media/BUILD index 6c41fc4f6..bb242d385 100644 --- a/test/benchmarks/media/BUILD +++ b/test/benchmarks/media/BUILD @@ -14,6 +14,7 @@ go_test( size = "large", srcs = ["ffmpeg_test.go"], library = ":media", + visibility = ["//:sandbox"], deps = [ "//pkg/test/dockerutil", "//test/benchmarks/harness", diff --git a/test/benchmarks/ml/BUILD b/test/benchmarks/ml/BUILD index 2430b60a7..970f52706 100644 --- a/test/benchmarks/ml/BUILD +++ b/test/benchmarks/ml/BUILD @@ -14,6 +14,7 @@ go_test( size = "large", srcs = ["tensorflow_test.go"], library = ":ml", + visibility = ["//:sandbox"], deps = [ "//pkg/test/dockerutil", "//test/benchmarks/harness", diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD index df5ff7265..bd3f6245c 100644 --- a/test/benchmarks/network/BUILD +++ b/test/benchmarks/network/BUILD @@ -25,6 +25,7 @@ go_test( "manual", "local", ], + visibility = ["//:sandbox"], deps = [ "//pkg/test/dockerutil", "//pkg/test/testutil", -- cgit v1.2.3