go_marshal: Implement automatic generation of ABI marshalling code.

This CL implements go_marshal, a code generation utility for
automatically serializing and deserializing ABI structs.

The go_marshal tool automatically generates implementations of the new
marshal interface. Unlike binary.Marshal/Unmarshal, the generated
interface implementations use no runtime reflection, and translates to
a single memcpy for most structs. See go_marshal/README.md for
details.

PiperOrigin-RevId: 268065475

5 files changed, 346 insertions, 0 deletions

diff --git a/tools/go_marshal/test/BUILD b/tools/go_marshal/test/BUILD
new file mode 100644
index 000000000..947011414
--- /dev/null
+++ b/tools/go_marshal/test/BUILD
@@ -0,0 +1,29 @@
+package(licenses = ["notice"])
+
+load("//tools/go_marshal:defs.bzl", "go_library", "go_test")
+
+package_group(
+    name = "gomarshal_test",
+    packages = [
+        "//tools/go_marshal/test/...",
+    ],
+)
+
+go_test(
+    name = "benchmark_test",
+    srcs = ["benchmark_test.go"],
+    deps = [
+        ":test",
+        "//pkg/binary",
+        "//pkg/sentry/usermem",
+        "//tools/go_marshal/analysis",
+    ],
+)
+
+go_library(
+    name = "test",
+    testonly = 1,
+    srcs = ["test.go"],
+    importpath = "gvisor.dev/gvisor/tools/go_marshal/test",
+    deps = ["//tools/go_marshal/test/external"],
+)
diff --git a/tools/go_marshal/test/benchmark_test.go b/tools/go_marshal/test/benchmark_test.go
new file mode 100644
index 000000000..e70db06d8
--- /dev/null
+++ b/tools/go_marshal/test/benchmark_test.go
@@ -0,0 +1,178 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_test
+
+import (
+	"bytes"
+	encbin "encoding/binary"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/binary"
+	"gvisor.dev/gvisor/pkg/sentry/usermem"
+	"gvisor.dev/gvisor/tools/go_marshal/analysis"
+	test "gvisor.dev/gvisor/tools/go_marshal/test"
+)
+
+// Marshalling using the standard encoding/binary package.
+func BenchmarkEncodingBinary(b *testing.B) {
+	var s1, s2 test.Stat
+	analysis.RandomizeValue(&s1)
+
+	size := encbin.Size(&s1)
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		buf := bytes.NewBuffer(make([]byte, size))
+		buf.Reset()
+		if err := encbin.Write(buf, usermem.ByteOrder, &s1); err != nil {
+			b.Error("Write:", err)
+		}
+		if err := encbin.Read(buf, usermem.ByteOrder, &s2); err != nil {
+			b.Error("Read:", err)
+		}
+	}
+
+	b.StopTimer()
+
+	// Sanity check, make sure the values were preserved.
+	if !reflect.DeepEqual(s1, s2) {
+		panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2))
+	}
+}
+
+// Marshalling using the sentry's binary.Marshal.
+func BenchmarkBinary(b *testing.B) {
+	var s1, s2 test.Stat
+	analysis.RandomizeValue(&s1)
+
+	size := binary.Size(s1)
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		buf := make([]byte, 0, size)
+		buf = binary.Marshal(buf, usermem.ByteOrder, &s1)
+		binary.Unmarshal(buf, usermem.ByteOrder, &s2)
+	}
+
+	b.StopTimer()
+
+	// Sanity check, make sure the values were preserved.
+	if !reflect.DeepEqual(s1, s2) {
+		panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2))
+	}
+}
+
+// Marshalling field-by-field with manually-written code.
+func BenchmarkMarshalManual(b *testing.B) {
+	var s1, s2 test.Stat
+	analysis.RandomizeValue(&s1)
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		buf := make([]byte, 0, s1.SizeBytes())
+
+		// Marshal
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, s1.Dev)
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, s1.Ino)
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, s1.Nlink)
+		buf = binary.AppendUint32(buf, usermem.ByteOrder, s1.Mode)
+		buf = binary.AppendUint32(buf, usermem.ByteOrder, s1.UID)
+		buf = binary.AppendUint32(buf, usermem.ByteOrder, s1.GID)
+		buf = binary.AppendUint32(buf, usermem.ByteOrder, 0)
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, s1.Rdev)
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.Size))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.Blksize))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.Blocks))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.ATime.Sec))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.ATime.Nsec))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.MTime.Sec))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.MTime.Nsec))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.CTime.Sec))
+		buf = binary.AppendUint64(buf, usermem.ByteOrder, uint64(s1.CTime.Nsec))
+
+		// Unmarshal
+		s2.Dev = usermem.ByteOrder.Uint64(buf[0:8])
+		s2.Ino = usermem.ByteOrder.Uint64(buf[8:16])
+		s2.Nlink = usermem.ByteOrder.Uint64(buf[16:24])
+		s2.Mode = usermem.ByteOrder.Uint32(buf[24:28])
+		s2.UID = usermem.ByteOrder.Uint32(buf[28:32])
+		s2.GID = usermem.ByteOrder.Uint32(buf[32:36])
+		// Padding: buf[36:40]
+		s2.Rdev = usermem.ByteOrder.Uint64(buf[40:48])
+		s2.Size = int64(usermem.ByteOrder.Uint64(buf[48:56]))
+		s2.Blksize = int64(usermem.ByteOrder.Uint64(buf[56:64]))
+		s2.Blocks = int64(usermem.ByteOrder.Uint64(buf[64:72]))
+		s2.ATime.Sec = int64(usermem.ByteOrder.Uint64(buf[72:80]))
+		s2.ATime.Nsec = int64(usermem.ByteOrder.Uint64(buf[80:88]))
+		s2.MTime.Sec = int64(usermem.ByteOrder.Uint64(buf[88:96]))
+		s2.MTime.Nsec = int64(usermem.ByteOrder.Uint64(buf[96:104]))
+		s2.CTime.Sec = int64(usermem.ByteOrder.Uint64(buf[104:112]))
+		s2.CTime.Nsec = int64(usermem.ByteOrder.Uint64(buf[112:120]))
+	}
+
+	b.StopTimer()
+
+	// Sanity check, make sure the values were preserved.
+	if !reflect.DeepEqual(s1, s2) {
+		panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2))
+	}
+}
+
+// Marshalling with the go_marshal safe API.
+func BenchmarkGoMarshalSafe(b *testing.B) {
+	var s1, s2 test.Stat
+	analysis.RandomizeValue(&s1)
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		buf := make([]byte, s1.SizeBytes())
+		s1.MarshalBytes(buf)
+		s2.UnmarshalBytes(buf)
+	}
+
+	b.StopTimer()
+
+	// Sanity check, make sure the values were preserved.
+	if !reflect.DeepEqual(s1, s2) {
+		panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2))
+	}
+}
+
+// Marshalling with the go_marshal unsafe API.
+func BenchmarkGoMarshalUnsafe(b *testing.B) {
+	var s1, s2 test.Stat
+	analysis.RandomizeValue(&s1)
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		buf := make([]byte, s1.SizeBytes())
+		s1.MarshalUnsafe(buf)
+		s2.UnmarshalUnsafe(buf)
+	}
+
+	b.StopTimer()
+
+	// Sanity check, make sure the values were preserved.
+	if !reflect.DeepEqual(s1, s2) {
+		panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2))
+	}
+}
diff --git a/tools/go_marshal/test/external/BUILD b/tools/go_marshal/test/external/BUILD
new file mode 100644
index 000000000..8fb43179b
--- /dev/null
+++ b/tools/go_marshal/test/external/BUILD
@@ -0,0 +1,11 @@
+package(licenses = ["notice"])
+
+load("//tools/go_marshal:defs.bzl", "go_library")
+
+go_library(
+    name = "external",
+    testonly = 1,
+    srcs = ["external.go"],
+    importpath = "gvisor.dev/gvisor/tools/go_marshal/test/external",
+    visibility = ["//tools/go_marshal/test:gomarshal_test"],
+)
diff --git a/tools/go_marshal/test/external/external.go b/tools/go_marshal/test/external/external.go
new file mode 100644
index 000000000..4be3722f3
--- /dev/null
+++ b/tools/go_marshal/test/external/external.go
@@ -0,0 +1,23 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package external defines types we can import for testing.
+package external
+
+// External is a public Marshallable type for use in testing.
+//
+// +marshal
+type External struct {
+	j int64
+}
diff --git a/tools/go_marshal/test/test.go b/tools/go_marshal/test/test.go
new file mode 100644
index 000000000..8de02d707
--- /dev/null
+++ b/tools/go_marshal/test/test.go
@@ -0,0 +1,105 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package test contains data structures for testing the go_marshal tool.
+package test
+
+import (
+	// We're intentionally using a package name alias here even though it's not
+	// necessary to test the code generator's ability to handle package aliases.
+	ex "gvisor.dev/gvisor/tools/go_marshal/test/external"
+)
+
+// Type1 is a test data type.
+//
+// +marshal
+type Type1 struct {
+	a    Type2
+	x, y int64 // Multiple field names.
+	b    byte  `marshal:"unaligned"` // Short field.
+	c    uint64
+	_    uint32  // Unnamed scalar field.
+	_    [6]byte // Unnamed vector field, typical padding.
+	_    [2]byte
+	xs   [8]int32
+	as   [10]Type2 `marshal:"unaligned"` // Array of Marshallable objects.
+	ss   Type3
+}
+
+// Type2 is a test data type.
+//
+// +marshal
+type Type2 struct {
+	n int64
+	c byte
+	_ [7]byte
+	m int64
+	a int64
+}
+
+// Type3 is a test data type.
+//
+// +marshal
+type Type3 struct {
+	s int64
+	x ex.External // Type defined in another package.
+}
+
+// Type4 is a test data type.
+//
+// +marshal
+type Type4 struct {
+	c byte
+	x int64 `marshal:"unaligned"`
+	d byte
+	_ [7]byte
+}
+
+// Type5 is a test data type.
+//
+// +marshal
+type Type5 struct {
+	n int64
+	t Type4
+	m int64
+}
+
+// Timespec represents struct timespec in <time.h>.
+//
+// +marshal
+type Timespec struct {
+	Sec  int64
+	Nsec int64
+}
+
+// Stat represents struct stat.
+//
+// +marshal
+type Stat struct {
+	Dev     uint64
+	Ino     uint64
+	Nlink   uint64
+	Mode    uint32
+	UID     uint32
+	GID     uint32
+	_       int32
+	Rdev    uint64
+	Size    int64
+	Blksize int64
+	Blocks  int64
+	ATime   Timespec
+	MTime   Timespec
+	CTime   Timespec
+	_       [3]int64
+}