diff options
Diffstat (limited to 'test/e2e/integration_test.go')
| -rw-r--r-- | test/e2e/integration_test.go | 719 |
1 files changed, 0 insertions, 719 deletions
diff --git a/test/e2e/integration_test.go b/test/e2e/integration_test.go deleted file mode 100644 index f53417cab..000000000 --- a/test/e2e/integration_test.go +++ /dev/null @@ -1,719 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package integration provides end-to-end integration tests for runsc. -// -// Each test calls docker commands to start up a container, and tests that it is -// behaving properly, with various runsc commands. The container is killed and -// deleted at the end. -// -// Setup instruction in test/README.md. -package integration - -import ( - "context" - "flag" - "fmt" - "io/ioutil" - "net" - "net/http" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "testing" - "time" - - "github.com/docker/docker/api/types/mount" - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// defaultWait is the default wait time used for tests. -const defaultWait = time.Minute - -// httpRequestSucceeds sends a request to a given url and checks that the status is OK. -func httpRequestSucceeds(client http.Client, server string, port int) error { - url := fmt.Sprintf("http://%s:%d", server, port) - // Ensure that content is being served. - resp, err := client.Get(url) - if err != nil { - return fmt.Errorf("error reaching http server: %v", err) - } - if want := http.StatusOK; resp.StatusCode != want { - return fmt.Errorf("wrong response code, got: %d, want: %d", resp.StatusCode, want) - } - return nil -} - -// TestLifeCycle tests a basic Create/Start/Stop docker container life cycle. -func TestLifeCycle(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - // Start the container. - port := 80 - if err := d.Create(ctx, dockerutil.RunOpts{ - Image: "basic/nginx", - Ports: []int{port}, - }); err != nil { - t.Fatalf("docker create failed: %v", err) - } - if err := d.Start(ctx); err != nil { - t.Fatalf("docker start failed: %v", err) - } - - ip, err := d.FindIP(ctx, false) - if err != nil { - t.Fatalf("docker.FindIP failed: %v", err) - } - if err := testutil.WaitForHTTP(ip.String(), port, defaultWait); err != nil { - t.Fatalf("WaitForHTTP() timeout: %v", err) - } - client := http.Client{Timeout: defaultWait} - if err := httpRequestSucceeds(client, ip.String(), port); err != nil { - t.Errorf("http request failed: %v", err) - } - - if err := d.Stop(ctx); err != nil { - t.Fatalf("docker stop failed: %v", err) - } - if err := d.Remove(ctx); err != nil { - t.Fatalf("docker rm failed: %v", err) - } -} - -func TestPauseResume(t *testing.T) { - if !testutil.IsCheckpointSupported() { - t.Skip("Checkpoint is not supported.") - } - - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - // Start the container. - port := 8080 - if err := d.Spawn(ctx, dockerutil.RunOpts{ - Image: "basic/python", - Ports: []int{port}, // See Dockerfile. - }); err != nil { - t.Fatalf("docker run failed: %v", err) - } - - // Find container IP address. - ip, err := d.FindIP(ctx, false) - if err != nil { - t.Fatalf("docker.FindIP failed: %v", err) - } - - // Wait until it's up and running. - if err := testutil.WaitForHTTP(ip.String(), port, defaultWait); err != nil { - t.Fatalf("WaitForHTTP() timeout: %v", err) - } - - // Check that container is working. - client := http.Client{Timeout: defaultWait} - if err := httpRequestSucceeds(client, ip.String(), port); err != nil { - t.Error("http request failed:", err) - } - - if err := d.Pause(ctx); err != nil { - t.Fatalf("docker pause failed: %v", err) - } - - // Check if container is paused. - client = http.Client{Timeout: 10 * time.Millisecond} // Don't wait a minute. - switch _, err := client.Get(fmt.Sprintf("http://%s:%d", ip.String(), port)); v := err.(type) { - case nil: - t.Errorf("http req expected to fail but it succeeded") - case net.Error: - if !v.Timeout() { - t.Errorf("http req got error %v, wanted timeout", v) - } - default: - t.Errorf("http req got unexpected error %v", v) - } - - if err := d.Unpause(ctx); err != nil { - t.Fatalf("docker unpause failed: %v", err) - } - - // Wait until it's up and running. - if err := testutil.WaitForHTTP(ip.String(), port, defaultWait); err != nil { - t.Fatalf("WaitForHTTP() timeout: %v", err) - } - - // Check if container is working again. - client = http.Client{Timeout: defaultWait} - if err := httpRequestSucceeds(client, ip.String(), port); err != nil { - t.Error("http request failed:", err) - } -} - -func TestCheckpointRestore(t *testing.T) { - if !testutil.IsCheckpointSupported() { - t.Skip("Pause/resume is not supported.") - } - - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - // Start the container. - port := 8080 - if err := d.Spawn(ctx, dockerutil.RunOpts{ - Image: "basic/python", - Ports: []int{port}, // See Dockerfile. - }); err != nil { - t.Fatalf("docker run failed: %v", err) - } - - // Create a snapshot. - if err := d.Checkpoint(ctx, "test"); err != nil { - t.Fatalf("docker checkpoint failed: %v", err) - } - if err := d.WaitTimeout(ctx, defaultWait); err != nil { - t.Fatalf("wait failed: %v", err) - } - - // TODO(b/143498576): Remove Poll after github.com/moby/moby/issues/38963 is fixed. - if err := testutil.Poll(func() error { return d.Restore(ctx, "test") }, defaultWait); err != nil { - t.Fatalf("docker restore failed: %v", err) - } - - // Find container IP address. - ip, err := d.FindIP(ctx, false) - if err != nil { - t.Fatalf("docker.FindIP failed: %v", err) - } - - // Wait until it's up and running. - if err := testutil.WaitForHTTP(ip.String(), port, defaultWait); err != nil { - t.Fatalf("WaitForHTTP() timeout: %v", err) - } - - // Check if container is working again. - client := http.Client{Timeout: defaultWait} - if err := httpRequestSucceeds(client, ip.String(), port); err != nil { - t.Error("http request failed:", err) - } -} - -// Create client and server that talk to each other using the local IP. -func TestConnectToSelf(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - // Creates server that replies "server" and exists. Sleeps at the end because - // 'docker exec' gets killed if the init process exists before it can finish. - if err := d.Spawn(ctx, dockerutil.RunOpts{ - Image: "basic/ubuntu", - }, "/bin/sh", "-c", "echo server | nc -l -p 8080 && sleep 1"); err != nil { - t.Fatalf("docker run failed: %v", err) - } - - // Finds IP address for host. - ip, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/bin/sh", "-c", "cat /etc/hosts | grep ${HOSTNAME} | awk '{print $1}'") - if err != nil { - t.Fatalf("docker exec failed: %v", err) - } - ip = strings.TrimRight(ip, "\n") - - // Runs client that sends "client" to the server and exits. - reply, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/bin/sh", "-c", fmt.Sprintf("echo client | nc %s 8080", ip)) - if err != nil { - t.Fatalf("docker exec failed: %v", err) - } - - // Ensure both client and server got the message from each other. - if want := "server\n"; reply != want { - t.Errorf("Error on server, want: %q, got: %q", want, reply) - } - if _, err := d.WaitForOutput(ctx, "^client\n$", defaultWait); err != nil { - t.Fatalf("docker.WaitForOutput(client) timeout: %v", err) - } -} - -func TestMemLimit(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - allocMemoryKb := 50 * 1024 - out, err := d.Run(ctx, dockerutil.RunOpts{ - Image: "basic/alpine", - Memory: allocMemoryKb * 1024, // In bytes. - }, "sh", "-c", "cat /proc/meminfo | grep MemTotal: | awk '{print $2}'") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - - // Remove warning message that swap isn't present. - if strings.HasPrefix(out, "WARNING") { - lines := strings.Split(out, "\n") - if len(lines) != 3 { - t.Fatalf("invalid output: %s", out) - } - out = lines[1] - } - - // Ensure the memory matches what we want. - got, err := strconv.ParseUint(strings.TrimSpace(out), 10, 64) - if err != nil { - t.Fatalf("failed to parse %q: %v", out, err) - } - if want := uint64(allocMemoryKb); got != want { - t.Errorf("MemTotal got: %d, want: %d", got, want) - } -} - -func TestNumCPU(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - // Read how many cores are in the container. - out, err := d.Run(ctx, dockerutil.RunOpts{ - Image: "basic/alpine", - CpusetCpus: "0", - }, "sh", "-c", "cat /proc/cpuinfo | grep 'processor.*:' | wc -l") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - - // Ensure it matches what we want. - got, err := strconv.Atoi(strings.TrimSpace(out)) - if err != nil { - t.Fatalf("failed to parse %q: %v", out, err) - } - if want := 1; got != want { - t.Errorf("MemTotal got: %d, want: %d", got, want) - } -} - -// TestJobControl tests that job control characters are handled properly. -func TestJobControl(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - // Start the container with an attached PTY. - p, err := d.SpawnProcess(ctx, dockerutil.RunOpts{ - Image: "basic/alpine", - }, "sh", "-c", "sleep 100 | cat") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - // Give shell a few seconds to start executing the sleep. - time.Sleep(2 * time.Second) - - if _, err := p.Write(time.Second, []byte{0x03}); err != nil { - t.Fatalf("error exit: %v", err) - } - - if err := d.WaitTimeout(ctx, 3*time.Second); err != nil { - t.Fatalf("WaitTimeout failed: %v", err) - } - - want := 130 - got, err := p.WaitExitStatus(ctx) - if err != nil { - t.Fatalf("wait for exit failed with: %v", err) - } else if got != want { - t.Fatalf("got: %d want: %d", got, want) - } -} - -// TestWorkingDirCreation checks that working dir is created if it doesn't exit. -func TestWorkingDirCreation(t *testing.T) { - for _, tc := range []struct { - name string - workingDir string - }{ - {name: "root", workingDir: "/foo"}, - {name: "tmp", workingDir: "/tmp/foo"}, - } { - for _, readonly := range []bool{true, false} { - name := tc.name - if readonly { - name += "-readonly" - } - t.Run(name, func(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - opts := dockerutil.RunOpts{ - Image: "basic/alpine", - WorkDir: tc.workingDir, - ReadOnly: readonly, - } - got, err := d.Run(ctx, opts, "sh", "-c", "echo ${PWD}") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - if want := tc.workingDir + "\n"; want != got { - t.Errorf("invalid working dir, want: %q, got: %q", want, got) - } - }) - } - } -} - -// TestTmpFile checks that files inside '/tmp' are not overridden. -func TestTmpFile(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - opts := dockerutil.RunOpts{Image: "basic/tmpfile"} - got, err := d.Run(ctx, opts, "cat", "/tmp/foo/file.txt") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - if want := "123\n"; want != got { - t.Errorf("invalid file content, want: %q, got: %q", want, got) - } -} - -// TestTmpMount checks that mounts inside '/tmp' are not overridden. -func TestTmpMount(t *testing.T) { - dir, err := ioutil.TempDir(testutil.TmpDir(), "tmp-mount") - if err != nil { - t.Fatalf("TempDir(): %v", err) - } - const want = "123" - if err := ioutil.WriteFile(filepath.Join(dir, "file.txt"), []byte("123"), 0666); err != nil { - t.Fatalf("WriteFile(): %v", err) - } - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - opts := dockerutil.RunOpts{ - Image: "basic/alpine", - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: dir, - Target: "/tmp/foo", - }, - }, - } - got, err := d.Run(ctx, opts, "cat", "/tmp/foo/file.txt") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - if want != got { - t.Errorf("invalid file content, want: %q, got: %q", want, got) - } -} - -// Test that it is allowed to mount a file on top of /dev files, e.g. -// /dev/random. -func TestMountOverDev(t *testing.T) { - if vfs2, err := dockerutil.UsingVFS2(); err != nil { - t.Fatalf("Failed to read config for runtime %s: %v", dockerutil.Runtime(), err) - } else if !vfs2 { - t.Skip("VFS1 doesn't allow /dev/random to be mounted.") - } - - random, err := ioutil.TempFile(testutil.TmpDir(), "random") - if err != nil { - t.Fatal("ioutil.TempFile() failed:", err) - } - const want = "123" - if _, err := random.WriteString(want); err != nil { - t.Fatalf("WriteString() to %q: %v", random.Name(), err) - } - - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - opts := dockerutil.RunOpts{ - Image: "basic/alpine", - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: random.Name(), - Target: "/dev/random", - }, - }, - } - cmd := "dd count=1 bs=5 if=/dev/random 2> /dev/null" - got, err := d.Run(ctx, opts, "sh", "-c", cmd) - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - if want != got { - t.Errorf("invalid file content, want: %q, got: %q", want, got) - } -} - -// TestSyntheticDirs checks that submounts can be created inside a readonly -// mount even if the target path does not exist. -func TestSyntheticDirs(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - opts := dockerutil.RunOpts{ - Image: "basic/alpine", - // Make the root read-only to force use of synthetic dirs - // inside the root gofer mount. - ReadOnly: true, - Mounts: []mount.Mount{ - // Mount inside read-only gofer-backed root. - { - Type: mount.TypeTmpfs, - Target: "/foo/bar/baz", - }, - // Mount inside sysfs, which always uses synthetic dirs - // for submounts. - { - Type: mount.TypeTmpfs, - Target: "/sys/foo/bar/baz", - }, - }, - } - // Make sure the directories exist. - if _, err := d.Run(ctx, opts, "ls", "/foo/bar/baz", "/sys/foo/bar/baz"); err != nil { - t.Fatalf("docker run failed: %v", err) - } - -} - -// TestHostOverlayfsCopyUp tests that the --overlayfs-stale-read option causes -// runsc to hide the incoherence of FDs opened before and after overlayfs -// copy-up on the host. -func TestHostOverlayfsCopyUp(t *testing.T) { - runIntegrationTest(t, nil, "./test_copy_up") -} - -// TestHostOverlayfsRewindDir tests that rewinddir() "causes the directory -// stream to refer to the current state of the corresponding directory, as a -// call to opendir() would have done" as required by POSIX, when the directory -// in question is host overlayfs. -// -// This test specifically targets host overlayfs because, per POSIX, "if a file -// is removed from or added to the directory after the most recent call to -// opendir() or rewinddir(), whether a subsequent call to readdir() returns an -// entry for that file is unspecified"; the host filesystems used by other -// automated tests yield newly-added files from readdir() even if the fsgofer -// does not explicitly rewinddir(), but overlayfs does not. -func TestHostOverlayfsRewindDir(t *testing.T) { - runIntegrationTest(t, nil, "./test_rewinddir") -} - -// Basic test for linkat(2). Syscall tests requires CAP_DAC_READ_SEARCH and it -// cannot use tricks like userns as root. For this reason, run a basic link test -// to ensure some coverage. -func TestLink(t *testing.T) { - runIntegrationTest(t, nil, "./link_test") -} - -// This test ensures we can run ping without errors. -func TestPing4Loopback(t *testing.T) { - if testutil.IsRunningWithHostNet() { - // TODO(gvisor.dev/issue/5011): support ICMP sockets in hostnet and enable - // this test. - t.Skip("hostnet only supports TCP/UDP sockets, so ping is not supported.") - } - - runIntegrationTest(t, nil, "./ping4.sh") -} - -// This test ensures we can enable ipv6 on loopback and run ping6 without -// errors. -func TestPing6Loopback(t *testing.T) { - if testutil.IsRunningWithHostNet() { - // TODO(gvisor.dev/issue/5011): support ICMP sockets in hostnet and enable - // this test. - t.Skip("hostnet only supports TCP/UDP sockets, so ping6 is not supported.") - } - - // The CAP_NET_ADMIN capability is required to use the `ip` utility, which - // we use to enable ipv6 on loopback. - // - // By default, ipv6 loopback is not enabled by runsc, because docker does - // not assign an ipv6 address to the test container. - runIntegrationTest(t, []string{"NET_ADMIN"}, "./ping6.sh") -} - -// This test checks that the owner of the sticky directory can delete files -// inside it belonging to other users. It also checks that the owner of a file -// can always delete its file when the file is inside a sticky directory owned -// by another user. -func TestStickyDir(t *testing.T) { - if vfs2Used, err := dockerutil.UsingVFS2(); err != nil { - t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err) - } else if !vfs2Used { - t.Skip("sticky bit test fails on VFS1.") - } - - runIntegrationTest(t, nil, "./test_sticky") -} - -func runIntegrationTest(t *testing.T, capAdd []string, args ...string) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - opts := dockerutil.RunOpts{ - Image: "basic/integrationtest", - WorkDir: "/root", - CapAdd: capAdd, - } - if got, err := d.Run(ctx, opts, args...); err != nil { - t.Fatalf("docker run failed: %v", err) - } else if got != "" { - t.Errorf("test failed:\n%s", got) - } -} - -// Test that UDS can be created using overlay when parent directory is in lower -// layer only (b/134090485). -// -// Prerequisite: the directory where the socket file is created must not have -// been open for write before bind(2) is called. -func TestBindOverlay(t *testing.T) { - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - // Run the container. - got, err := d.Run(ctx, dockerutil.RunOpts{ - Image: "basic/ubuntu", - }, "bash", "-c", "nc -q -1 -l -U /var/run/sock & p=$! && sleep 1 && echo foobar-asdf | nc -q 0 -U /var/run/sock && wait $p") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - - // Check the output contains what we want. - if want := "foobar-asdf"; !strings.Contains(got, want) { - t.Fatalf("docker run output is missing %q: %s", want, got) - } -} - -func TestStdios(t *testing.T) { - if vfs2, err := dockerutil.UsingVFS2(); err != nil { - t.Fatalf("Failed to read config for runtime %s: %v", dockerutil.Runtime(), err) - } else if !vfs2 { - t.Skip("VFS1 doesn't adjust stdios user") - } - - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - testStdios(t, func(user string, args ...string) (string, error) { - defer d.CleanUp(ctx) - opts := dockerutil.RunOpts{ - Image: "basic/alpine", - User: user, - } - return d.Run(ctx, opts, args...) - }) -} - -func TestStdiosExec(t *testing.T) { - if vfs2, err := dockerutil.UsingVFS2(); err != nil { - t.Fatalf("Failed to read config for runtime %s: %v", dockerutil.Runtime(), err) - } else if !vfs2 { - t.Skip("VFS1 doesn't adjust stdios user") - } - - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - runOpts := dockerutil.RunOpts{Image: "basic/alpine"} - if err := d.Spawn(ctx, runOpts, "sleep", "100"); err != nil { - t.Fatalf("docker run failed: %v", err) - } - - testStdios(t, func(user string, args ...string) (string, error) { - opts := dockerutil.ExecOpts{User: user} - return d.Exec(ctx, opts, args...) - }) -} - -func testStdios(t *testing.T, run func(string, ...string) (string, error)) { - const cmd = "stat -L /proc/self/fd/0 /proc/self/fd/1 /proc/self/fd/2 | grep 'Uid:'" - got, err := run("123", "/bin/sh", "-c", cmd) - if err != nil { - t.Fatalf("docker exec failed: %v", err) - } - if len(got) == 0 { - t.Errorf("Unexpected empty output from %q", cmd) - } - re := regexp.MustCompile(`Uid: \(\s*(\w+)\/.*\)`) - for _, line := range strings.SplitN(got, "\n", 3) { - t.Logf("stat -L: %s", line) - matches := re.FindSubmatch([]byte(line)) - if len(matches) != 2 { - t.Fatalf("wrong output format: %q: matches: %v", line, matches) - } - if want, got := "123", string(matches[1]); want != got { - t.Errorf("wrong user, want: %q, got: %q", want, got) - } - } - - // Check that stdout and stderr can be open and written to. This checks - // that ownership and permissions are correct inside gVisor. - got, err = run("456", "/bin/sh", "-c", "echo foobar | tee /proc/self/fd/1 > /proc/self/fd/2") - if err != nil { - t.Fatalf("docker run failed: %v", err) - } - t.Logf("echo foobar: %q", got) - // Check it repeats twice, once for stdout and once for stderr. - if want := "foobar\nfoobar\n"; want != got { - t.Errorf("Wrong echo output, want: %q, got: %q", want, got) - } - - // Check that timestamps can be changed. Setting timestamps require an extra - // write check _after_ the file was opened, and may fail if the underlying - // host file is not setup correctly. - if _, err := run("789", "touch", "/proc/self/fd/0", "/proc/self/fd/1", "/proc/self/fd/2"); err != nil { - t.Fatalf("docker run failed: %v", err) - } -} - -func TestStdiosChown(t *testing.T) { - if vfs2, err := dockerutil.UsingVFS2(); err != nil { - t.Fatalf("Failed to read config for runtime %s: %v", dockerutil.Runtime(), err) - } else if !vfs2 { - t.Skip("VFS1 doesn't adjust stdios user") - } - - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, t) - defer d.CleanUp(ctx) - - opts := dockerutil.RunOpts{Image: "basic/alpine"} - if _, err := d.Run(ctx, opts, "chown", "123", "/proc/self/fd/0", "/proc/self/fd/1", "/proc/self/fd/2"); err != nil { - t.Fatalf("docker run failed: %v", err) - } -} - -func TestMain(m *testing.M) { - dockerutil.EnsureSupportedDockerVersion() - flag.Parse() - os.Exit(m.Run()) -} |