From fbabec42349880407c4308211129c07ff51c484a Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 29 Jul 2024 12:32:56 +0200 Subject: tests: replace test runner shell script with ucode implementation The ucode interpreter and libraries are mature enough to execute their own testcases now, so replace the existing shell script with an equivalent ucode implementation. Signed-off-by: Jo-Philipp Wich --- tests/custom/run_tests.uc | 254 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100755 tests/custom/run_tests.uc (limited to 'tests/custom/run_tests.uc') diff --git a/tests/custom/run_tests.uc b/tests/custom/run_tests.uc new file mode 100755 index 0000000..ff81afb --- /dev/null +++ b/tests/custom/run_tests.uc @@ -0,0 +1,254 @@ +#!/usr/bin/env -S ucode -S + +import * as fs from 'fs'; + +let testdir = sourcepath(0, true); +let topdir = fs.realpath(`${testdir}/../..`); + +let line = '........................................'; +let ucode_bin = getenv('UCODE_BIN') || `${topdir}/ucode`; +let ucode_lib = getenv('UCODE_LIB') || topdir; + +function mkdir_p(path) { + let parts = split(rtrim(path, '/') || '/', /\/+/); + let current = ''; + + for (let part in parts) { + current += part + '/'; + + let s = fs.stat(current); + + if (s == null) { + if (!fs.mkdir(current)) + die(`Error creating directory '${current}': ${fs.error()}`); + } + else if (s.type != 'directory') { + die(`Path '${current}' exists but is not a directory`); + } + } +} + +function shellquote(s) { + return `'${replace(s, "'", "'\\''")}'`; +} + +function getpid() { + return +fs.popen('echo $PPID', 'r').read('all'); +} + +function has_expectations(testcase) +{ + return (testcase?.stdout != null || testcase?.stderr != null || testcase?.exitcode != null); +} + +function parse_testcases(file, dir) { + let fp = fs.open(file, 'r') ?? die(`Unable to open ${file}: ${fs.error()}`); + let testcases, testcase, section, m; + let code_first = false; + + for (let line = fp.read('line'); length(line); line = fp.read('line')) { + if (line == '-- Args --\n') { + section = [ 'args', [] ]; + } + else if (line == '-- Vars --\n') { + section = [ 'env', {} ]; + } + else if (line == '-- Testcase --\n') { + section = [ 'code', '' ]; + } + else if ((m = match(line, /^-- Expect (stdout|stderr|exitcode) --$/s)) != null) { + section = [ m[1], '' ]; + } + else if ((m = match(line, /^-- File (.*)--$/s)) != null) { + section = [ 'file', `${dir}/files/${trim(m[1]) || 'file'}`, '' ]; + } + else if ((m = match(line, /^-- End( \(no-eol\))? --$/s)) != null) { + if (m[1] != null && type(section[-1]) == 'string') + section[-1] = substr(section[-1], 0, -1); + + if (section[0] == 'code') { + if (testcases == null && !has_expectations(testcase)) + code_first = true; + + if (code_first) { + if (testcase?.code != null) { + push(testcases ??= [], testcase); + testcase = null; + } + + (testcase ??= {}).code = section[1]; + } + else { + push(testcases ??= [], { ...testcase, code: section[1] }); + testcase = null; + } + } + else if (section[0] == 'file') { + ((testcase ??= {}).files ??= {})[section[1]] = section[2]; + } + else { + (testcase ??= {})[section[0]] = section[1]; + } + + section = null; + } + else if (section) { + switch (section[0]) { + case 'args': + if ((m = trim(line)) != '') + push(section[1], ...split(m, /[ \t\r\n]+/)); + break; + + case 'env': + if ((m = match(line, /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s)) != null) + section[1][m[1]] = m[2]; + break; + + default: + section[-1] += line; + break; + } + } + } + + if (code_first && testcase.code != null && has_expectations(testcase)) + push(testcases ??= [], testcase); + + return testcases; +} + +function diff(tag, ...ab) { + let cmd = [ 'diff', '-au', '--color=always', `--label=Expected ${tag}`, `--label=Resulting ${tag}` ]; + let tmpfiles = []; + + for (let i, f in ab) { + if (type(f) != 'resource') { + push(tmpfiles, fs.mkstemp()); + tmpfiles[-1].write(f); + f = tmpfiles[-1]; + } + + f.seek(0); + push(cmd, `/dev/fd/${f.fileno()}`); + } + + system(cmd); +} + +function run_testcase(num, dir, testcase) { + let fout = fs.mkstemp(`${dir}/stdout.XXXXXX`); + let ferr = fs.mkstemp(`${dir}/stderr.XXXXXX`); + + let eout = testcase.stdout ?? ''; + let eerr = testcase.stderr ?? ''; + let ecode = testcase.exitcode ? +testcase.exitcode : null; + + let cmd = join(' ', [ + ...map(keys(testcase.env) ?? [], k => `export ${k}=${shellquote(testcase.env[k])};`), + `cd ${shellquote(dir)};`, + `exec ${ucode_bin}`, + `-T','`, + `-L ${shellquote(`${ucode_lib}/*.so`)}`, + `-D TESTFILES_PATH=${shellquote(`${fs.realpath(dir)}/files`)}`, + `${join(' ', map(testcase.args ?? [], shellquote))} -`, + `>/dev/fd/${fout.fileno()} 2>/dev/fd/${ferr.fileno()}` + ]); + + let proc = fs.popen(cmd, 'w') ?? die(`Error launching test command "${cmd}": ${fs.error()}\n`); + + if (testcase.code != null) + proc.write(testcase.code); + + let exitcode = proc.close(); + + fout.seek(0); + ferr.seek(0); + + let ok = true; + + if (replace(ferr.read('all'), dir, '.') != eerr) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stderr did not match:\n", num); + diff('stderr', eerr, ferr); + print("---\n"); + ok = false; + } + + if (replace(fout.read('all'), dir, '.') != eout) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stdout did not match:\n", num); + diff('stdout', eout, fout); + print("---\n"); + ok = false; + } + + if (ecode != null && exitcode != ecode) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected exit code did not match:\n", num); + diff('code', `${ecode}\n`, `${exitcode}\n`); + print("---\n"); + ok = false; + } + + return ok; +} + +function run_test(file) { + let name = fs.basename(file); + printf('%s %s ', name, substr(line, length(name))); + + let tmpdir = sprintf('/tmp/test.%d', getpid()); + let testcases = parse_testcases(file, tmpdir); + let failed = 0; + + fs.mkdir(tmpdir); + + try { + for (let i, testcase in testcases) { + for (let path, data in testcase.files) { + mkdir_p(fs.dirname(path)); + fs.writefile(path, data) ?? die(`Error writing testcase file "${path}": ${fs.error()}\n`); + } + + failed += !run_testcase(i + 1, tmpdir, testcase); + } + } + catch (e) { + warn(`${e.type}: ${e.message}\n${e.stacktrace[0].context}\n`); + } + + system(['rm', '-r', tmpdir]); + + if (failed == 0) + print('OK\n'); + else + printf('%s %s FAILED (%d/%d)\n', name, substr(line, length(name)), failed, length(testcases)); + + return failed; +} + +let n_tests = 0; +let n_fails = 0; +let select_tests = filter(map(ARGV, p => fs.realpath(p)), length); + +function use_test(input) { + return fs.access(input = fs.realpath(input)) && + (!length(select_tests) || filter(select_tests, p => p == input)[0]); +} + +for (let catdir in fs.glob(`${testdir}/[0-9][0-9]_*`)) { + if (fs.stat(catdir)?.type != 'directory') + continue; + + printf('\n##\n## Running %s tests\n##\n\n', substr(fs.basename(catdir), 3)); + + for (let testfile in fs.glob(`${catdir}/[0-9][0-9]_*`)) { + if (!use_test(testfile)) continue; + + n_tests++; + n_fails += run_test(testfile); + } +} + +printf('\nRan %d tests, %d okay, %d failures\n', n_tests, n_tests - n_fails, n_fails); +exit(n_fails); -- cgit v1.2.3