From fbabec42349880407c4308211129c07ff51c484a Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 29 Jul 2024 12:32:56 +0200 Subject: tests: replace test runner shell script with ucode implementation The ucode interpreter and libraries are mature enough to execute their own testcases now, so replace the existing shell script with an equivalent ucode implementation. Signed-off-by: Jo-Philipp Wich --- tests/custom/CMakeLists.txt | 4 +- tests/custom/run_tests.sh | 240 ----------------------------------------- tests/custom/run_tests.uc | 254 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 242 deletions(-) delete mode 100755 tests/custom/run_tests.sh create mode 100755 tests/custom/run_tests.uc diff --git a/tests/custom/CMakeLists.txt b/tests/custom/CMakeLists.txt index c8007a0..c94278e 100644 --- a/tests/custom/CMakeLists.txt +++ b/tests/custom/CMakeLists.txt @@ -1,6 +1,6 @@ ADD_TEST( NAME custom - COMMAND run_tests.sh + COMMAND $ -L $/*.so -S run_tests.uc WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT @@ -11,7 +11,7 @@ SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT IF(CMAKE_C_COMPILER_ID STREQUAL "Clang") ADD_TEST( NAME custom-san - COMMAND run_tests.sh + COMMAND $ -L $/*.so -S run_tests.uc WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/tests/custom/run_tests.sh b/tests/custom/run_tests.sh deleted file mode 100755 index 96ac783..0000000 --- a/tests/custom/run_tests.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env bash - -if greadlink -f . &>/dev/null; then - readlink=greadlink -else - readlink=readlink -fi - -testdir=$(dirname "$0") -topdir=$($readlink -f "$testdir/../..") - -line='........................................' -ucode_bin=${UCODE_BIN:-"$topdir/ucode"} -ucode_lib=${UCODE_LIB:-"$topdir"} - -extract_sections() { - local file=$1 - local dir=$2 - local count=0 - local tag line outfile - - while IFS= read -r line; do - case "$line" in - "-- Args --") - tag="args" - count=$((count + 1)) - outfile=$(printf "%s/%03d.args" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Vars --") - tag="vars" - count=$((count + 1)) - outfile=$(printf "%s/%03d.vars" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Testcase --") - tag="test" - count=$((count + 1)) - outfile=$(printf "%s/%03d.in" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --") - tag="${line#-- Expect }" - tag="${tag% --}" - count=$((count + 1)) - outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag") - printf "" > "$outfile" - ;; - "-- File "*" --") - tag="file" - outfile="${line#-- File }" - outfile="$(echo "${outfile% --}" | xargs)" - outfile="$dir/files$($readlink -m "/${outfile:-file}")" - mkdir -p "$(dirname "$outfile")" - printf "" > "$outfile" - ;; - "-- End (no-eol) --") - truncate -s -1 "$outfile" - tag="" - outfile="" - ;; - "-- End --") - tag="" - outfile="" - ;; - *) - if [ -n "$tag" ]; then - printf "%s\\n" "$line" >> "$outfile" - fi - ;; - esac - done < "$file" - - return $(ls -l "$dir/"*.in 2>/dev/null | wc -l) -} - -run_testcase() { - local num=$1 - local dir=$2 - local in=$3 - local out=$4 - local err=$5 - local code=$6 - local args=$7 - local vars=$8 - local fail=0 - - ( - cd "$dir" - - IFS=$'\n' - - local var - for var in $vars; do - case "$var" in - *=*) export "$var" ;; - esac - done - - IFS=$' \t\n' - - $ucode_bin -T"," -L "$ucode_lib/*.so" -D TESTFILES_PATH="$($readlink -f "$dir/files")" $args - <"$in" >"$dir/res.out" 2>"$dir/res.err" - ) - - printf "%d\n" $? > "$dir/res.code" - touch "$dir/empty" - - sed -i -e "s#$dir#.#g" "$dir/res.out" "$dir/res.err" - - if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected stderr did not match:\n" $num - diff -au --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err" - printf -- "---\n" - fail=1 - fi - - if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected stdout did not match:\n" $num - diff -au --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out" - printf -- "---\n" - fail=1 - fi - - if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected exit code did not match:\n" $num - diff -au --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code" - printf -- "---\n" - fail=1 - fi - - return $fail -} - -run_test() { - local file=$1 - local name=${file##*/} - local res ecode eout eerr ein eargs tests - local testcase_first=0 failed=0 count=0 - - printf "%s %s " "$name" "${line:${#name}}" - - mkdir "/tmp/test.$$" - - extract_sections "$file" "/tmp/test.$$" - tests=$? - - [ -f "/tmp/test.$$/001.in" ] && testcase_first=1 - - for res in "/tmp/test.$$/"[0-9]*; do - case "$res" in - *.in) - count=$((count + 1)) - - if [ $testcase_first = 1 ]; then - # Flush previous test - if [ -n "$ein" ]; then - run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - eout="" - eerr="" - ecode="" - eargs="" - evars="" - fi - - ein=$res - else - run_testcase $count "/tmp/test.$$" "$res" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - - eout="" - eerr="" - ecode="" - eargs="" - evars="" - fi - - ;; - *.stdout) eout=$res ;; - *.stderr) eerr=$res ;; - *.exitcode) ecode=$res ;; - *.args) eargs=$(cat "$res") ;; - *.vars) evars=$(cat "$res") ;; - esac - done - - # Flush last test - if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then - run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - fi - - rm -r "/tmp/test.$$" - - if [ $failed = 0 ]; then - printf "OK\n" - else - printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests - fi - - return $failed -} - - -n_tests=0 -n_fails=0 - -select_tests="$@" - -use_test() { - local input="$($readlink -f "$1")" - local test - - [ -f "$input" ] || return 1 - [ -n "$select_tests" ] || return 0 - - for test in "$select_tests"; do - test="$($readlink -f "$test")" - - [ "$test" != "$input" ] || return 0 - done - - return 1 -} - -for catdir in "$testdir/"[0-9][0-9]_*; do - [ -d "$catdir" ] || continue - - printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}" - - for testfile in "$catdir/"[0-9][0-9]_*; do - use_test "$testfile" || continue - - n_tests=$((n_tests + 1)) - run_test "$testfile" || n_fails=$((n_fails + 1)) - done -done - -printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails -exit $n_fails diff --git a/tests/custom/run_tests.uc b/tests/custom/run_tests.uc new file mode 100755 index 0000000..ff81afb --- /dev/null +++ b/tests/custom/run_tests.uc @@ -0,0 +1,254 @@ +#!/usr/bin/env -S ucode -S + +import * as fs from 'fs'; + +let testdir = sourcepath(0, true); +let topdir = fs.realpath(`${testdir}/../..`); + +let line = '........................................'; +let ucode_bin = getenv('UCODE_BIN') || `${topdir}/ucode`; +let ucode_lib = getenv('UCODE_LIB') || topdir; + +function mkdir_p(path) { + let parts = split(rtrim(path, '/') || '/', /\/+/); + let current = ''; + + for (let part in parts) { + current += part + '/'; + + let s = fs.stat(current); + + if (s == null) { + if (!fs.mkdir(current)) + die(`Error creating directory '${current}': ${fs.error()}`); + } + else if (s.type != 'directory') { + die(`Path '${current}' exists but is not a directory`); + } + } +} + +function shellquote(s) { + return `'${replace(s, "'", "'\\''")}'`; +} + +function getpid() { + return +fs.popen('echo $PPID', 'r').read('all'); +} + +function has_expectations(testcase) +{ + return (testcase?.stdout != null || testcase?.stderr != null || testcase?.exitcode != null); +} + +function parse_testcases(file, dir) { + let fp = fs.open(file, 'r') ?? die(`Unable to open ${file}: ${fs.error()}`); + let testcases, testcase, section, m; + let code_first = false; + + for (let line = fp.read('line'); length(line); line = fp.read('line')) { + if (line == '-- Args --\n') { + section = [ 'args', [] ]; + } + else if (line == '-- Vars --\n') { + section = [ 'env', {} ]; + } + else if (line == '-- Testcase --\n') { + section = [ 'code', '' ]; + } + else if ((m = match(line, /^-- Expect (stdout|stderr|exitcode) --$/s)) != null) { + section = [ m[1], '' ]; + } + else if ((m = match(line, /^-- File (.*)--$/s)) != null) { + section = [ 'file', `${dir}/files/${trim(m[1]) || 'file'}`, '' ]; + } + else if ((m = match(line, /^-- End( \(no-eol\))? --$/s)) != null) { + if (m[1] != null && type(section[-1]) == 'string') + section[-1] = substr(section[-1], 0, -1); + + if (section[0] == 'code') { + if (testcases == null && !has_expectations(testcase)) + code_first = true; + + if (code_first) { + if (testcase?.code != null) { + push(testcases ??= [], testcase); + testcase = null; + } + + (testcase ??= {}).code = section[1]; + } + else { + push(testcases ??= [], { ...testcase, code: section[1] }); + testcase = null; + } + } + else if (section[0] == 'file') { + ((testcase ??= {}).files ??= {})[section[1]] = section[2]; + } + else { + (testcase ??= {})[section[0]] = section[1]; + } + + section = null; + } + else if (section) { + switch (section[0]) { + case 'args': + if ((m = trim(line)) != '') + push(section[1], ...split(m, /[ \t\r\n]+/)); + break; + + case 'env': + if ((m = match(line, /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s)) != null) + section[1][m[1]] = m[2]; + break; + + default: + section[-1] += line; + break; + } + } + } + + if (code_first && testcase.code != null && has_expectations(testcase)) + push(testcases ??= [], testcase); + + return testcases; +} + +function diff(tag, ...ab) { + let cmd = [ 'diff', '-au', '--color=always', `--label=Expected ${tag}`, `--label=Resulting ${tag}` ]; + let tmpfiles = []; + + for (let i, f in ab) { + if (type(f) != 'resource') { + push(tmpfiles, fs.mkstemp()); + tmpfiles[-1].write(f); + f = tmpfiles[-1]; + } + + f.seek(0); + push(cmd, `/dev/fd/${f.fileno()}`); + } + + system(cmd); +} + +function run_testcase(num, dir, testcase) { + let fout = fs.mkstemp(`${dir}/stdout.XXXXXX`); + let ferr = fs.mkstemp(`${dir}/stderr.XXXXXX`); + + let eout = testcase.stdout ?? ''; + let eerr = testcase.stderr ?? ''; + let ecode = testcase.exitcode ? +testcase.exitcode : null; + + let cmd = join(' ', [ + ...map(keys(testcase.env) ?? [], k => `export ${k}=${shellquote(testcase.env[k])};`), + `cd ${shellquote(dir)};`, + `exec ${ucode_bin}`, + `-T','`, + `-L ${shellquote(`${ucode_lib}/*.so`)}`, + `-D TESTFILES_PATH=${shellquote(`${fs.realpath(dir)}/files`)}`, + `${join(' ', map(testcase.args ?? [], shellquote))} -`, + `>/dev/fd/${fout.fileno()} 2>/dev/fd/${ferr.fileno()}` + ]); + + let proc = fs.popen(cmd, 'w') ?? die(`Error launching test command "${cmd}": ${fs.error()}\n`); + + if (testcase.code != null) + proc.write(testcase.code); + + let exitcode = proc.close(); + + fout.seek(0); + ferr.seek(0); + + let ok = true; + + if (replace(ferr.read('all'), dir, '.') != eerr) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stderr did not match:\n", num); + diff('stderr', eerr, ferr); + print("---\n"); + ok = false; + } + + if (replace(fout.read('all'), dir, '.') != eout) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stdout did not match:\n", num); + diff('stdout', eout, fout); + print("---\n"); + ok = false; + } + + if (ecode != null && exitcode != ecode) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected exit code did not match:\n", num); + diff('code', `${ecode}\n`, `${exitcode}\n`); + print("---\n"); + ok = false; + } + + return ok; +} + +function run_test(file) { + let name = fs.basename(file); + printf('%s %s ', name, substr(line, length(name))); + + let tmpdir = sprintf('/tmp/test.%d', getpid()); + let testcases = parse_testcases(file, tmpdir); + let failed = 0; + + fs.mkdir(tmpdir); + + try { + for (let i, testcase in testcases) { + for (let path, data in testcase.files) { + mkdir_p(fs.dirname(path)); + fs.writefile(path, data) ?? die(`Error writing testcase file "${path}": ${fs.error()}\n`); + } + + failed += !run_testcase(i + 1, tmpdir, testcase); + } + } + catch (e) { + warn(`${e.type}: ${e.message}\n${e.stacktrace[0].context}\n`); + } + + system(['rm', '-r', tmpdir]); + + if (failed == 0) + print('OK\n'); + else + printf('%s %s FAILED (%d/%d)\n', name, substr(line, length(name)), failed, length(testcases)); + + return failed; +} + +let n_tests = 0; +let n_fails = 0; +let select_tests = filter(map(ARGV, p => fs.realpath(p)), length); + +function use_test(input) { + return fs.access(input = fs.realpath(input)) && + (!length(select_tests) || filter(select_tests, p => p == input)[0]); +} + +for (let catdir in fs.glob(`${testdir}/[0-9][0-9]_*`)) { + if (fs.stat(catdir)?.type != 'directory') + continue; + + printf('\n##\n## Running %s tests\n##\n\n', substr(fs.basename(catdir), 3)); + + for (let testfile in fs.glob(`${catdir}/[0-9][0-9]_*`)) { + if (!use_test(testfile)) continue; + + n_tests++; + n_fails += run_test(testfile); + } +} + +printf('\nRan %d tests, %d okay, %d failures\n', n_tests, n_tests - n_fails, n_fails); +exit(n_fails); -- cgit v1.2.3