diff options
| -rw-r--r-- | tests/custom/CMakeLists.txt | 4 | ||||
| -rwxr-xr-x | tests/custom/run_tests.sh | 240 | ||||
| -rwxr-xr-x | tests/custom/run_tests.uc | 254 |
3 files changed, 256 insertions, 242 deletions
diff --git a/tests/custom/CMakeLists.txt b/tests/custom/CMakeLists.txt index c8007a0..c94278e 100644 --- a/tests/custom/CMakeLists.txt +++ b/tests/custom/CMakeLists.txt @@ -1,6 +1,6 @@ ADD_TEST( NAME custom - COMMAND run_tests.sh + COMMAND $<TARGET_FILE:ucode> -L $<TARGET_FILE_DIR:fs_lib>/*.so -S run_tests.uc WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT @@ -11,7 +11,7 @@ SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT IF(CMAKE_C_COMPILER_ID STREQUAL "Clang") ADD_TEST( NAME custom-san - COMMAND run_tests.sh + COMMAND $<TARGET_FILE:ucode> -L $<TARGET_FILE_DIR:fs_lib>/*.so -S run_tests.uc WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/tests/custom/run_tests.sh b/tests/custom/run_tests.sh deleted file mode 100755 index 96ac783..0000000 --- a/tests/custom/run_tests.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env bash - -if greadlink -f . &>/dev/null; then - readlink=greadlink -else - readlink=readlink -fi - -testdir=$(dirname "$0") -topdir=$($readlink -f "$testdir/../..") - -line='........................................' -ucode_bin=${UCODE_BIN:-"$topdir/ucode"} -ucode_lib=${UCODE_LIB:-"$topdir"} - -extract_sections() { - local file=$1 - local dir=$2 - local count=0 - local tag line outfile - - while IFS= read -r line; do - case "$line" in - "-- Args --") - tag="args" - count=$((count + 1)) - outfile=$(printf "%s/%03d.args" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Vars --") - tag="vars" - count=$((count + 1)) - outfile=$(printf "%s/%03d.vars" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Testcase --") - tag="test" - count=$((count + 1)) - outfile=$(printf "%s/%03d.in" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --") - tag="${line#-- Expect }" - tag="${tag% --}" - count=$((count + 1)) - outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag") - printf "" > "$outfile" - ;; - "-- File "*" --") - tag="file" - outfile="${line#-- File }" - outfile="$(echo "${outfile% --}" | xargs)" - outfile="$dir/files$($readlink -m "/${outfile:-file}")" - mkdir -p "$(dirname "$outfile")" - printf "" > "$outfile" - ;; - "-- End (no-eol) --") - truncate -s -1 "$outfile" - tag="" - outfile="" - ;; - "-- End --") - tag="" - outfile="" - ;; - *) - if [ -n "$tag" ]; then - printf "%s\\n" "$line" >> "$outfile" - fi - ;; - esac - done < "$file" - - return $(ls -l "$dir/"*.in 2>/dev/null | wc -l) -} - -run_testcase() { - local num=$1 - local dir=$2 - local in=$3 - local out=$4 - local err=$5 - local code=$6 - local args=$7 - local vars=$8 - local fail=0 - - ( - cd "$dir" - - IFS=$'\n' - - local var - for var in $vars; do - case "$var" in - *=*) export "$var" ;; - esac - done - - IFS=$' \t\n' - - $ucode_bin -T"," -L "$ucode_lib/*.so" -D TESTFILES_PATH="$($readlink -f "$dir/files")" $args - <"$in" >"$dir/res.out" 2>"$dir/res.err" - ) - - printf "%d\n" $? > "$dir/res.code" - touch "$dir/empty" - - sed -i -e "s#$dir#.#g" "$dir/res.out" "$dir/res.err" - - if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected stderr did not match:\n" $num - diff -au --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err" - printf -- "---\n" - fail=1 - fi - - if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected stdout did not match:\n" $num - diff -au --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out" - printf -- "---\n" - fail=1 - fi - - if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected exit code did not match:\n" $num - diff -au --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code" - printf -- "---\n" - fail=1 - fi - - return $fail -} - -run_test() { - local file=$1 - local name=${file##*/} - local res ecode eout eerr ein eargs tests - local testcase_first=0 failed=0 count=0 - - printf "%s %s " "$name" "${line:${#name}}" - - mkdir "/tmp/test.$$" - - extract_sections "$file" "/tmp/test.$$" - tests=$? - - [ -f "/tmp/test.$$/001.in" ] && testcase_first=1 - - for res in "/tmp/test.$$/"[0-9]*; do - case "$res" in - *.in) - count=$((count + 1)) - - if [ $testcase_first = 1 ]; then - # Flush previous test - if [ -n "$ein" ]; then - run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - eout="" - eerr="" - ecode="" - eargs="" - evars="" - fi - - ein=$res - else - run_testcase $count "/tmp/test.$$" "$res" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - - eout="" - eerr="" - ecode="" - eargs="" - evars="" - fi - - ;; - *.stdout) eout=$res ;; - *.stderr) eerr=$res ;; - *.exitcode) ecode=$res ;; - *.args) eargs=$(cat "$res") ;; - *.vars) evars=$(cat "$res") ;; - esac - done - - # Flush last test - if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then - run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - fi - - rm -r "/tmp/test.$$" - - if [ $failed = 0 ]; then - printf "OK\n" - else - printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests - fi - - return $failed -} - - -n_tests=0 -n_fails=0 - -select_tests="$@" - -use_test() { - local input="$($readlink -f "$1")" - local test - - [ -f "$input" ] || return 1 - [ -n "$select_tests" ] || return 0 - - for test in "$select_tests"; do - test="$($readlink -f "$test")" - - [ "$test" != "$input" ] || return 0 - done - - return 1 -} - -for catdir in "$testdir/"[0-9][0-9]_*; do - [ -d "$catdir" ] || continue - - printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}" - - for testfile in "$catdir/"[0-9][0-9]_*; do - use_test "$testfile" || continue - - n_tests=$((n_tests + 1)) - run_test "$testfile" || n_fails=$((n_fails + 1)) - done -done - -printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails -exit $n_fails diff --git a/tests/custom/run_tests.uc b/tests/custom/run_tests.uc new file mode 100755 index 0000000..ff81afb --- /dev/null +++ b/tests/custom/run_tests.uc @@ -0,0 +1,254 @@ +#!/usr/bin/env -S ucode -S + +import * as fs from 'fs'; + +let testdir = sourcepath(0, true); +let topdir = fs.realpath(`${testdir}/../..`); + +let line = '........................................'; +let ucode_bin = getenv('UCODE_BIN') || `${topdir}/ucode`; +let ucode_lib = getenv('UCODE_LIB') || topdir; + +function mkdir_p(path) { + let parts = split(rtrim(path, '/') || '/', /\/+/); + let current = ''; + + for (let part in parts) { + current += part + '/'; + + let s = fs.stat(current); + + if (s == null) { + if (!fs.mkdir(current)) + die(`Error creating directory '${current}': ${fs.error()}`); + } + else if (s.type != 'directory') { + die(`Path '${current}' exists but is not a directory`); + } + } +} + +function shellquote(s) { + return `'${replace(s, "'", "'\\''")}'`; +} + +function getpid() { + return +fs.popen('echo $PPID', 'r').read('all'); +} + +function has_expectations(testcase) +{ + return (testcase?.stdout != null || testcase?.stderr != null || testcase?.exitcode != null); +} + +function parse_testcases(file, dir) { + let fp = fs.open(file, 'r') ?? die(`Unable to open ${file}: ${fs.error()}`); + let testcases, testcase, section, m; + let code_first = false; + + for (let line = fp.read('line'); length(line); line = fp.read('line')) { + if (line == '-- Args --\n') { + section = [ 'args', [] ]; + } + else if (line == '-- Vars --\n') { + section = [ 'env', {} ]; + } + else if (line == '-- Testcase --\n') { + section = [ 'code', '' ]; + } + else if ((m = match(line, /^-- Expect (stdout|stderr|exitcode) --$/s)) != null) { + section = [ m[1], '' ]; + } + else if ((m = match(line, /^-- File (.*)--$/s)) != null) { + section = [ 'file', `${dir}/files/${trim(m[1]) || 'file'}`, '' ]; + } + else if ((m = match(line, /^-- End( \(no-eol\))? --$/s)) != null) { + if (m[1] != null && type(section[-1]) == 'string') + section[-1] = substr(section[-1], 0, -1); + + if (section[0] == 'code') { + if (testcases == null && !has_expectations(testcase)) + code_first = true; + + if (code_first) { + if (testcase?.code != null) { + push(testcases ??= [], testcase); + testcase = null; + } + + (testcase ??= {}).code = section[1]; + } + else { + push(testcases ??= [], { ...testcase, code: section[1] }); + testcase = null; + } + } + else if (section[0] == 'file') { + ((testcase ??= {}).files ??= {})[section[1]] = section[2]; + } + else { + (testcase ??= {})[section[0]] = section[1]; + } + + section = null; + } + else if (section) { + switch (section[0]) { + case 'args': + if ((m = trim(line)) != '') + push(section[1], ...split(m, /[ \t\r\n]+/)); + break; + + case 'env': + if ((m = match(line, /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s)) != null) + section[1][m[1]] = m[2]; + break; + + default: + section[-1] += line; + break; + } + } + } + + if (code_first && testcase.code != null && has_expectations(testcase)) + push(testcases ??= [], testcase); + + return testcases; +} + +function diff(tag, ...ab) { + let cmd = [ 'diff', '-au', '--color=always', `--label=Expected ${tag}`, `--label=Resulting ${tag}` ]; + let tmpfiles = []; + + for (let i, f in ab) { + if (type(f) != 'resource') { + push(tmpfiles, fs.mkstemp()); + tmpfiles[-1].write(f); + f = tmpfiles[-1]; + } + + f.seek(0); + push(cmd, `/dev/fd/${f.fileno()}`); + } + + system(cmd); +} + +function run_testcase(num, dir, testcase) { + let fout = fs.mkstemp(`${dir}/stdout.XXXXXX`); + let ferr = fs.mkstemp(`${dir}/stderr.XXXXXX`); + + let eout = testcase.stdout ?? ''; + let eerr = testcase.stderr ?? ''; + let ecode = testcase.exitcode ? +testcase.exitcode : null; + + let cmd = join(' ', [ + ...map(keys(testcase.env) ?? [], k => `export ${k}=${shellquote(testcase.env[k])};`), + `cd ${shellquote(dir)};`, + `exec ${ucode_bin}`, + `-T','`, + `-L ${shellquote(`${ucode_lib}/*.so`)}`, + `-D TESTFILES_PATH=${shellquote(`${fs.realpath(dir)}/files`)}`, + `${join(' ', map(testcase.args ?? [], shellquote))} -`, + `>/dev/fd/${fout.fileno()} 2>/dev/fd/${ferr.fileno()}` + ]); + + let proc = fs.popen(cmd, 'w') ?? die(`Error launching test command "${cmd}": ${fs.error()}\n`); + + if (testcase.code != null) + proc.write(testcase.code); + + let exitcode = proc.close(); + + fout.seek(0); + ferr.seek(0); + + let ok = true; + + if (replace(ferr.read('all'), dir, '.') != eerr) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stderr did not match:\n", num); + diff('stderr', eerr, ferr); + print("---\n"); + ok = false; + } + + if (replace(fout.read('all'), dir, '.') != eout) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stdout did not match:\n", num); + diff('stdout', eout, fout); + print("---\n"); + ok = false; + } + + if (ecode != null && exitcode != ecode) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected exit code did not match:\n", num); + diff('code', `${ecode}\n`, `${exitcode}\n`); + print("---\n"); + ok = false; + } + + return ok; +} + +function run_test(file) { + let name = fs.basename(file); + printf('%s %s ', name, substr(line, length(name))); + + let tmpdir = sprintf('/tmp/test.%d', getpid()); + let testcases = parse_testcases(file, tmpdir); + let failed = 0; + + fs.mkdir(tmpdir); + + try { + for (let i, testcase in testcases) { + for (let path, data in testcase.files) { + mkdir_p(fs.dirname(path)); + fs.writefile(path, data) ?? die(`Error writing testcase file "${path}": ${fs.error()}\n`); + } + + failed += !run_testcase(i + 1, tmpdir, testcase); + } + } + catch (e) { + warn(`${e.type}: ${e.message}\n${e.stacktrace[0].context}\n`); + } + + system(['rm', '-r', tmpdir]); + + if (failed == 0) + print('OK\n'); + else + printf('%s %s FAILED (%d/%d)\n', name, substr(line, length(name)), failed, length(testcases)); + + return failed; +} + +let n_tests = 0; +let n_fails = 0; +let select_tests = filter(map(ARGV, p => fs.realpath(p)), length); + +function use_test(input) { + return fs.access(input = fs.realpath(input)) && + (!length(select_tests) || filter(select_tests, p => p == input)[0]); +} + +for (let catdir in fs.glob(`${testdir}/[0-9][0-9]_*`)) { + if (fs.stat(catdir)?.type != 'directory') + continue; + + printf('\n##\n## Running %s tests\n##\n\n', substr(fs.basename(catdir), 3)); + + for (let testfile in fs.glob(`${catdir}/[0-9][0-9]_*`)) { + if (!use_test(testfile)) continue; + + n_tests++; + n_fails += run_test(testfile); + } +} + +printf('\nRan %d tests, %d okay, %d failures\n', n_tests, n_tests - n_fails, n_fails); +exit(n_fails); |