tests: replace test runner shell script with ucode implementation

The ucode interpreter and libraries are mature enough to execute their
own testcases now, so replace the existing shell script with an equivalent
ucode implementation.

Signed-off-by: Jo-Philipp Wich <jo@mein.io>

3 files changed, 256 insertions, 242 deletions

diff --git a/tests/custom/CMakeLists.txt b/tests/custom/CMakeLists.txt
index c8007a0..c94278e 100644
--- a/tests/custom/CMakeLists.txt
+++ b/tests/custom/CMakeLists.txt
@@ -1,6 +1,6 @@
 ADD_TEST(
 	NAME custom
-	COMMAND run_tests.sh
+	COMMAND $<TARGET_FILE:ucode> -L $<TARGET_FILE_DIR:fs_lib>/*.so -S run_tests.uc
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT
@@ -11,7 +11,7 @@ SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT
 IF(CMAKE_C_COMPILER_ID STREQUAL "Clang")
 	ADD_TEST(
 		NAME custom-san
-		COMMAND run_tests.sh
+		COMMAND $<TARGET_FILE:ucode> -L $<TARGET_FILE_DIR:fs_lib>/*.so -S run_tests.uc
 		WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 	)
 
diff --git a/tests/custom/run_tests.sh b/tests/custom/run_tests.sh
deleted file mode 100755
index 96ac783..0000000
--- a/tests/custom/run_tests.sh
+++ /dev/null
@@ -1,240 +0,0 @@
-#!/usr/bin/env bash
-
-if greadlink -f . &>/dev/null; then
-	readlink=greadlink
-else
-	readlink=readlink
-fi
-
-testdir=$(dirname "$0")
-topdir=$($readlink -f "$testdir/../..")
-
-line='........................................'
-ucode_bin=${UCODE_BIN:-"$topdir/ucode"}
-ucode_lib=${UCODE_LIB:-"$topdir"}
-
-extract_sections() {
-	local file=$1
-	local dir=$2
-	local count=0
-	local tag line outfile
-
-	while IFS= read -r line; do
-		case "$line" in
-			"-- Args --")
-				tag="args"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.args" "$dir" $count)
-				printf "" > "$outfile"
-			;;
-			"-- Vars --")
-				tag="vars"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.vars" "$dir" $count)
-				printf "" > "$outfile"
-			;;
-			"-- Testcase --")
-				tag="test"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.in" "$dir" $count)
-				printf "" > "$outfile"
-			;;
-			"-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --")
-				tag="${line#-- Expect }"
-				tag="${tag% --}"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag")
-				printf "" > "$outfile"
-			;;
-			"-- File "*" --")
-				tag="file"
-				outfile="${line#-- File }"
-				outfile="$(echo "${outfile% --}" | xargs)"
-				outfile="$dir/files$($readlink -m "/${outfile:-file}")"
-				mkdir -p "$(dirname "$outfile")"
-				printf "" > "$outfile"
-			;;
-			"-- End (no-eol) --")
-				truncate -s -1 "$outfile"
-				tag=""
-				outfile=""
-			;;
-			"-- End --")
-				tag=""
-				outfile=""
-			;;
-			*)
-				if [ -n "$tag" ]; then
-					printf "%s\\n" "$line" >> "$outfile"
-				fi
-			;;
-		esac
-	done < "$file"
-
-	return $(ls -l "$dir/"*.in 2>/dev/null | wc -l)
-}
-
-run_testcase() {
-	local num=$1
-	local dir=$2
-	local in=$3
-	local out=$4
-	local err=$5
-	local code=$6
-	local args=$7
-	local vars=$8
-	local fail=0
-
-	(
-		cd "$dir"
-
-		IFS=$'\n'
-
-		local var
-		for var in $vars; do
-			case "$var" in
-				*=*) export "$var" ;;
-			esac
-		done
-
-		IFS=$' \t\n'
-
-		$ucode_bin -T"," -L "$ucode_lib/*.so" -D TESTFILES_PATH="$($readlink -f "$dir/files")" $args - <"$in" >"$dir/res.out" 2>"$dir/res.err"
-	)
-
-	printf "%d\n" $? > "$dir/res.code"
-	touch "$dir/empty"
-
-	sed -i -e "s#$dir#.#g" "$dir/res.out" "$dir/res.err"
-
-	if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then
-		[ $fail = 0 ] && printf "!\n"
-		printf "Testcase #%d: Expected stderr did not match:\n" $num
-		diff -au --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err"
-		printf -- "---\n"
-		fail=1
-	fi
-
-	if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then
-		[ $fail = 0 ] && printf "!\n"
-		printf "Testcase #%d: Expected stdout did not match:\n" $num
-		diff -au --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out"
-		printf -- "---\n"
-		fail=1
-	fi
-
-	if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then
-		[ $fail = 0 ] && printf "!\n"
-		printf "Testcase #%d: Expected exit code did not match:\n" $num
-		diff -au --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code"
-		printf -- "---\n"
-		fail=1
-	fi
-
-	return $fail
-}
-
-run_test() {
-	local file=$1
-	local name=${file##*/}
-	local res ecode eout eerr ein eargs tests
-	local testcase_first=0 failed=0 count=0
-
-	printf "%s %s " "$name" "${line:${#name}}"
-
-	mkdir "/tmp/test.$$"
-
-	extract_sections "$file" "/tmp/test.$$"
-	tests=$?
-
-	[ -f "/tmp/test.$$/001.in" ] && testcase_first=1
-
-	for res in "/tmp/test.$$/"[0-9]*; do
-		case "$res" in
-			*.in)
-				count=$((count + 1))
-
-				if [ $testcase_first = 1 ]; then
-					# Flush previous test
-					if [ -n "$ein" ]; then
-						run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1))
-						eout=""
-						eerr=""
-						ecode=""
-						eargs=""
-						evars=""
-					fi
-
-					ein=$res
-				else
-					run_testcase $count "/tmp/test.$$" "$res" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1))
-
-					eout=""
-					eerr=""
-					ecode=""
-					eargs=""
-					evars=""
-				fi
-
-			;;
-			*.stdout) eout=$res ;;
-			*.stderr) eerr=$res ;;
-			*.exitcode) ecode=$res ;;
-			*.args) eargs=$(cat "$res") ;;
-			*.vars) evars=$(cat "$res") ;;
-		esac
-	done
-
-	# Flush last test
-	if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then
-		run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1))
-	fi
-
-	rm -r "/tmp/test.$$"
-
-	if [ $failed = 0 ]; then
-		printf "OK\n"
-	else
-		printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests
-	fi
-
-	return $failed
-}
-
-
-n_tests=0
-n_fails=0
-
-select_tests="$@"
-
-use_test() {
-	local input="$($readlink -f "$1")"
-	local test
-
-	[ -f "$input" ] || return 1
-	[ -n "$select_tests" ] || return 0
-
-	for test in "$select_tests"; do
-		test="$($readlink -f "$test")"
-
-		[ "$test" != "$input" ] || return 0
-	done
-
-	return 1
-}
-
-for catdir in "$testdir/"[0-9][0-9]_*; do
-	[ -d "$catdir" ] || continue
-
-	printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}"
-
-	for testfile in "$catdir/"[0-9][0-9]_*; do
-		use_test "$testfile" || continue
-
-		n_tests=$((n_tests + 1))
-		run_test "$testfile" || n_fails=$((n_fails + 1))
-	done
-done
-
-printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails
-exit $n_fails
diff --git a/tests/custom/run_tests.uc b/tests/custom/run_tests.uc
new file mode 100755
index 0000000..ff81afb
--- /dev/null
+++ b/tests/custom/run_tests.uc
@@ -0,0 +1,254 @@
+#!/usr/bin/env -S ucode -S
+
+import * as fs from 'fs';
+
+let testdir = sourcepath(0, true);
+let topdir = fs.realpath(`${testdir}/../..`);
+
+let line = '........................................';
+let ucode_bin = getenv('UCODE_BIN') || `${topdir}/ucode`;
+let ucode_lib = getenv('UCODE_LIB') || topdir;
+
+function mkdir_p(path) {
+	let parts = split(rtrim(path, '/') || '/', /\/+/);
+	let current = '';
+
+	for (let part in parts) {
+		current += part + '/';
+
+		let s = fs.stat(current);
+
+		if (s == null) {
+			if (!fs.mkdir(current))
+				die(`Error creating directory '${current}': ${fs.error()}`);
+		}
+		else if (s.type != 'directory') {
+			die(`Path '${current}' exists but is not a directory`);
+		}
+	}
+}
+
+function shellquote(s) {
+	return `'${replace(s, "'", "'\\''")}'`;
+}
+
+function getpid() {
+	return +fs.popen('echo $PPID', 'r').read('all');
+}
+
+function has_expectations(testcase)
+{
+	return (testcase?.stdout != null || testcase?.stderr != null || testcase?.exitcode != null);
+}
+
+function parse_testcases(file, dir) {
+	let fp = fs.open(file, 'r') ?? die(`Unable to open ${file}: ${fs.error()}`);
+	let testcases, testcase, section, m;
+	let code_first = false;
+
+	for (let line = fp.read('line'); length(line); line = fp.read('line')) {
+		if (line == '-- Args --\n') {
+			section = [ 'args', [] ];
+		}
+		else if (line == '-- Vars --\n') {
+			section = [ 'env', {} ];
+		}
+		else if (line == '-- Testcase --\n') {
+			section = [ 'code', '' ];
+		}
+		else if ((m = match(line, /^-- Expect (stdout|stderr|exitcode) --$/s)) != null) {
+			section = [ m[1], '' ];
+		}
+		else if ((m = match(line, /^-- File (.*)--$/s)) != null) {
+			section = [ 'file', `${dir}/files/${trim(m[1]) || 'file'}`, '' ];
+		}
+		else if ((m = match(line, /^-- End( \(no-eol\))? --$/s)) != null) {
+			if (m[1] != null && type(section[-1]) == 'string')
+				section[-1] = substr(section[-1], 0, -1);
+
+			if (section[0] == 'code') {
+				if (testcases == null && !has_expectations(testcase))
+					code_first = true;
+
+				if (code_first) {
+					if (testcase?.code != null) {
+						push(testcases ??= [], testcase);
+						testcase = null;
+					}
+
+					(testcase ??= {}).code = section[1];
+				}
+				else {
+					push(testcases ??= [], { ...testcase, code: section[1] });
+					testcase = null;
+				}
+			}
+			else if (section[0] == 'file') {
+				((testcase ??= {}).files ??= {})[section[1]] = section[2];
+			}
+			else {
+				(testcase ??= {})[section[0]] = section[1];
+			}
+
+			section = null;
+		}
+		else if (section) {
+			switch (section[0]) {
+			case 'args':
+				if ((m = trim(line)) != '')
+					push(section[1], ...split(m, /[ \t\r\n]+/));
+				break;
+
+			case 'env':
+				if ((m = match(line, /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s)) != null)
+					section[1][m[1]] = m[2];
+				break;
+
+			default:
+				section[-1] += line;
+				break;
+			}
+		}
+	}
+
+	if (code_first && testcase.code != null && has_expectations(testcase))
+		push(testcases ??= [], testcase);
+
+	return testcases;
+}
+
+function diff(tag, ...ab) {
+	let cmd = [ 'diff', '-au', '--color=always', `--label=Expected ${tag}`, `--label=Resulting ${tag}` ];
+	let tmpfiles = [];
+
+	for (let i, f in ab) {
+		if (type(f) != 'resource') {
+			push(tmpfiles, fs.mkstemp());
+			tmpfiles[-1].write(f);
+			f = tmpfiles[-1];
+		}
+
+		f.seek(0);
+		push(cmd, `/dev/fd/${f.fileno()}`);
+	}
+
+	system(cmd);
+}
+
+function run_testcase(num, dir, testcase) {
+	let fout = fs.mkstemp(`${dir}/stdout.XXXXXX`);
+	let ferr = fs.mkstemp(`${dir}/stderr.XXXXXX`);
+
+	let eout = testcase.stdout ?? '';
+	let eerr = testcase.stderr ?? '';
+	let ecode = testcase.exitcode ? +testcase.exitcode : null;
+
+	let cmd = join(' ', [
+		...map(keys(testcase.env) ?? [], k => `export ${k}=${shellquote(testcase.env[k])};`),
+		`cd ${shellquote(dir)};`,
+		`exec ${ucode_bin}`,
+		`-T','`,
+		`-L ${shellquote(`${ucode_lib}/*.so`)}`,
+		`-D TESTFILES_PATH=${shellquote(`${fs.realpath(dir)}/files`)}`,
+		`${join(' ', map(testcase.args ?? [], shellquote))} -`,
+		`>/dev/fd/${fout.fileno()} 2>/dev/fd/${ferr.fileno()}`
+	]);
+
+	let proc = fs.popen(cmd, 'w') ?? die(`Error launching test command "${cmd}": ${fs.error()}\n`);
+
+	if (testcase.code != null)
+		proc.write(testcase.code);
+
+	let exitcode = proc.close();
+
+	fout.seek(0);
+	ferr.seek(0);
+
+	let ok = true;
+
+	if (replace(ferr.read('all'), dir, '.') != eerr) {
+		if (ok) print('!\n');
+		printf("Testcase #%d: Expected stderr did not match:\n", num);
+		diff('stderr', eerr, ferr);
+		print("---\n");
+		ok = false;
+	}
+
+	if (replace(fout.read('all'), dir, '.') != eout) {
+		if (ok) print('!\n');
+		printf("Testcase #%d: Expected stdout did not match:\n", num);
+		diff('stdout', eout, fout);
+		print("---\n");
+		ok = false;
+	}
+
+	if (ecode != null && exitcode != ecode) {
+		if (ok) print('!\n');
+		printf("Testcase #%d: Expected exit code did not match:\n", num);
+		diff('code', `${ecode}\n`, `${exitcode}\n`);
+		print("---\n");
+		ok = false;
+	}
+
+	return ok;
+}
+
+function run_test(file) {
+	let name = fs.basename(file);
+	printf('%s %s ', name, substr(line, length(name)));
+
+	let tmpdir = sprintf('/tmp/test.%d', getpid());
+	let testcases = parse_testcases(file, tmpdir);
+	let failed = 0;
+
+	fs.mkdir(tmpdir);
+
+	try {
+		for (let i, testcase in testcases) {
+			for (let path, data in testcase.files) {
+				mkdir_p(fs.dirname(path));
+				fs.writefile(path, data) ?? die(`Error writing testcase file "${path}": ${fs.error()}\n`);
+			}
+
+			failed += !run_testcase(i + 1, tmpdir, testcase);
+		}
+	}
+	catch (e) {
+		warn(`${e.type}: ${e.message}\n${e.stacktrace[0].context}\n`);
+	}
+
+	system(['rm', '-r', tmpdir]);
+
+	if (failed == 0)
+		print('OK\n');
+	else
+		printf('%s %s FAILED (%d/%d)\n', name, substr(line, length(name)), failed, length(testcases));
+
+	return failed;
+}
+
+let n_tests = 0;
+let n_fails = 0;
+let select_tests = filter(map(ARGV, p => fs.realpath(p)), length);
+
+function use_test(input) {
+	return fs.access(input = fs.realpath(input)) &&
+		(!length(select_tests) || filter(select_tests, p => p == input)[0]);
+}
+
+for (let catdir in fs.glob(`${testdir}/[0-9][0-9]_*`)) {
+	if (fs.stat(catdir)?.type != 'directory')
+		continue;
+
+	printf('\n##\n## Running %s tests\n##\n\n', substr(fs.basename(catdir), 3));
+
+	for (let testfile in fs.glob(`${catdir}/[0-9][0-9]_*`)) {
+		if (!use_test(testfile)) continue;
+
+		n_tests++;
+		n_fails += run_test(testfile);
+	}
+}
+
+printf('\nRan %d tests, %d okay, %d failures\n', n_tests, n_tests - n_fails, n_fails);
+exit(n_fails);