Merge pull request #214 from jow-/ucode-test-runnerHEADmaster

tests: replace test runner shell script with ucode implementation

4 files changed, 264 insertions, 242 deletions

diff --git a/main.c b/main.c
index a51c370..96f4adb 100644
--- a/main.c
+++ b/main.c
@@ -201,6 +201,14 @@ read_stdin(void)
 
 	stdin_unused = NULL;
 
+	/* On empty stdin, provide a dummy buffer and ensure that it is
+	 * at least one byte long, due to
+	 * https://github.com/google/sanitizers/issues/627 */
+	if (p == NULL) {
+		p = xstrdup("\n");
+		tlen = 1;
+	}
+
 	return uc_source_new_buffer("[stdin]", p, tlen);
 }
 
diff --git a/tests/custom/CMakeLists.txt b/tests/custom/CMakeLists.txt
index c8007a0..c94278e 100644
--- a/tests/custom/CMakeLists.txt
+++ b/tests/custom/CMakeLists.txt
@@ -1,6 +1,6 @@
 ADD_TEST(
 	NAME custom
-	COMMAND run_tests.sh
+	COMMAND $<TARGET_FILE:ucode> -L $<TARGET_FILE_DIR:fs_lib>/*.so -S run_tests.uc
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT
@@ -11,7 +11,7 @@ SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT
 IF(CMAKE_C_COMPILER_ID STREQUAL "Clang")
 	ADD_TEST(
 		NAME custom-san
-		COMMAND run_tests.sh
+		COMMAND $<TARGET_FILE:ucode> -L $<TARGET_FILE_DIR:fs_lib>/*.so -S run_tests.uc
 		WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 	)
 
diff --git a/tests/custom/run_tests.sh b/tests/custom/run_tests.sh
deleted file mode 100755
index 96ac783..0000000
--- a/tests/custom/run_tests.sh
+++ /dev/null
@@ -1,240 +0,0 @@
-#!/usr/bin/env bash
-
-if greadlink -f . &>/dev/null; then
-	readlink=greadlink
-else
-	readlink=readlink
-fi
-
-testdir=$(dirname "$0")
-topdir=$($readlink -f "$testdir/../..")
-
-line='........................................'
-ucode_bin=${UCODE_BIN:-"$topdir/ucode"}
-ucode_lib=${UCODE_LIB:-"$topdir"}
-
-extract_sections() {
-	local file=$1
-	local dir=$2
-	local count=0
-	local tag line outfile
-
-	while IFS= read -r line; do
-		case "$line" in
-			"-- Args --")
-				tag="args"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.args" "$dir" $count)
-				printf "" > "$outfile"
-			;;
-			"-- Vars --")
-				tag="vars"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.vars" "$dir" $count)
-				printf "" > "$outfile"
-			;;
-			"-- Testcase --")
-				tag="test"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.in" "$dir" $count)
-				printf "" > "$outfile"
-			;;
-			"-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --")
-				tag="${line#-- Expect }"
-				tag="${tag% --}"
-				count=$((count + 1))
-				outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag")
-				printf "" > "$outfile"
-			;;
-			"-- File "*" --")
-				tag="file"
-				outfile="${line#-- File }"
-				outfile="$(echo "${outfile% --}" | xargs)"
-				outfile="$dir/files$($readlink -m "/${outfile:-file}")"
-				mkdir -p "$(dirname "$outfile")"
-				printf "" > "$outfile"
-			;;
-			"-- End (no-eol) --")
-				truncate -s -1 "$outfile"
-				tag=""
-				outfile=""
-			;;
-			"-- End --")
-				tag=""
-				outfile=""
-			;;
-			*)
-				if [ -n "$tag" ]; then
-					printf "%s\\n" "$line" >> "$outfile"
-				fi
-			;;
-		esac
-	done < "$file"
-
-	return $(ls -l "$dir/"*.in 2>/dev/null | wc -l)
-}
-
-run_testcase() {
-	local num=$1
-	local dir=$2
-	local in=$3
-	local out=$4
-	local err=$5
-	local code=$6
-	local args=$7
-	local vars=$8
-	local fail=0
-
-	(
-		cd "$dir"
-
-		IFS=$'\n'
-
-		local var
-		for var in $vars; do
-			case "$var" in
-				*=*) export "$var" ;;
-			esac
-		done
-
-		IFS=$' \t\n'
-
-		$ucode_bin -T"," -L "$ucode_lib/*.so" -D TESTFILES_PATH="$($readlink -f "$dir/files")" $args - <"$in" >"$dir/res.out" 2>"$dir/res.err"
-	)
-
-	printf "%d\n" $? > "$dir/res.code"
-	touch "$dir/empty"
-
-	sed -i -e "s#$dir#.#g" "$dir/res.out" "$dir/res.err"
-
-	if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then
-		[ $fail = 0 ] && printf "!\n"
-		printf "Testcase #%d: Expected stderr did not match:\n" $num
-		diff -au --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err"
-		printf -- "---\n"
-		fail=1
-	fi
-
-	if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then
-		[ $fail = 0 ] && printf "!\n"
-		printf "Testcase #%d: Expected stdout did not match:\n" $num
-		diff -au --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out"
-		printf -- "---\n"
-		fail=1
-	fi
-
-	if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then
-		[ $fail = 0 ] && printf "!\n"
-		printf "Testcase #%d: Expected exit code did not match:\n" $num
-		diff -au --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code"
-		printf -- "---\n"
-		fail=1
-	fi
-
-	return $fail
-}
-
-run_test() {
-	local file=$1
-	local name=${file##*/}
-	local res ecode eout eerr ein eargs tests
-	local testcase_first=0 failed=0 count=0
-
-	printf "%s %s " "$name" "${line:${#name}}"
-
-	mkdir "/tmp/test.$$"
-
-	extract_sections "$file" "/tmp/test.$$"
-	tests=$?
-
-	[ -f "/tmp/test.$$/001.in" ] && testcase_first=1
-
-	for res in "/tmp/test.$$/"[0-9]*; do
-		case "$res" in
-			*.in)
-				count=$((count + 1))
-
-				if [ $testcase_first = 1 ]; then
-					# Flush previous test
-					if [ -n "$ein" ]; then
-						run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1))
-						eout=""
-						eerr=""
-						ecode=""
-						eargs=""
-						evars=""
-					fi
-
-					ein=$res
-				else
-					run_testcase $count "/tmp/test.$$" "$res" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1))
-
-					eout=""
-					eerr=""
-					ecode=""
-					eargs=""
-					evars=""
-				fi
-
-			;;
-			*.stdout) eout=$res ;;
-			*.stderr) eerr=$res ;;
-			*.exitcode) ecode=$res ;;
-			*.args) eargs=$(cat "$res") ;;
-			*.vars) evars=$(cat "$res") ;;
-		esac
-	done
-
-	# Flush last test
-	if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then
-		run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1))
-	fi
-
-	rm -r "/tmp/test.$$"
-
-	if [ $failed = 0 ]; then
-		printf "OK\n"
-	else
-		printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests
-	fi
-
-	return $failed
-}
-
-
-n_tests=0
-n_fails=0
-
-select_tests="$@"
-
-use_test() {
-	local input="$($readlink -f "$1")"
-	local test
-
-	[ -f "$input" ] || return 1
-	[ -n "$select_tests" ] || return 0
-
-	for test in "$select_tests"; do
-		test="$($readlink -f "$test")"
-
-		[ "$test" != "$input" ] || return 0
-	done
-
-	return 1
-}
-
-for catdir in "$testdir/"[0-9][0-9]_*; do
-	[ -d "$catdir" ] || continue
-
-	printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}"
-
-	for testfile in "$catdir/"[0-9][0-9]_*; do
-		use_test "$testfile" || continue
-
-		n_tests=$((n_tests + 1))
-		run_test "$testfile" || n_fails=$((n_fails + 1))
-	done
-done
-
-printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails
-exit $n_fails
diff --git a/tests/custom/run_tests.uc b/tests/custom/run_tests.uc
new file mode 100755
index 0000000..ff81afb
--- /dev/null
+++ b/tests/custom/run_tests.uc
@@ -0,0 +1,254 @@
+#!/usr/bin/env -S ucode -S
+
+import * as fs from 'fs';
+
+let testdir = sourcepath(0, true);
+let topdir = fs.realpath(`${testdir}/../..`);
+
+let line = '........................................';
+let ucode_bin = getenv('UCODE_BIN') || `${topdir}/ucode`;
+let ucode_lib = getenv('UCODE_LIB') || topdir;
+
+function mkdir_p(path) {
+	let parts = split(rtrim(path, '/') || '/', /\/+/);
+	let current = '';
+
+	for (let part in parts) {
+		current += part + '/';
+
+		let s = fs.stat(current);
+
+		if (s == null) {
+			if (!fs.mkdir(current))
+				die(`Error creating directory '${current}': ${fs.error()}`);
+		}
+		else if (s.type != 'directory') {
+			die(`Path '${current}' exists but is not a directory`);
+		}
+	}
+}
+
+function shellquote(s) {
+	return `'${replace(s, "'", "'\\''")}'`;
+}
+
+function getpid() {
+	return +fs.popen('echo $PPID', 'r').read('all');
+}
+
+function has_expectations(testcase)
+{
+	return (testcase?.stdout != null || testcase?.stderr != null || testcase?.exitcode != null);
+}
+
+function parse_testcases(file, dir) {
+	let fp = fs.open(file, 'r') ?? die(`Unable to open ${file}: ${fs.error()}`);
+	let testcases, testcase, section, m;
+	let code_first = false;
+
+	for (let line = fp.read('line'); length(line); line = fp.read('line')) {
+		if (line == '-- Args --\n') {
+			section = [ 'args', [] ];
+		}
+		else if (line == '-- Vars --\n') {
+			section = [ 'env', {} ];
+		}
+		else if (line == '-- Testcase --\n') {
+			section = [ 'code', '' ];
+		}
+		else if ((m = match(line, /^-- Expect (stdout|stderr|exitcode) --$/s)) != null) {
+			section = [ m[1], '' ];
+		}
+		else if ((m = match(line, /^-- File (.*)--$/s)) != null) {
+			section = [ 'file', `${dir}/files/${trim(m[1]) || 'file'}`, '' ];
+		}
+		else if ((m = match(line, /^-- End( \(no-eol\))? --$/s)) != null) {
+			if (m[1] != null && type(section[-1]) == 'string')
+				section[-1] = substr(section[-1], 0, -1);
+
+			if (section[0] == 'code') {
+				if (testcases == null && !has_expectations(testcase))
+					code_first = true;
+
+				if (code_first) {
+					if (testcase?.code != null) {
+						push(testcases ??= [], testcase);
+						testcase = null;
+					}
+
+					(testcase ??= {}).code = section[1];
+				}
+				else {
+					push(testcases ??= [], { ...testcase, code: section[1] });
+					testcase = null;
+				}
+			}
+			else if (section[0] == 'file') {
+				((testcase ??= {}).files ??= {})[section[1]] = section[2];
+			}
+			else {
+				(testcase ??= {})[section[0]] = section[1];
+			}
+
+			section = null;
+		}
+		else if (section) {
+			switch (section[0]) {
+			case 'args':
+				if ((m = trim(line)) != '')
+					push(section[1], ...split(m, /[ \t\r\n]+/));
+				break;
+
+			case 'env':
+				if ((m = match(line, /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s)) != null)
+					section[1][m[1]] = m[2];
+				break;
+
+			default:
+				section[-1] += line;
+				break;
+			}
+		}
+	}
+
+	if (code_first && testcase.code != null && has_expectations(testcase))
+		push(testcases ??= [], testcase);
+
+	return testcases;
+}
+
+function diff(tag, ...ab) {
+	let cmd = [ 'diff', '-au', '--color=always', `--label=Expected ${tag}`, `--label=Resulting ${tag}` ];
+	let tmpfiles = [];
+
+	for (let i, f in ab) {
+		if (type(f) != 'resource') {
+			push(tmpfiles, fs.mkstemp());
+			tmpfiles[-1].write(f);
+			f = tmpfiles[-1];
+		}
+
+		f.seek(0);
+		push(cmd, `/dev/fd/${f.fileno()}`);
+	}
+
+	system(cmd);
+}
+
+function run_testcase(num, dir, testcase) {
+	let fout = fs.mkstemp(`${dir}/stdout.XXXXXX`);
+	let ferr = fs.mkstemp(`${dir}/stderr.XXXXXX`);
+
+	let eout = testcase.stdout ?? '';
+	let eerr = testcase.stderr ?? '';
+	let ecode = testcase.exitcode ? +testcase.exitcode : null;
+
+	let cmd = join(' ', [
+		...map(keys(testcase.env) ?? [], k => `export ${k}=${shellquote(testcase.env[k])};`),
+		`cd ${shellquote(dir)};`,
+		`exec ${ucode_bin}`,
+		`-T','`,
+		`-L ${shellquote(`${ucode_lib}/*.so`)}`,
+		`-D TESTFILES_PATH=${shellquote(`${fs.realpath(dir)}/files`)}`,
+		`${join(' ', map(testcase.args ?? [], shellquote))} -`,
+		`>/dev/fd/${fout.fileno()} 2>/dev/fd/${ferr.fileno()}`
+	]);
+
+	let proc = fs.popen(cmd, 'w') ?? die(`Error launching test command "${cmd}": ${fs.error()}\n`);
+
+	if (testcase.code != null)
+		proc.write(testcase.code);
+
+	let exitcode = proc.close();
+
+	fout.seek(0);
+	ferr.seek(0);
+
+	let ok = true;
+
+	if (replace(ferr.read('all'), dir, '.') != eerr) {
+		if (ok) print('!\n');
+		printf("Testcase #%d: Expected stderr did not match:\n", num);
+		diff('stderr', eerr, ferr);
+		print("---\n");
+		ok = false;
+	}
+
+	if (replace(fout.read('all'), dir, '.') != eout) {
+		if (ok) print('!\n');
+		printf("Testcase #%d: Expected stdout did not match:\n", num);
+		diff('stdout', eout, fout);
+		print("---\n");
+		ok = false;
+	}
+
+	if (ecode != null && exitcode != ecode) {
+		if (ok) print('!\n');
+		printf("Testcase #%d: Expected exit code did not match:\n", num);
+		diff('code', `${ecode}\n`, `${exitcode}\n`);
+		print("---\n");
+		ok = false;
+	}
+
+	return ok;
+}
+
+function run_test(file) {
+	let name = fs.basename(file);
+	printf('%s %s ', name, substr(line, length(name)));
+
+	let tmpdir = sprintf('/tmp/test.%d', getpid());
+	let testcases = parse_testcases(file, tmpdir);
+	let failed = 0;
+
+	fs.mkdir(tmpdir);
+
+	try {
+		for (let i, testcase in testcases) {
+			for (let path, data in testcase.files) {
+				mkdir_p(fs.dirname(path));
+				fs.writefile(path, data) ?? die(`Error writing testcase file "${path}": ${fs.error()}\n`);
+			}
+
+			failed += !run_testcase(i + 1, tmpdir, testcase);
+		}
+	}
+	catch (e) {
+		warn(`${e.type}: ${e.message}\n${e.stacktrace[0].context}\n`);
+	}
+
+	system(['rm', '-r', tmpdir]);
+
+	if (failed == 0)
+		print('OK\n');
+	else
+		printf('%s %s FAILED (%d/%d)\n', name, substr(line, length(name)), failed, length(testcases));
+
+	return failed;
+}
+
+let n_tests = 0;
+let n_fails = 0;
+let select_tests = filter(map(ARGV, p => fs.realpath(p)), length);
+
+function use_test(input) {
+	return fs.access(input = fs.realpath(input)) &&
+		(!length(select_tests) || filter(select_tests, p => p == input)[0]);
+}
+
+for (let catdir in fs.glob(`${testdir}/[0-9][0-9]_*`)) {
+	if (fs.stat(catdir)?.type != 'directory')
+		continue;
+
+	printf('\n##\n## Running %s tests\n##\n\n', substr(fs.basename(catdir), 3));
+
+	for (let testfile in fs.glob(`${catdir}/[0-9][0-9]_*`)) {
+		if (!use_test(testfile)) continue;
+
+		n_tests++;
+		n_fails += run_test(testfile);
+	}
+}
+
+printf('\nRan %d tests, %d okay, %d failures\n', n_tests, n_tests - n_fails, n_fails);
+exit(n_fails);