vm: stop executing bytecode on return of nested calls

When a managed function is indirectly invoked during bytecode execution,
e.g. when calling the tostring() method of an object prototype during
string concatenation, the invoked function must stop executing bytecode
upon return to hand control back to caller.

Extend `uc_vm_execute_chunk()` to track the amount of nested function
calls it performs and hand back control to the caller once the toplevel
callframe returns. Also bubble unhandled exceptions only as far as up
to the original caller.

Signed-off-by: Jo-Philipp Wich <jo@mein.io>

2 files changed, 62 insertions, 11 deletions

diff --git a/tests/custom/04_bugs/36_vm_nested_call_return b/tests/custom/04_bugs/36_vm_nested_call_return
new file mode 100644
index 0000000..6a52b78
--- /dev/null
+++ b/tests/custom/04_bugs/36_vm_nested_call_return
@@ -0,0 +1,52 @@
+When indirectly invoking a managed function from manged code, e.g.
+on stringifying an object using it's tostring() prototype method
+during string concatenation, bytecode execution of the nested managed
+function call did not stop and return to the caller, but continued
+past the return of the invoked function, clobbering the VM context.
+
+
+-- Testcase --
+{%
+	let o = proto(
+		{ color: "red" },
+		{ tostring: function() { return "I am a " + this.color + " object" } }
+	);
+
+	print("Result: " + o + ".\n");
+%}
+-- End --
+
+-- Expect stdout --
+Result: I am a red object.
+-- End --
+
+
+-- Testcase --
+{%
+	let o = proto(
+		{ color: "red" },
+		{ tostring: function() { die("Exception while stringifying") } }
+	);
+
+	function t() {
+		try {
+			print("Result: " + o + ".\n");
+		}
+		catch (e) {
+			warn("Caught exception: " + e.stacktrace[0].context + "\n");
+		}
+	}
+
+	t();
+%}
+-- End --
+
+-- Expect stderr --
+Caught exception: In [anonymous function](), line 4, byte 62:
+  called from function t ([stdin]:9:23)
+  called from anonymous function ([stdin]:16:4)
+
+ `        { tostring: function() { die("Exception while stringifying") } }`
+  Near here ---------------------------------------------------------^
+
+-- End --
diff --git a/vm.c b/vm.c
index 7d4c10e..29ace38 100644
--- a/vm.c
+++ b/vm.c
@@ -2358,11 +2358,12 @@ uc_vm_execute_chunk(uc_vm_t *vm)
 {
 	uc_callframe_t *frame = uc_vm_current_frame(vm);
 	uc_chunk_t *chunk = uc_vm_frame_chunk(frame);
+	size_t caller = vm->callframes.count - 1;
 	uc_value_t *retval;
 	uc_vm_insn_t insn;
 	uint8_t *ip;
 
-	while (chunk) {
+	while (chunk && vm->callframes.count > caller) {
 		if (vm->trace) {
 			ip = frame->ip;
 			insn = uc_vm_decode_insn(vm, frame, chunk);
@@ -2596,23 +2597,21 @@ uc_vm_execute_chunk(uc_vm_t *vm)
 				return STATUS_EXIT;
 			}
 
-			/* walk up callframes until something handles the exception or the root is reached */
+			/* walk up callframes until something handles the exception or the original caller is reached */
 			while (!uc_vm_handle_exception(vm)) {
-				/* no further callframe to pop, report unhandled exception and terminate */
-				if (vm->callframes.count <= 1) {
-					uc_vm_reset_callframes(vm);
-
-					return ERROR_RUNTIME;
-				}
-
 				/* if VM returned into native function, don't bubble up */
 				if (!chunk)
 					return ERROR_RUNTIME;
 
 				/* no exception handler in current function, pop callframe */
-				ucv_put(uc_vm_callframe_pop(vm));
+				if (vm->callframes.count > 0)
+					ucv_put(uc_vm_callframe_pop(vm));
+
+				/* no further callframe, report unhandled exception and terminate */
+				if (vm->callframes.count == 0 || vm->callframes.count <= caller)
+					return ERROR_RUNTIME;
 
-				/* resume execution at topmost remaining callframe */
+				/* resume execution in next remaining callframe */
 				frame = uc_vector_last(&vm->callframes);
 				chunk = uc_vm_frame_chunk(frame);
 			}