Filter: Stacks moved to thread-local storage if available.

3 files changed, 74 insertions, 72 deletions

diff --git a/filter/decl.m4 b/filter/decl.m4
index d8497535..9934d0ba 100644
--- a/filter/decl.m4
+++ b/filter/decl.m4
@@ -137,7 +137,7 @@ break;
 FID_INTERPRET
 case INST_NAME():
 #define whati (&(what->i_]]INST_NAME()[[))
-m4_ifelse(m4_eval(INST_INVAL() > 0), 1, [[if (vstk.cnt < INST_INVAL()) runtime("Stack underflow"); vstk.cnt -= INST_INVAL(); ]])
+m4_ifelse(m4_eval(INST_INVAL() > 0), 1, [[if (fstk->vcnt < INST_INVAL()) runtime("Stack underflow"); fstk->vcnt -= INST_INVAL(); ]])
 m4_undivert(108)
 #undef whati
 break;
@@ -206,12 +206,12 @@ FID_ALL()')
 
 m4_define(LINEX, `FID_INTERPRET_BODY
 do {
-  estk.item[estk.cnt].pos = 0;
-  estk.item[estk.cnt].line = $1;
-  estk.item[estk.cnt].ventry = vstk.cnt;
-  estk.item[estk.cnt].vbase = estk.item[estk.cnt-1].vbase;
-  estk.item[estk.cnt].emask = 0;
-  estk.cnt++;
+  fstk->estk[fstk->ecnt].pos = 0;
+  fstk->estk[fstk->ecnt].line = $1;
+  fstk->estk[fstk->ecnt].ventry = fstk->vcnt;
+  fstk->estk[fstk->ecnt].vbase = fstk->estk[fstk->ecnt-1].vbase;
+  fstk->estk[fstk->ecnt].emask = 0;
+  fstk->ecnt++;
 } while (0)m4_dnl
 FID_ALL()')
 
@@ -236,7 +236,7 @@ do { if (whati->fl$1) {
 } } while(0)m4_dnl
 FID_ALL()')
 
-m4_define(RESULT_OK, `FID_INTERPRET_BODY()vstk.cnt++FID_ALL()')
+m4_define(RESULT_OK, `FID_INTERPRET_BODY()fstk->vcnt++FID_ALL()')
 m4_define(RESULT, `RESULT_VAL([[ (struct f_val) { .type = $1, .val.$2 = $3 } ]])')
 m4_define(RESULT_VAL, `FID_INTERPRET_BODY()do { res = $1; RESULT_OK; } while (0)FID_ALL()')
 
diff --git a/filter/f-inst.c b/filter/f-inst.c
index d0bfa95b..749e072c 100644
--- a/filter/f-inst.c
+++ b/filter/f-inst.c
@@ -156,12 +156,12 @@
       what->count = len;
     FID_ALL
 
-    if (vstk.cnt < whati->count) /* TODO: make this check systematic */
+    if (fstk->vcnt < whati->count) /* TODO: make this check systematic */
       runtime("Construction of BGP path mask from %u elements must have at least that number of elements", whati->count);
 
     struct f_path_mask *pm = lp_alloc(fs->pool, sizeof(struct f_path_mask) + whati->count * sizeof(struct f_path_mask_item));
     for (uint i=0; i<whati->count; i++) {
-#define pv vstk.val[vstk.cnt - whati->count + i]
+#define pv fstk->vstk[fstk->vcnt - whati->count + i]
       switch (pv.type) {
 	case T_PATH_MASK_ITEM:
 	  pm->item[i] = pv.val.pmi;
@@ -177,7 +177,7 @@
       }
     }
 
-    vstk.cnt -= whati->count;
+    fstk->vcnt -= whati->count;
     pm->len = whati->count;
 
     RESULT(T_PATH_MASK, path_mask, pm);
@@ -276,12 +276,12 @@
 	runtime( "Assigning to variable of incompatible type" );
     }
 
-    vstk.val[curline.vbase + sym->offset] = v1;
+    fstk->vstk[curline.vbase + sym->offset] = v1;
   }
 
   INST(FI_VAR_GET, 0, 1) {
     SYMBOL(1);
-    res = vstk.val[curline.vbase + sym->offset];
+    res = fstk->vstk[curline.vbase + sym->offset];
     RESULT_OK;
   }
 
@@ -747,16 +747,16 @@
   INST(FI_RETURN, 1, 1) {
     /* Acquire the return value */
     ARG_ANY(1);
-    uint retpos = vstk.cnt;
+    uint retpos = fstk->vcnt;
 
     /* Drop every sub-block including ourselves */
-    while ((estk.cnt-- > 0) && !(estk.item[estk.cnt].emask & FE_RETURN))
+    while ((fstk->ecnt-- > 0) && !(fstk->estk[fstk->ecnt].emask & FE_RETURN))
       ;
 
     /* Now we are at the caller frame; if no such, try to convert to accept/reject. */
-    if (!estk.cnt)
-      if (vstk.val[retpos].type == T_BOOL)
-	if (vstk.val[retpos].val.i)
+    if (!fstk->ecnt)
+      if (fstk->vstk[retpos].type == T_BOOL)
+	if (fstk->vstk[retpos].val.i)
 
 	  return F_ACCEPT;
 	else
@@ -765,10 +765,10 @@
 	runtime("Can't return non-bool from non-function");
 
     /* Set the value stack position, overwriting the former implicit void */
-    vstk.cnt = estk.item[estk.cnt].ventry - 1;
+    fstk->vcnt = fstk->estk[fstk->ecnt].ventry - 1;
 
     /* Copy the return value */
-    RESULT_VAL(vstk.val[retpos]);
+    RESULT_VAL(fstk->vstk[retpos]);
   }
 
   INST(FI_CALL, 0, 1) {
@@ -787,8 +787,8 @@
     curline.vbase = curline.ventry;
 
     /* Storage for local variables */
-    memset(&(vstk.val[vstk.cnt]), 0, sizeof(struct f_val) * sym->function->vars);
-    vstk.cnt += sym->function->vars;
+    memset(&(fstk->vstk[fstk->vcnt]), 0, sizeof(struct f_val) * sym->function->vars);
+    fstk->vcnt += sym->function->vars;
   }
 
   INST(FI_DROP_RESULT, 1, 0) {
diff --git a/filter/filter.c b/filter/filter.c
index 65572583..c651253c 100644
--- a/filter/filter.c
+++ b/filter/filter.c
@@ -50,8 +50,37 @@
 #include "filter/f-inst.h"
 #include "filter/data.h"
 
+
+/* Exception bits */
+enum f_exception {
+  FE_RETURN = 0x1,
+};
+
+
+struct filter_stack {
+  /* Value stack for execution */
+#define F_VAL_STACK_MAX	4096
+  uint vcnt;				/* Current value stack size; 0 for empty */
+  uint ecnt;				/* Current execute stack size; 0 for empty */
+
+  struct f_val vstk[F_VAL_STACK_MAX];	/* The stack itself */
+
+  /* Instruction stack for execution */
+#define F_EXEC_STACK_MAX 4096
+  struct {
+    const struct f_line *line;		/* The line that is being executed */
+    uint pos;				/* Instruction index in the line */
+    uint ventry;			/* Value stack depth on entry */
+    uint vbase;				/* Where to index variable positions from */
+    enum f_exception emask;		/* Exception mask */
+  } estk[F_EXEC_STACK_MAX];
+};
+
 /* Internal filter state, to be allocated on stack when executing filters */
 struct filter_state {
+  /* Stacks needed for execution */
+  struct filter_stack *stack;
+
   /* The route we are processing. This may be NULL to indicate no route available. */
   struct rte **rte;
 
@@ -67,9 +96,10 @@ struct filter_state {
 
 #if HAVE_THREAD_LOCAL
 _Thread_local static struct filter_state filter_state;
-#define FS_INIT(...) filter_state = (struct filter_state) { __VA_ARGS__ }
+_Thread_local static struct filter_stack filter_stack;
+#define FS_INIT(...)	filter_state = (struct filter_state) { .stack = &filter_stack, __VA_ARGS__ }
 #else
-#define FS_INIT(...) struct filter_state filter_state = { __VA_ARGS__ }
+#define FS_INIT(...)	struct filter_state filter_state = { .stack = alloca(sizeof(struct filter_stack)), __VA_ARGS__ };
 #endif
 
 void (*bt_assert_hook)(int result, const struct f_line_item *assert);
@@ -145,61 +175,33 @@ interpret(struct filter_state *fs, const struct f_line *line, struct f_val *val)
   /* No arguments allowed */
   ASSERT(line->args == 0);
 
-#define F_VAL_STACK_MAX	4096
-  /* Value stack for execution */
-  struct f_val_stack {
-    uint cnt;				/* Current stack size; 0 for empty */
-    struct f_val val[F_VAL_STACK_MAX];	/* The stack itself */
-  } vstk;
-
-  /* The stack itself is intentionally kept as-is for performance reasons.
-   * Do NOT rewrite this to initialization by struct literal. It's slow.
-   *
-   * Reserving space for local variables. */
+  /* Initialize the filter stack */
+  struct filter_stack *fstk = fs->stack;
 
-  vstk.cnt = line->vars;
-  memset(vstk.val, 0, sizeof(struct f_val) * line->vars);
-
-#define F_EXEC_STACK_MAX 4096
-
-  /* Exception bits */
-  enum f_exception {
-    FE_RETURN = 0x1,
-  };
-
-  /* Instruction stack for execution */
-  struct f_exec_stack {
-    struct {
-      const struct f_line *line;		/* The line that is being executed */
-      uint pos;				/* Instruction index in the line */
-      uint ventry;			/* Value stack depth on entry */
-      uint vbase;			/* Where to index variable positions from */
-      enum f_exception emask;		/* Exception mask */
-    } item[F_EXEC_STACK_MAX];
-    uint cnt;				/* Current stack size; 0 for empty */
-  } estk;
+  fstk->vcnt = line->vars;
+  memset(fstk->vstk, 0, sizeof(struct f_val) * line->vars);
 
   /* The same as with the value stack. Not resetting the stack for performance reasons. */
-  estk.cnt = 1;
-  estk.item[0].line = line;		
-  estk.item[0].pos = 0;
+  fstk->ecnt = 1;
+  fstk->estk[0].line = line;		
+  fstk->estk[0].pos = 0;
 
-#define curline estk.item[estk.cnt-1]
+#define curline fstk->estk[fstk->ecnt-1]
 
 #if DEBUGGING
   debug("Interpreting line.");
   f_dump_line(line, 1);
 #endif
 
-  while (estk.cnt > 0) {
+  while (fstk->ecnt > 0) {
     while (curline.pos < curline.line->len) {
       const struct f_line_item *what = &(curline.line->items[curline.pos++]);
 
       switch (what->fi_code) {
-#define res vstk.val[vstk.cnt]
-#define v1 vstk.val[vstk.cnt]
-#define v2 vstk.val[vstk.cnt + 1]
-#define v3 vstk.val[vstk.cnt + 2]
+#define res fstk->vstk[fstk->vcnt]
+#define v1 fstk->vstk[fstk->vcnt]
+#define v2 fstk->vstk[fstk->vcnt + 1]
+#define v3 fstk->vstk[fstk->vcnt + 2]
 
 #define runtime(fmt, ...) do { \
   if (!(fs->flags & FF_SILENT)) \
@@ -222,12 +224,12 @@ interpret(struct filter_state *fs, const struct f_line *line, struct f_val *val)
     }
     
     /* End of current line. Drop local variables before exiting. */
-    vstk.cnt -= curline.line->vars;
-    vstk.cnt -= curline.line->args;
-    estk.cnt--;
+    fstk->vcnt -= curline.line->vars;
+    fstk->vcnt -= curline.line->args;
+    fstk->ecnt--;
   }
 
-  if (vstk.cnt == 0) {
+  if (fstk->vcnt == 0) {
     if (val) {
       log_rl(&rl_runtime_err, L_ERR "filters: No value left on stack");
       return F_ERROR;
@@ -235,12 +237,12 @@ interpret(struct filter_state *fs, const struct f_line *line, struct f_val *val)
     return F_NOP;
   }
 
-  if (val && (vstk.cnt == 1)) {
-    *val = vstk.val[0];
+  if (val && (fstk->vcnt == 1)) {
+    *val = fstk->vstk[0];
     return F_NOP;
   }
 
-  log_rl(&rl_runtime_err, L_ERR "Too many items left on stack: %u", vstk.cnt);
+  log_rl(&rl_runtime_err, L_ERR "Too many items left on stack: %u", fstk->vcnt);
   return F_ERROR;
 }