Filter: Optimize IPv4 prefix sets

Use separate IPv4 and IPv6 implementation of prefix sets. Just this
change makes IPv4 prefix sets 60% smaller and 50% faster.

4 files changed, 243 insertions, 88 deletions

diff --git a/filter/config.Y b/filter/config.Y
index c8b868af..995f6cd4 100644
--- a/filter/config.Y
+++ b/filter/config.Y
@@ -775,8 +775,8 @@ fprefix:
  ;
 
 fprefix_set:
-   fprefix { $$ = f_new_trie(cfg_mem, sizeof(struct f_trie_node)); trie_add_prefix($$, &($1.net), $1.lo, $1.hi); }
- | fprefix_set ',' fprefix { $$ = $1; trie_add_prefix($$, &($3.net), $3.lo, $3.hi); }
+   fprefix { $$ = f_new_trie(cfg_mem, 0); trie_add_prefix($$, &($1.net), $1.lo, $1.hi); }
+ | fprefix_set ',' fprefix { $$ = $1; if (!trie_add_prefix($$, &($3.net), $3.lo, $3.hi)) cf_error("Mixed IPv4/IPv6 prefixes in prefix set"); }
  ;
 
 switch_body: /* EMPTY */ { $$ = NULL; }
diff --git a/filter/data.h b/filter/data.h
index 578ff98f..dd9830df 100644
--- a/filter/data.h
+++ b/filter/data.h
@@ -138,19 +138,35 @@ struct f_tree {
   void *data;
 };
 
-struct f_trie_node
+struct f_trie_node4
+{
+  ip4_addr addr, mask, accept;
+  uint plen;
+  struct f_trie_node4 *c[2];
+};
+
+struct f_trie_node6
 {
-  ip_addr addr, mask, accept;
+  ip6_addr addr, mask, accept;
   uint plen;
-  struct f_trie_node *c[2];
+  struct f_trie_node6 *c[2];
+};
+
+struct f_trie_node
+{
+  union {
+    struct f_trie_node4 v4;
+    struct f_trie_node6 v6;
+  };
 };
 
 struct f_trie
 {
   linpool *lp;
-  int zero;
-  uint node_size;
-  struct f_trie_node root[0];		/* Root trie node follows */
+  u8 zero;
+  s8 ipv4;				/* -1 for undefined / empty */
+  u16 data_size;			/* Additional data for each trie node */
+  struct f_trie_node root;		/* Root trie node */
 };
 
 struct f_tree *f_new_tree(void);
@@ -159,7 +175,7 @@ const struct f_tree *find_tree(const struct f_tree *t, const struct f_val *val);
 int same_tree(const struct f_tree *t0, const struct f_tree *t2);
 void tree_format(const struct f_tree *t, buffer *buf);
 
-struct f_trie *f_new_trie(linpool *lp, uint node_size);
+struct f_trie *f_new_trie(linpool *lp, uint data_size);
 void *trie_add_prefix(struct f_trie *t, const net_addr *n, uint l, uint h);
 int trie_match_net(const struct f_trie *t, const net_addr *n);
 int trie_same(const struct f_trie *t1, const struct f_trie *t2);
diff --git a/filter/trie.c b/filter/trie.c
index 3038f5ec..a0deeaa1 100644
--- a/filter/trie.c
+++ b/filter/trie.c
@@ -77,9 +77,10 @@
 
 
 /*
- * In the trie code, the prefix length is internally treated as for the whole
- * ip_addr, regardless whether it contains an IPv4 or IPv6 address. Therefore,
- * remaining definitions make sense.
+ * In the trie_add_prefix(), we use ip_addr (assuming that it is the same as
+ * ip6_addr) to handle both IPv4 and IPv6 prefixes. In contrast to rest of the
+ * BIRD, IPv4 addresses are just zero-padded from right. That is why we have
+ * ipt_from_ip4() and ipt_to_ip4() macros below.
  */
 
 #define ipa_mkmask(x) ip6_mkmask(x)
@@ -87,26 +88,41 @@
 #define ipa_pxlen(x,y) ip6_pxlen(x,y)
 #define ipa_getbit(x,n) ip6_getbit(x,n)
 
+#define ipt_from_ip4(x) _MI6(_I(x), 0, 0, 0)
+#define ipt_to_ip4(x) _MI4(_I0(x))
+
 
 /**
  * f_new_trie - allocates and returns a new empty trie
  * @lp: linear pool to allocate items from
- * @node_size: node size to be used (&f_trie_node and user data)
+ * @data_size: user data attached to node
  */
 struct f_trie *
-f_new_trie(linpool *lp, uint node_size)
+f_new_trie(linpool *lp, uint data_size)
 {
   struct f_trie * ret;
-  ret = lp_allocz(lp, sizeof(struct f_trie) + node_size);
+  ret = lp_allocz(lp, sizeof(struct f_trie) + data_size);
   ret->lp = lp;
-  ret->node_size = node_size;
+  ret->ipv4 = -1;
+  ret->data_size = data_size;
   return ret;
 }
 
-static inline struct f_trie_node *
-new_node(struct f_trie *t, int plen, ip_addr paddr, ip_addr pmask, ip_addr amask)
+static inline struct f_trie_node4 *
+new_node4(struct f_trie *t, int plen, ip4_addr paddr, ip4_addr pmask, ip4_addr amask)
+{
+  struct f_trie_node4 *n = lp_allocz(t->lp, sizeof(struct f_trie_node4) + t->data_size);
+  n->plen = plen;
+  n->addr = paddr;
+  n->mask = pmask;
+  n->accept = amask;
+  return n;
+}
+
+static inline struct f_trie_node6 *
+new_node6(struct f_trie *t, int plen, ip6_addr paddr, ip6_addr pmask, ip6_addr amask)
 {
-  struct f_trie_node *n = lp_allocz(t->lp, t->node_size);
+  struct f_trie_node6 *n = lp_allocz(t->lp, sizeof(struct f_trie_node6) + t->data_size);
   n->plen = plen;
   n->addr = paddr;
   n->mask = pmask;
@@ -114,12 +130,40 @@ new_node(struct f_trie *t, int plen, ip_addr paddr, ip_addr pmask, ip_addr amask
   return n;
 }
 
+static inline struct f_trie_node *
+new_node(struct f_trie *t, int plen, ip_addr paddr, ip_addr pmask, ip_addr amask)
+{
+  if (t->ipv4)
+    return (struct f_trie_node *) new_node4(t, plen, ipt_to_ip4(paddr), ipt_to_ip4(pmask), ipt_to_ip4(amask));
+  else
+    return (struct f_trie_node *) new_node6(t, plen, ipa_to_ip6(paddr), ipa_to_ip6(pmask), ipa_to_ip6(amask));
+}
+
 static inline void
-attach_node(struct f_trie_node *parent, struct f_trie_node *child)
+attach_node4(struct f_trie_node4 *parent, struct f_trie_node4 *child)
 {
-  parent->c[ipa_getbit(child->addr, parent->plen) ? 1 : 0] = child;
+  parent->c[ip4_getbit(child->addr, parent->plen) ? 1 : 0] = child;
 }
 
+static inline void
+attach_node6(struct f_trie_node6 *parent, struct f_trie_node6 *child)
+{
+  parent->c[ip6_getbit(child->addr, parent->plen) ? 1 : 0] = child;
+}
+
+static inline void
+attach_node(struct f_trie_node *parent, struct f_trie_node *child, int v4)
+{
+  if (v4)
+    attach_node4(&parent->v4, &child->v4);
+  else
+    attach_node6(&parent->v6, &child->v6);
+}
+
+#define GET_ADDR(N,F,X) ((X) ? ipt_from_ip4((N)->v4.F) : ipa_from_ip6((N)->v6.F))
+#define SET_ADDR(N,F,X,V) ({ if (X) (N)->v4.F =ipt_to_ip4(V); else (N)->v6.F =ipa_to_ip6(V); })
+
+#define GET_CHILD(N,F,X,I) ((X) ? (struct f_trie_node *) (N)->v4.c[I] : (struct f_trie_node *) (N)->v6.c[I])
 /**
  * trie_add_prefix
  * @t: trie to add to
@@ -133,21 +177,30 @@ attach_node(struct f_trie_node *parent, struct f_trie_node *child)
  *
  * Returns a pointer to the allocated node. The function can return a pointer to
  * an existing node if @px and @plen are the same. If px/plen == 0/0 (or ::/0),
- * a pointer to the root node is returned.
+ * a pointer to the root node is returned. Returns NULL when called with
+ * mismatched IPv4/IPv6 net type.
  */
 
 void *
 trie_add_prefix(struct f_trie *t, const net_addr *net, uint l, uint h)
 {
-  ip_addr px = net_prefix(net);
   uint plen = net_pxlen(net);
+  ip_addr px;
+  int v4;
 
-  if (net->type == NET_IP4)
+  switch (net->type)
   {
-    const uint delta = IP6_MAX_PREFIX_LENGTH - IP4_MAX_PREFIX_LENGTH;
-    plen += delta;
-    l += delta;
-    h += delta;
+  case NET_IP4: px = ipt_from_ip4(net4_prefix(net)); v4 = 1; break;
+  case NET_IP6: px = ipa_from_ip6(net6_prefix(net)); v4 = 0; break;
+  default: bug("invalid type");
+  }
+
+  if (t->ipv4 != v4)
+  {
+    if (t->ipv4 < 0)
+      t->ipv4 = v4;
+    else
+      return NULL;
   }
 
   if (l == 0)
@@ -162,95 +215,136 @@ trie_add_prefix(struct f_trie *t, const net_addr *net, uint l, uint h)
   ip_addr pmask = ipa_mkmask(plen);
   ip_addr paddr = ipa_and(px, pmask);
   struct f_trie_node *o = NULL;
-  struct f_trie_node *n = t->root;
+  struct f_trie_node *n = &t->root;
 
   while (n)
     {
-      ip_addr cmask = ipa_and(n->mask, pmask);
-
-      if (ipa_compare(ipa_and(paddr, cmask), ipa_and(n->addr, cmask)))
-	{
+      ip_addr naddr = GET_ADDR(n, addr, v4);
+      ip_addr nmask = GET_ADDR(n, mask, v4);
+      ip_addr accept = GET_ADDR(n, accept, v4);
+      ip_addr cmask = ipa_and(nmask, pmask);
+      uint nlen = v4 ? n->v4.plen : n->v6.plen;
+
+      if (ipa_compare(ipa_and(paddr, cmask), ipa_and(naddr, cmask)))
+        {
 	  /* We are out of path - we have to add branching node 'b'
 	     between node 'o' and node 'n', and attach new node 'a'
 	     as the other child of 'b'. */
-	  int blen = ipa_pxlen(paddr, n->addr);
+	  int blen = ipa_pxlen(paddr, naddr);
 	  ip_addr bmask = ipa_mkmask(blen);
 	  ip_addr baddr = ipa_and(px, bmask);
 
 	  /* Merge accept masks from children to get accept mask for node 'b' */
-	  ip_addr baccm = ipa_and(ipa_or(amask, n->accept), bmask);
+	  ip_addr baccm = ipa_and(ipa_or(amask, accept), bmask);
 
 	  struct f_trie_node *a = new_node(t, plen, paddr, pmask, amask);
 	  struct f_trie_node *b = new_node(t, blen, baddr, bmask, baccm);
-	  attach_node(o, b);
-	  attach_node(b, n);
-	  attach_node(b, a);
+	  attach_node(o, b, v4);
+	  attach_node(b, n, v4);
+	  attach_node(b, a, v4);
 	  return a;
 	}
 
-      if (plen < n->plen)
+      if (plen < nlen)
 	{
 	  /* We add new node 'a' between node 'o' and node 'n' */
-	  amask = ipa_or(amask, ipa_and(n->accept, pmask));
+	  amask = ipa_or(amask, ipa_and(accept, pmask));
 	  struct f_trie_node *a = new_node(t, plen, paddr, pmask, amask);
-	  attach_node(o, a);
-	  attach_node(a, n);
+	  attach_node(o, a, v4);
+	  attach_node(a, n, v4);
 	  return a;
 	}
 
-      if (plen == n->plen)
+      if (plen == nlen)
 	{
 	  /* We already found added node in trie. Just update accept mask */
-	  n->accept = ipa_or(n->accept, amask);
+	  accept = ipa_or(accept, amask);
+	  SET_ADDR(n, accept, v4, accept);
 	  return n;
 	}
 
       /* Update accept mask part M2 and go deeper */
-      n->accept = ipa_or(n->accept, ipa_and(amask, n->mask));
+      accept = ipa_or(accept, ipa_and(amask, nmask));
+      SET_ADDR(n, accept, v4, accept);
 
       /* n->plen < plen and plen <= 32 (128) */
       o = n;
-      n = n->c[ipa_getbit(paddr, n->plen) ? 1 : 0];
+      n = GET_CHILD(n, c, v4, ipa_getbit(paddr, nlen) ? 1 : 0);
     }
 
   /* We add new tail node 'a' after node 'o' */
   struct f_trie_node *a = new_node(t, plen, paddr, pmask, amask);
-  attach_node(o, a);
+  attach_node(o, a, v4);
 
   return a;
 }
 
 static int
-trie_match_prefix(const struct f_trie *t, ip_addr px, uint plen)
+trie_match_net4(const struct f_trie *t, ip4_addr px, uint plen)
 {
-  ip_addr pmask = ipa_mkmask(plen);
-  ip_addr paddr = ipa_and(px, pmask);
+  ip4_addr pmask = ip4_mkmask(plen);
+  ip4_addr paddr = ip4_and(px, pmask);
 
   if (plen == 0)
     return t->zero;
 
   int plentest = plen - 1;
-  const struct f_trie_node *n = t->root;
+  const struct f_trie_node4 *n = &t->root.v4;
 
-  while(n)
-    {
-      ip_addr cmask = ipa_and(n->mask, pmask);
+  while (n)
+  {
+    ip4_addr cmask = ip4_and(n->mask, pmask);
 
-      /* We are out of path */
-      if (ipa_compare(ipa_and(paddr, cmask), ipa_and(n->addr, cmask)))
-	return 0;
+    /* We are out of path */
+    if (ip4_compare(ip4_and(paddr, cmask), ip4_and(n->addr, cmask)))
+      return 0;
 
-      /* Check accept mask */
-      if (ipa_getbit(n->accept, plentest))
-	return 1;
+    /* Check accept mask */
+    if (ip4_getbit(n->accept, plentest))
+      return 1;
 
-      /* We finished trie walk and still no match */
-      if (plen <= n->plen)
-	return 0;
+    /* We finished trie walk and still no match */
+    if (plen <= n->plen)
+      return 0;
 
-      /* Choose children */
-      n =  n->c[(ipa_getbit(paddr, n->plen)) ? 1 : 0];
-    }
+    /* Choose children */
+    n =  n->c[(ip4_getbit(paddr, n->plen)) ? 1 : 0];
+  }
+
+  return 0;
+}
+
+static int
+trie_match_net6(const struct f_trie *t, ip6_addr px, uint plen)
+{
+  ip6_addr pmask = ip6_mkmask(plen);
+  ip6_addr paddr = ip6_and(px, pmask);
+
+  if (plen == 0)
+    return t->zero;
+
+  int plentest = plen - 1;
+  const struct f_trie_node6 *n = &t->root.v6;
+
+  while (n)
+  {
+    ip6_addr cmask = ip6_and(n->mask, pmask);
+
+    /* We are out of path */
+    if (ip6_compare(ip6_and(paddr, cmask), ip6_and(n->addr, cmask)))
+      return 0;
+
+    /* Check accept mask */
+    if (ip6_getbit(n->accept, plentest))
+      return 1;
+
+    /* We finished trie walk and still no match */
+    if (plen <= n->plen)
+      return 0;
+
+    /* Choose children */
+    n =  n->c[(ip6_getbit(paddr, n->plen)) ? 1 : 0];
+  }
 
   return 0;
 }
@@ -267,20 +361,42 @@ trie_match_prefix(const struct f_trie *t, ip_addr px, uint plen)
 int
 trie_match_net(const struct f_trie *t, const net_addr *n)
 {
-  uint add = 0;
+  switch (n->type)
+  {
+  case NET_IP4:
+  case NET_VPN4:
+  case NET_ROA4:
+    return t->ipv4 ? trie_match_net4(t, net4_prefix(n), net_pxlen(n)) : 0;
+
+  case NET_IP6:
+  case NET_VPN6:
+  case NET_ROA6:
+    return !t->ipv4 ? trie_match_net6(t, net6_prefix(n), net_pxlen(n)) : 0;
 
-  switch (n->type) {
-    case NET_IP4:
-    case NET_VPN4:
-    case NET_ROA4:
-      add = IP6_MAX_PREFIX_LENGTH - IP4_MAX_PREFIX_LENGTH;
+  default:
+    return 0;
   }
+}
 
-  return trie_match_prefix(t, net_prefix(n), net_pxlen(n) + add);
+static int
+trie_node_same4(const struct f_trie_node4 *t1, const struct f_trie_node4 *t2)
+{
+  if ((t1 == NULL) && (t2 == NULL))
+    return 1;
+
+  if ((t1 == NULL) || (t2 == NULL))
+    return 0;
+
+  if ((t1->plen != t2->plen) ||
+      (! ip4_equal(t1->addr, t2->addr)) ||
+      (! ip4_equal(t1->accept, t2->accept)))
+    return 0;
+
+  return trie_node_same4(t1->c[0], t2->c[0]) && trie_node_same4(t1->c[1], t2->c[1]);
 }
 
 static int
-trie_node_same(const struct f_trie_node *t1, const struct f_trie_node *t2)
+trie_node_same6(const struct f_trie_node6 *t1, const struct f_trie_node6 *t2)
 {
   if ((t1 == NULL) && (t2 == NULL))
     return 1;
@@ -289,11 +405,11 @@ trie_node_same(const struct f_trie_node *t1, const struct f_trie_node *t2)
     return 0;
 
   if ((t1->plen != t2->plen) ||
-      (! ipa_equal(t1->addr, t2->addr)) ||
-      (! ipa_equal(t1->accept, t2->accept)))
+      (! ip6_equal(t1->addr, t2->addr)) ||
+      (! ip6_equal(t1->accept, t2->accept)))
     return 0;
 
-  return trie_node_same(t1->c[0], t2->c[0]) && trie_node_same(t1->c[1], t2->c[1]);
+  return trie_node_same6(t1->c[0], t2->c[0]) && trie_node_same6(t1->c[1], t2->c[1]);
 }
 
 /**
@@ -306,20 +422,39 @@ trie_node_same(const struct f_trie_node *t1, const struct f_trie_node *t2)
 int
 trie_same(const struct f_trie *t1, const struct f_trie *t2)
 {
-  return (t1->zero == t2->zero) && trie_node_same(t1->root, t2->root);
+  if ((t1->zero != t2->zero) || (t1->ipv4 != t2->ipv4))
+    return 0;
+
+  if (t1->ipv4)
+    return trie_node_same4(&t1->root.v4, &t2->root.v4);
+  else
+    return trie_node_same6(&t1->root.v6, &t2->root.v6);
 }
 
 static void
-trie_node_format(const struct f_trie_node *t, buffer *buf)
+trie_node_format4(const struct f_trie_node4 *t, buffer *buf)
 {
   if (t == NULL)
     return;
 
-  if (ipa_nonzero(t->accept))
-    buffer_print(buf, "%I/%d{%I}, ", t->addr, t->plen, t->accept);
+  if (ip4_nonzero(t->accept))
+    buffer_print(buf, "%4I/%d{%4I}, ", t->addr, t->plen, t->accept);
 
-  trie_node_format(t->c[0], buf);
-  trie_node_format(t->c[1], buf);
+  trie_node_format4(t->c[0], buf);
+  trie_node_format4(t->c[1], buf);
+}
+
+static void
+trie_node_format6(const struct f_trie_node6 *t, buffer *buf)
+{
+  if (t == NULL)
+    return;
+
+  if (ip6_nonzero(t->accept))
+    buffer_print(buf, "%6I/%d{%6I}, ", t->addr, t->plen, t->accept);
+
+  trie_node_format6(t->c[0], buf);
+  trie_node_format6(t->c[1], buf);
 }
 
 /**
@@ -335,8 +470,12 @@ trie_format(const struct f_trie *t, buffer *buf)
   buffer_puts(buf, "[");
 
   if (t->zero)
-    buffer_print(buf, "%I/%d, ", IPA_NONE, 0);
-  trie_node_format(t->root, buf);
+    buffer_print(buf, "%I/%d, ", t->ipv4 ? IPA_NONE4 : IPA_NONE6, 0);
+
+  if (t->ipv4)
+    trie_node_format4(&t->root.v4, buf);
+  else
+    trie_node_format6(&t->root.v6, buf);
 
   if (buf->pos == buf->end)
     return;
diff --git a/filter/trie_test.c b/filter/trie_test.c
index 38c387b0..b2b36716 100644
--- a/filter/trie_test.c
+++ b/filter/trie_test.c
@@ -103,7 +103,7 @@ t_match_net(void)
   {
     list prefixes; /* of structs f_extended_prefix */
     init_list(&prefixes);
-    struct f_trie *trie = f_new_trie(config->mem, sizeof(struct f_trie_node));
+    struct f_trie *trie = f_new_trie(config->mem, 0);
 
     generate_random_ipv6_prefixes(&prefixes);
     struct f_prefix_node *n;
@@ -143,8 +143,8 @@ t_trie_same(void)
   int round;
   for (round = 0; round < TESTS_NUM*4; round++)
   {
-    struct f_trie * trie1 = f_new_trie(config->mem, sizeof(struct f_trie_node));
-    struct f_trie * trie2 = f_new_trie(config->mem, sizeof(struct f_trie_node));
+    struct f_trie * trie1 = f_new_trie(config->mem, 0);
+    struct f_trie * trie2 = f_new_trie(config->mem, 0);
 
     list prefixes; /* a list of f_extended_prefix structures */
     init_list(&prefixes);