Interface updates are asynchronous

Instead of propagating interface updates as they are loaded from kernel,
they are enqueued and all the notifications are called from a
protocol-specific event. This change allows to break the locking loop
between protocols and interfaces.

Anyway, this change is based on v2 branch to keep the changes between v2
and v3 smaller.

1 files changed, 179 insertions, 24 deletions

diff --git a/nest/iface.c b/nest/iface.c
index fc63dc75..740c1878 100644
--- a/nest/iface.c
+++ b/nest/iface.c
@@ -34,6 +34,9 @@
 #include "conf/conf.h"
 #include "sysdep/unix/krt.h"
 
+
+static TLIST_LIST(ifsub) iface_sub_list;
+static slab *iface_sub_slab;
 static pool *if_pool;
 
 list iface_list;
@@ -140,13 +143,51 @@ if_copy(struct iface *to, struct iface *from)
   to->flags = from->flags | (to->flags & IF_TMP_DOWN);
   to->mtu = from->mtu;
   to->master_index = from->master_index;
-  to->master = from->master;
+
+  if_unlink(to->master);
+  if_link(to->master = from->master);
+}
+
+void
+if_enqueue_notify_to(struct iface_notification x, struct iface_subscription *s)
+{
+  switch (x.type) {
+    case IFNOT_ADDRESS:
+      if (!s->ifa_notify) return;
+      ifa_link(x.a);
+      break;
+    case IFNOT_INTERFACE:
+      if (!s->if_notify) return;
+      if_link(x.i);
+      break;
+    case IFNOT_NEIGHBOR:
+      if (!s->neigh_notify) return;
+      neigh_link(x.n);
+      break;
+    default:
+      bug("Unknown interface notification type: %d", x.type);
+  }
+
+  struct iface_notification *in = sl_alloc(iface_sub_slab);
+  *in = x;
+
+  ifnot_add_tail(&s->queue, in);
+  ev_schedule(&s->event);
+}
+
+void
+if_enqueue_notify(struct iface_notification x)
+{
+  WALK_TLIST(ifsub, s, &iface_sub_list)
+    if_enqueue_notify_to(x, s);
 }
 
 static inline void
-ifa_send_notify(struct proto *p, unsigned c, struct ifa *a)
+ifa_send_notify(struct iface_subscription *s, unsigned c, struct ifa *a)
 {
-  if (p->ifa_notify &&
+  struct proto *p = SKIP_BACK(struct proto, iface_sub, s);
+
+  if (s->ifa_notify &&
       (p->proto_state != PS_DOWN) &&
       (!p->vrf || p->vrf == a->iface->master))
     {
@@ -154,19 +195,21 @@ ifa_send_notify(struct proto *p, unsigned c, struct ifa *a)
 	log(L_TRACE "%s < address %N on interface %s %s",
 	    p->name, &a->prefix, a->iface->name,
 	    (c & IF_CHANGE_UP) ? "added" : "removed");
-      p->ifa_notify(p, c, a);
+      s->ifa_notify(p, c, a);
     }
 }
 
 static void
 ifa_notify_change_(unsigned c, struct ifa *a)
 {
-  struct proto *p;
-
   DBG("IFA change notification (%x) for %s:%I\n", c, a->iface->name, a->ip);
 
-  WALK_LIST(p, proto_list)
-    ifa_send_notify(p, c, a);
+  if_enqueue_notify((struct iface_notification) {
+	.type = IFNOT_ADDRESS,
+	.a = a,
+	.flags = c,
+      });
+
 }
 
 static inline void
@@ -182,9 +225,11 @@ ifa_notify_change(unsigned c, struct ifa *a)
 }
 
 static inline void
-if_send_notify(struct proto *p, unsigned c, struct iface *i)
+if_send_notify(struct iface_subscription *s, unsigned c, struct iface *i)
 {
-  if (p->if_notify &&
+  struct proto *p = SKIP_BACK(struct proto, iface_sub, s);
+
+  if (s->if_notify &&
       (p->proto_state != PS_DOWN) &&
       (!p->vrf || p->vrf == i->master))
     {
@@ -197,14 +242,13 @@ if_send_notify(struct proto *p, unsigned c, struct iface *i)
 	    (c & IF_CHANGE_PREFERRED) ? "changes preferred address" :
 	    (c & IF_CHANGE_CREATE) ? "created" :
 	    "sends unknown event");
-      p->if_notify(p, c, i);
+      s->if_notify(p, c, i);
     }
 }
 
 static void
 if_notify_change(unsigned c, struct iface *i)
 {
-  struct proto *p;
   struct ifa *a;
 
   if (i->flags & IF_JUST_CREATED)
@@ -225,8 +269,11 @@ if_notify_change(unsigned c, struct iface *i)
     WALK_LIST(a, i->addrs)
       ifa_notify_change_(IF_CHANGE_DOWN, a);
 
-  WALK_LIST(p, proto_list)
-    if_send_notify(p, c, i);
+  if_enqueue_notify((struct iface_notification) {
+	.type = IFNOT_INTERFACE,
+	.i = i,
+	.flags = c,
+      });
 
   if (c & IF_CHANGE_UP)
     WALK_LIST(a, i->addrs)
@@ -320,6 +367,7 @@ if_update(struct iface *new)
 	    new->llv6 = i->llv6;
 	    new->sysdep = i->sysdep;
 	    memcpy(&new->addrs, &i->addrs, sizeof(i->addrs));
+	    memcpy(&new->neighbors, &i->neighbors, sizeof(i->neighbors));
 	    memcpy(i, new, sizeof(*i));
 	    i->flags &= ~IF_UP;		/* IF_TMP_DOWN will be added later */
 	    goto newif;
@@ -334,9 +382,10 @@ if_update(struct iface *new)
       }
   i = mb_alloc(if_pool, sizeof(struct iface));
   memcpy(i, new, sizeof(*i));
+  if_link(i->master);
   init_list(&i->addrs);
-newif:
   init_list(&i->neighbors);
+newif:
   i->flags |= IF_UPDATED | IF_TMP_DOWN;		/* Tmp down as we don't have addresses yet */
   add_tail(&iface_list, &i->n);
   return i;
@@ -386,32 +435,117 @@ if_end_update(void)
     }
 }
 
+void
+if_link(struct iface *i)
+{
+  if (i)
+    i->uc++;
+}
+
+void
+if_unlink(struct iface *i)
+{
+  if (i)
+    i->uc--;
+  /* TODO: Do some interface object cleanup */
+}
+
+static void
+iface_notify_hook(void *_s)
+{
+  struct iface_subscription *s = _s;
+
+  while (!EMPTY_TLIST(ifnot, &s->queue))
+  {
+    struct iface_notification *n = THEAD(ifnot, &s->queue);
+    switch (n->type) {
+      case IFNOT_ADDRESS:
+	ifa_send_notify(s, n->flags, n->a);
+	ifa_unlink(n->a);
+	break;
+      case IFNOT_INTERFACE:
+	if_send_notify(s, n->flags, n->i);
+	if_unlink(n->i);
+	break;
+      case IFNOT_NEIGHBOR:
+	s->neigh_notify(n->n);
+	neigh_unlink(n->n);
+	break;
+      default:
+	bug("Bad interface notification type: %d", n->type);
+    }
+
+    ifnot_rem_node(&s->queue, n);
+    sl_free(n);
+  }
+}
+
+
 /**
- * if_feed_baby - advertise interfaces to a new protocol
- * @p: protocol to feed
+ * iface_subscribe - request interface updates
+ * @s: subscription structure
  *
  * When a new protocol starts, this function sends it a series
  * of notifications about all existing interfaces.
  */
 void
-if_feed_baby(struct proto *p)
+iface_subscribe(struct iface_subscription *s)
 {
-  struct iface *i;
-  struct ifa *a;
+  ifsub_add_tail(&iface_sub_list, s);
+  s->event = (event) {
+    .hook = iface_notify_hook,
+    .data = s,
+  };
 
-  if (!p->if_notify && !p->ifa_notify)	/* shortcut */
+  if (!s->if_notify && !s->ifa_notify)	/* shortcut */
     return;
+
+  struct iface *i;
   DBG("Announcing interfaces to new protocol %s\n", p->name);
   WALK_LIST(i, iface_list)
     {
-      if_send_notify(p, IF_CHANGE_CREATE | ((i->flags & IF_UP) ? IF_CHANGE_UP : 0), i);
+      if_send_notify(s, IF_CHANGE_CREATE | ((i->flags & IF_UP) ? IF_CHANGE_UP : 0), i);
+
+      struct ifa *a;
       if (i->flags & IF_UP)
 	WALK_LIST(a, i->addrs)
-	  ifa_send_notify(p, IF_CHANGE_CREATE | IF_CHANGE_UP, a);
+	  ifa_send_notify(s, IF_CHANGE_CREATE | IF_CHANGE_UP, a);
     }
 }
 
 /**
+ * iface_unsubscribe - unsubscribe from interface updates
+ * @s: subscription structure
+ */
+void
+iface_unsubscribe(struct iface_subscription *s)
+{
+  ifsub_rem_node(&iface_sub_list, s);
+  ev_postpone(&s->event);
+
+  WALK_TLIST_DELSAFE(ifnot, n, &s->queue)
+  {
+    switch (n->type)
+    {
+      case IFNOT_ADDRESS:
+	ifa_unlink(n->a);
+	break;
+      case IFNOT_INTERFACE:
+	if_unlink(n->i);
+	break;
+      case IFNOT_NEIGHBOR:
+	neigh_unlink(n->n);
+	break;
+      default:
+	bug("Bad interface notification type: %d", n->type);
+    }
+
+    ifnot_rem_node(&s->queue, n);
+    sl_free(n);
+  }
+}
+
+/**
  * if_find_by_index - find interface by ifindex
  * @idx: ifindex
  *
@@ -600,6 +734,8 @@ ifa_update(struct ifa *a)
 
   b = mb_alloc(if_pool, sizeof(struct ifa));
   memcpy(b, a, sizeof(struct ifa));
+  ifa_link(b);
+  if_link(i);
   add_tail(&i->addrs, &b->n);
   b->flags |= IA_UPDATED;
 
@@ -646,11 +782,29 @@ ifa_delete(struct ifa *a)
 	if (i->flags & IF_UP)
 	  ifa_notify_change(IF_CHANGE_DOWN, b);
 
-	mb_free(b);
+	ifa_unlink(b);
 	return;
       }
 }
 
+void ifa_link(struct ifa *a)
+{
+  if (a)
+    a->uc++;
+}
+
+void ifa_unlink(struct ifa *a)
+{
+  if (!a)
+    return;
+
+  if (--a->uc)
+    return;
+
+  if_unlink(a->iface);
+  mb_free(a);
+}
+
 u32
 if_choose_router_id(struct iface_patt *mask, u32 old_id)
 {
@@ -706,6 +860,7 @@ if_init(void)
 {
   if_pool = rp_new(&root_pool, "Interfaces");
   init_list(&iface_list);
+  iface_sub_slab = sl_new(if_pool, sizeof(struct iface_notification));
   strcpy(default_vrf.name, "default");
   neigh_init(if_pool);
 }