Interface subsystem locking

1 files changed, 221 insertions, 74 deletions

diff --git a/nest/iface.c b/nest/iface.c
index c49ad95e..f1938664 100644
--- a/nest/iface.c
+++ b/nest/iface.c
@@ -31,19 +31,51 @@
 #include "nest/cli.h"
 #include "lib/resource.h"
 #include "lib/string.h"
+#include "lib/locking.h"
 #include "conf/conf.h"
 #include "sysdep/unix/krt.h"
 
+DOMAIN(attrs) iface_domain;
+
+#define IFACE_LOCK	LOCK_DOMAIN(attrs, iface_domain)
+#define IFACE_UNLOCK	UNLOCK_DOMAIN(attrs, iface_domain)
+#define IFACE_ASSERT_LOCKED	ASSERT_DIE(DOMAIN_IS_LOCKED(attrs, iface_domain))
 
 static TLIST_LIST(ifsub) iface_sub_list;
 static slab *iface_sub_slab;
 static pool *if_pool;
 
-list iface_list;
+list global_iface_list;
 struct iface default_vrf;
 
 static void if_recalc_preferred(struct iface *i);
 
+static void ifa_dump_locked(struct ifa *);
+static void if_dump_locked(struct iface *);
+
+struct iface *
+if_walk_first(void)
+{
+  IFACE_LOCK;
+  struct iface *i = HEAD(global_iface_list);
+  return NODE_VALID(i) ? i : NULL;
+}
+
+struct iface *
+if_walk_next(struct iface *i)
+{
+  IFACE_ASSERT_LOCKED;
+  i = NODE_NEXT(i);
+  return NODE_VALID(i) ? i : NULL;
+}
+
+void
+if_walk_done(void)
+{
+  IFACE_ASSERT_LOCKED;
+  IFACE_UNLOCK;
+}
+
 /**
  * ifa_dump - dump interface address
  * @a: interface address descriptor
@@ -53,6 +85,14 @@ static void if_recalc_preferred(struct iface *i);
 void
 ifa_dump(struct ifa *a)
 {
+  IFACE_LOCK;
+  ifa_dump_locked(a);
+  IFACE_UNLOCK;
+}
+
+static void
+ifa_dump_locked(struct ifa *a)
+{
   debug("\t%I, net %N bc %I -> %I%s%s%s%s\n", a->ip, &a->prefix, a->brd, a->opposite,
 	(a->flags & IA_PRIMARY) ? " PRIMARY" : "",
 	(a->flags & IA_SECONDARY) ? " SEC" : "",
@@ -70,6 +110,14 @@ ifa_dump(struct ifa *a)
 void
 if_dump(struct iface *i)
 {
+  IFACE_LOCK;
+  if_dump_locked(i);
+  IFACE_UNLOCK;
+}
+
+static void
+if_dump_locked(struct iface *i)
+{
   struct ifa *a;
 
   debug("IF%d: %s", i->index, i->name);
@@ -96,7 +144,7 @@ if_dump(struct iface *i)
   debug(" MTU=%d\n", i->mtu);
   WALK_LIST(a, i->addrs)
     {
-      ifa_dump(a);
+      ifa_dump_locked(a);
       ASSERT(!!(a->flags & IA_PRIMARY) ==
 	     ((a == i->addr4) || (a == i->addr6) || (a == i->llv6)));
     }
@@ -111,14 +159,60 @@ if_dump(struct iface *i)
 void
 if_dump_all(void)
 {
-  struct iface *i;
-
   debug("Known network interfaces:\n");
-  WALK_LIST(i, iface_list)
+  IFACE_WALK(i)
     if_dump(i);
   debug("Router ID: %08x\n", config->router_id);
 }
 
+void
+if_link(struct iface *i)
+{
+  IFACE_ASSERT_LOCKED;
+
+  if (i)
+    i->uc++;
+}
+
+void
+if_unlink(struct iface *i)
+{
+  IFACE_ASSERT_LOCKED;
+
+  if (i)
+    i->uc--;
+  /* TODO: Do some interface object cleanup */
+}
+
+void ifa_link(struct ifa *a)
+{
+  IFACE_ASSERT_LOCKED;
+
+  if (a)
+  {
+    debug("ifa_link: %p %d\n", a, a->uc);
+    a->uc++;
+  }
+}
+
+void ifa_unlink(struct ifa *a)
+{
+  IFACE_ASSERT_LOCKED;
+
+  if (!a)
+    return;
+
+  debug("ifa_unlink: %p %d\n", a, a->uc);
+  if (--a->uc)
+    return;
+
+  if_unlink(a->iface);
+#if DEBUGGING
+  memset(a, 0x5b, sizeof(struct ifa));
+#endif
+  mb_free(a);
+}
+
 static inline unsigned
 if_what_changed(struct iface *i, struct iface *j)
 {
@@ -151,6 +245,8 @@ if_copy(struct iface *to, struct iface *from)
 void
 if_enqueue_notify_to(struct iface_notification x, struct iface_subscription *s)
 {
+  IFACE_ASSERT_LOCKED;
+
   switch (x.type) {
     case IFNOT_ADDRESS:
       if (!s->ifa_notify) return;
@@ -174,12 +270,14 @@ if_enqueue_notify_to(struct iface_notification x, struct iface_subscription *s)
   debug("Enqueue notify %d/%p (%p) to %p\n", x.type, x.a, in, s);
 
   ifnot_add_tail(&s->queue, in);
-  ev_schedule(&s->event);
+  ev_send(s->target, &s->event);
 }
 
 void
 if_enqueue_notify(struct iface_notification x)
 {
+  IFACE_ASSERT_LOCKED;
+
   WALK_TLIST(ifsub, s, &iface_sub_list)
     if_enqueue_notify_to(x, s);
 }
@@ -261,7 +359,7 @@ if_notify_change(unsigned c, struct iface *i)
 
   DBG("Interface change notification (%x) for %s\n", c, i->name);
 #ifdef LOCAL_DEBUG
-  if_dump(i);
+  if_dump_locked(i);
 #endif
 
   if (c & IF_CHANGE_DOWN)
@@ -323,10 +421,12 @@ if_change_flags(struct iface *i, uint flags)
 void
 if_delete(struct iface *old)
 {
+  IFACE_LOCK;
   struct iface f = {};
   strncpy(f.name, old->name, sizeof(f.name)-1);
   f.flags = IF_SHUTDOWN;
-  if_update(&f);
+  if_update_locked(&f);
+  IFACE_UNLOCK;
 }
 
 /**
@@ -348,13 +448,22 @@ if_delete(struct iface *old)
 struct iface *
 if_update(struct iface *new)
 {
+  IFACE_LOCK;
+  struct iface *i = if_update_locked(new);
+  IFACE_UNLOCK;
+  return i;
+}
+
+struct iface *
+if_update_locked(struct iface *new)
+{
   struct iface *i;
   unsigned c;
 
   if (!new->master)
     new->master = &default_vrf;
 
-  WALK_LIST(i, iface_list)
+  WALK_LIST(i, global_iface_list)
     if (!strcmp(new->name, i->name))
       {
 	new->flags = if_recalc_flags(new, new->flags);
@@ -389,17 +498,16 @@ if_update(struct iface *new)
   init_list(&i->neighbors);
 newif:
   i->flags |= IF_UPDATED | IF_TMP_DOWN;		/* Tmp down as we don't have addresses yet */
-  add_tail(&iface_list, &i->n);
+  add_tail(&global_iface_list, &i->n);
   return i;
 }
 
 void
 if_start_update(void)
 {
-  struct iface *i;
   struct ifa *a;
 
-  WALK_LIST(i, iface_list)
+  IFACE_WALK(i)
     {
       i->flags &= ~IF_UPDATED;
       WALK_LIST(a, i->addrs)
@@ -407,8 +515,8 @@ if_start_update(void)
     }
 }
 
-void
-if_end_partial_update(struct iface *i)
+static void
+if_end_partial_update_locked(struct iface *i)
 {
   if (i->flags & IF_NEEDS_RECALC)
     if_recalc_preferred(i);
@@ -418,12 +526,19 @@ if_end_partial_update(struct iface *i)
 }
 
 void
+if_end_partial_update(struct iface *i)
+{
+  IFACE_LOCK;
+  if_end_partial_update_locked(i);
+  IFACE_UNLOCK;
+}
+
+void
 if_end_update(void)
 {
-  struct iface *i;
   struct ifa *a, *b;
 
-  WALK_LIST(i, iface_list)
+  IFACE_WALK(i)
     {
       if (!(i->flags & IF_UPDATED))
 	if_change_flags(i, (i->flags & ~IF_ADMIN_UP) | IF_SHUTDOWN);
@@ -432,55 +547,53 @@ if_end_update(void)
 	  WALK_LIST_DELSAFE(a, b, i->addrs)
 	    if (!(a->flags & IA_UPDATED))
 	      ifa_delete(a);
-	  if_end_partial_update(i);
+	  if_end_partial_update_locked(i);
 	}
     }
 }
 
-void
-if_link(struct iface *i)
-{
-  if (i)
-    i->uc++;
-}
-
-void
-if_unlink(struct iface *i)
-{
-  if (i)
-    i->uc--;
-  /* TODO: Do some interface object cleanup */
-}
-
 static void
 iface_notify_hook(void *_s)
 {
   struct iface_subscription *s = _s;
 
+  IFACE_LOCK;
+
   while (!EMPTY_TLIST(ifnot, &s->queue))
   {
     struct iface_notification *n = THEAD(ifnot, &s->queue);
     debug("Process notify %d/%p (%p) to %p\n", n->type, n->a, n, s);
+    IFACE_UNLOCK;
+
     switch (n->type) {
       case IFNOT_ADDRESS:
 	ifa_send_notify(s, n->flags, n->a);
+	IFACE_LOCK;
 	ifa_unlink(n->a);
+	IFACE_UNLOCK;
 	break;
       case IFNOT_INTERFACE:
 	if_send_notify(s, n->flags, n->i);
+	IFACE_LOCK;
 	if_unlink(n->i);
+	IFACE_UNLOCK;
 	break;
       case IFNOT_NEIGHBOR:
 	s->neigh_notify(n->n);
+	IFACE_LOCK;
 	neigh_unlink(n->n);
+	IFACE_UNLOCK;
 	break;
       default:
 	bug("Bad interface notification type: %d", n->type);
     }
 
+    IFACE_LOCK;
     ifnot_rem_node(&s->queue, n);
     sl_free(n);
   }
+
+  IFACE_UNLOCK;
 }
 
 
@@ -494,6 +607,7 @@ iface_notify_hook(void *_s)
 void
 iface_subscribe(struct iface_subscription *s)
 {
+  IFACE_LOCK;
   ifsub_add_tail(&iface_sub_list, s);
   s->event = (event) {
     .hook = iface_notify_hook,
@@ -501,19 +615,34 @@ iface_subscribe(struct iface_subscription *s)
   };
 
   if (!s->if_notify && !s->ifa_notify)	/* shortcut */
+  {
+    IFACE_UNLOCK;
     return;
+  }
 
   struct iface *i;
   DBG("Announcing interfaces to new protocol %s\n", p->name);
-  WALK_LIST(i, iface_list)
+  WALK_LIST(i, global_iface_list)
     {
-      if_send_notify(s, IF_CHANGE_CREATE | ((i->flags & IF_UP) ? IF_CHANGE_UP : 0), i);
+      if_enqueue_notify_to(
+	  (struct iface_notification) {
+	  .type = IFNOT_INTERFACE,
+	  .i = i,
+	  .flags = IF_CHANGE_CREATE | ((i->flags & IF_UP) ? IF_CHANGE_UP : 0),
+	  }, s);
 
       struct ifa *a;
       if (i->flags & IF_UP)
 	WALK_LIST(a, i->addrs)
-	  ifa_send_notify(s, IF_CHANGE_CREATE | IF_CHANGE_UP, a);
+	  if_enqueue_notify_to(
+	      (struct iface_notification) {
+	      .type = IFNOT_ADDRESS,
+	      .a = a,
+	      .flags = IF_CHANGE_CREATE | IF_CHANGE_UP,
+	      }, s);
     }
+
+  IFACE_UNLOCK;
 }
 
 /**
@@ -523,6 +652,12 @@ iface_subscribe(struct iface_subscription *s)
 void
 iface_unsubscribe(struct iface_subscription *s)
 {
+  IFACE_LOCK;
+
+  struct proto *p = SKIP_BACK(struct proto, iface_sub, s);
+  WALK_TLIST_DELSAFE(proto_neigh, n, &p->neighbors)
+    neigh_unlink(n);
+
   ifsub_rem_node(&iface_sub_list, s);
   ev_postpone(&s->event);
 
@@ -547,6 +682,10 @@ iface_unsubscribe(struct iface_subscription *s)
     ifnot_rem_node(&s->queue, n);
     sl_free(n);
   }
+
+  ASSERT_DIE(EMPTY_TLIST(proto_neigh, &p->neighbors));
+
+  IFACE_UNLOCK;
 }
 
 /**
@@ -558,16 +697,26 @@ iface_unsubscribe(struct iface_subscription *s)
  * if no such structure exists.
  */
 struct iface *
-if_find_by_index(unsigned idx)
+if_find_by_index_locked(unsigned idx)
 {
   struct iface *i;
 
-  WALK_LIST(i, iface_list)
+  WALK_LIST(i, global_iface_list)
     if (i->index == idx && !(i->flags & IF_SHUTDOWN))
       return i;
+
   return NULL;
 }
 
+struct iface *
+if_find_by_index(unsigned idx)
+{
+  IFACE_LOCK;
+  struct iface *i = if_find_by_index_locked(idx);
+  IFACE_UNLOCK;
+  return i;
+}
+
 /**
  * if_find_by_name - find interface by name
  * @name: interface name
@@ -581,9 +730,15 @@ if_find_by_name(const char *name)
 {
   struct iface *i;
 
-  WALK_LIST(i, iface_list)
+  IFACE_LOCK;
+  WALK_LIST(i, global_iface_list)
     if (!strcmp(i->name, name) && !(i->flags & IF_SHUTDOWN))
+    {
+      IFACE_UNLOCK;
       return i;
+    }
+
+  IFACE_UNLOCK;
   return NULL;
 }
 
@@ -592,9 +747,13 @@ if_get_by_name(const char *name)
 {
   struct iface *i;
 
-  WALK_LIST(i, iface_list)
+  IFACE_LOCK;
+  WALK_LIST(i, global_iface_list)
     if (!strcmp(i->name, name))
+    {
+      IFACE_UNLOCK;
       return i;
+    }
 
   /* No active iface, create a dummy */
   i = mb_allocz(if_pool, sizeof(struct iface));
@@ -602,7 +761,9 @@ if_get_by_name(const char *name)
   i->flags = IF_SHUTDOWN;
   init_list(&i->addrs);
   init_list(&i->neighbors);
-  add_tail(&iface_list, &i->n);
+  add_tail(&global_iface_list, &i->n);
+
+  IFACE_UNLOCK;
   return i;
 }
 
@@ -686,9 +847,7 @@ if_recalc_preferred(struct iface *i)
 void
 if_recalc_all_preferred_addresses(void)
 {
-  struct iface *i;
-
-  WALK_LIST(i, iface_list)
+  IFACE_WALK(i)
   {
     if_recalc_preferred(i);
 
@@ -715,6 +874,8 @@ ifa_same(struct ifa *a, struct ifa *b)
 struct ifa *
 ifa_update(struct ifa *a)
 {
+  IFACE_LOCK;
+
   struct iface *i = a->iface;
   struct ifa *b;
 
@@ -727,6 +888,8 @@ ifa_update(struct ifa *a)
 	    !((b->flags ^ a->flags) & (IA_SECONDARY | IA_PEER | IA_HOST)))
 	  {
 	    b->flags |= IA_UPDATED;
+
+	    IFACE_UNLOCK;
 	    return b;
 	  }
 	ifa_delete(b);
@@ -746,6 +909,8 @@ ifa_update(struct ifa *a)
   i->flags |= IF_NEEDS_RECALC;
   if (i->flags & IF_UP)
     ifa_notify_change(IF_CHANGE_CREATE | IF_CHANGE_UP, b);
+
+  IFACE_UNLOCK;
   return b;
 }
 
@@ -763,6 +928,8 @@ ifa_delete(struct ifa *a)
   struct iface *i = a->iface;
   struct ifa *b;
 
+  IFACE_LOCK;
+
   WALK_LIST(b, i->addrs)
     if (ifa_same(b, a))
       {
@@ -787,43 +954,23 @@ ifa_delete(struct ifa *a)
 	  ifa_notify_change(IF_CHANGE_DOWN, b);
 
 	ifa_unlink(b);
+	IFACE_UNLOCK;
 	return;
       }
-}
-
-void ifa_link(struct ifa *a)
-{
-  if (a)
-  {
-    debug("ifa_link: %p %d\n", a, a->uc);
-    a->uc++;
-  }
-}
-
-void ifa_unlink(struct ifa *a)
-{
-  if (!a)
-    return;
-
-  debug("ifa_unlink: %p %d\n", a, a->uc);
-  if (--a->uc)
-    return;
 
-  if_unlink(a->iface);
-#if DEBUGGING
-  memset(a, 0x5b, sizeof(struct ifa));
-#endif
-  mb_free(a);
+  IFACE_UNLOCK;
 }
 
 u32
 if_choose_router_id(struct iface_patt *mask, u32 old_id)
 {
+  IFACE_LOCK;
+
   struct iface *i;
   struct ifa *a, *b;
 
   b = NULL;
-  WALK_LIST(i, iface_list)
+  WALK_LIST(i, global_iface_list)
     {
       if (!(i->flags & IF_ADMIN_UP) ||
 	  (i->flags & IF_SHUTDOWN))
@@ -850,6 +997,8 @@ if_choose_router_id(struct iface_patt *mask, u32 old_id)
 	}
     }
 
+  IFACE_UNLOCK;
+
   if (!b)
     return 0;
 
@@ -870,10 +1019,11 @@ void
 if_init(void)
 {
   if_pool = rp_new(&root_pool, "Interfaces");
-  init_list(&iface_list);
+  init_list(&global_iface_list);
   iface_sub_slab = sl_new(if_pool, sizeof(struct iface_notification));
   strcpy(default_vrf.name, "default");
   neigh_init(if_pool);
+  iface_domain = DOMAIN_NEW(attrs, "Interfaces");
 }
 
 /*
@@ -995,11 +1145,10 @@ if_show_addr(struct ifa *a)
 void
 if_show(void)
 {
-  struct iface *i;
   struct ifa *a;
   char *type;
 
-  WALK_LIST(i, iface_list)
+  IFACE_WALK(i)
     {
       if (i->flags & IF_SHUTDOWN)
 	continue;
@@ -1039,10 +1188,8 @@ if_show(void)
 void
 if_show_summary(void)
 {
-  struct iface *i;
-
   cli_msg(-2005, "%-10s %-6s %-18s %s", "Interface", "State", "IPv4 address", "IPv6 address");
-  WALK_LIST(i, iface_list)
+  IFACE_WALK(i)
     {
       byte a4[IPA_MAX_TEXT_LENGTH + 17];
       byte a6[IPA_MAX_TEXT_LENGTH + 17];