Route sources have an explicit owner

This commit prevents use-after-free of routes belonging to protocols
which have been already destroyed, delaying also all the protocols'
shutdown until all of their routes have been finally propagated through
all the pipes down to the appropriate exports.

The use-after-free was somehow hypothetic yet theoretically possible in
rare conditions, when one BGP protocol authors a lot of routes and the
user deletes that protocol by reconfiguring in the same time as next hop
update is requested, causing rte_better() to be called on a
not-yet-pruned network prefix while the owner protocol has been already
freed.

In parallel execution environments, this would happen an inter-thread
use-after-free, causing possible heisenbugs or other nasty problems.

1 files changed, 102 insertions, 21 deletions

diff --git a/nest/rt-attr.c b/nest/rt-attr.c
index 77fd3c3b..f7e33d72 100644
--- a/nest/rt-attr.c
+++ b/nest/rt-attr.c
@@ -85,6 +85,8 @@ const char * rta_dest_names[RTD_MAX] = {
   [RTD_PROHIBIT]	= "prohibited",
 };
 
+DOMAIN(attrs) attrs_domain;
+
 pool *rta_pool;
 
 static slab *rta_slab_[4];
@@ -96,16 +98,14 @@ static struct idm src_ids;
 
 /* rte source hash */
 
-#define RSH_KEY(n)		n->proto, n->private_id
+#define RSH_KEY(n)		n->private_id
 #define RSH_NEXT(n)		n->next
-#define RSH_EQ(p1,n1,p2,n2)	p1 == p2 && n1 == n2
-#define RSH_FN(p,n)		p->hash_key ^ u32_hash(n)
+#define RSH_EQ(n1,n2)		n1 == n2
+#define RSH_FN(n)		u32_hash(n)
 
 #define RSH_REHASH		rte_src_rehash
 #define RSH_PARAMS		/2, *2, 1, 1, 8, 20
-#define RSH_INIT_ORDER		6
-
-static HASH(struct rte_src) src_hash;
+#define RSH_INIT_ORDER		2
 
 static void
 rte_src_init(void)
@@ -113,55 +113,134 @@ rte_src_init(void)
   rte_src_slab = sl_new(rta_pool, sizeof(struct rte_src));
 
   idm_init(&src_ids, rta_pool, SRC_ID_INIT_SIZE);
-
-  HASH_INIT(src_hash, rta_pool, RSH_INIT_ORDER);
 }
 
-
 HASH_DEFINE_REHASH_FN(RSH, struct rte_src)
 
-struct rte_src *
-rt_find_source(struct proto *p, u32 id)
+static struct rte_src *
+rt_find_source(struct rte_owner *p, u32 id)
 {
-  return HASH_FIND(src_hash, RSH, p, id);
+  return HASH_FIND(p->hash, RSH, id);
 }
 
 struct rte_src *
-rt_get_source(struct proto *p, u32 id)
+rt_get_source_o(struct rte_owner *p, u32 id)
 {
+  if (p->stop)
+    bug("Stopping route owner asked for another source.");
+
   struct rte_src *src = rt_find_source(p, id);
 
   if (src)
+  {
+    UNUSED u64 uc = atomic_fetch_add_explicit(&src->uc, 1, memory_order_acq_rel);
     return src;
+  }
 
+  RTA_LOCK;
   src = sl_allocz(rte_src_slab);
-  src->proto = p;
+  src->owner = p;
   src->private_id = id;
   src->global_id = idm_alloc(&src_ids);
-  src->uc = 0;
 
-  HASH_INSERT2(src_hash, RSH, rta_pool, src);
+  atomic_store_explicit(&src->uc, 1, memory_order_release);
+  p->uc++;
+
+  HASH_INSERT2(p->hash, RSH, rta_pool, src);
+  if (config->table_debug)
+    log(L_TRACE "Allocated new rte_src for %s, ID %uL %uG, have %u sources now",
+	p->name, src->private_id, src->global_id, p->uc);
+
+  RTA_UNLOCK;
 
   return src;
 }
 
+static inline void
+rt_done_sources(struct rte_owner *o)
+{
+  if (o->stop->list)
+    ev_send(o->stop->list, o->stop);
+  else
+    ev_send(o->list, o->stop);
+}
+
 void
-rt_prune_sources(void)
+rt_prune_sources(void *data)
 {
-  HASH_WALK_FILTER(src_hash, next, src, sp)
+  struct rte_owner *o = data;
+
+  HASH_WALK_FILTER(o->hash, next, src, sp)
   {
-    if (src->uc == 0)
+    u64 uc;
+    while ((uc = atomic_load_explicit(&src->uc, memory_order_acquire)) >> RTE_SRC_PU_SHIFT)
+      ;
+
+    if (uc == 0)
     {
-      HASH_DO_REMOVE(src_hash, RSH, sp);
+      o->uc--;
+
+      HASH_DO_REMOVE(o->hash, RSH, sp);
+
+      RTA_LOCK;
       idm_free(&src_ids, src->global_id);
       sl_free(rte_src_slab, src);
+      RTA_UNLOCK;
     }
   }
   HASH_WALK_FILTER_END;
 
-  HASH_MAY_RESIZE_DOWN(src_hash, RSH, rta_pool);
+  RTA_LOCK;
+  HASH_MAY_RESIZE_DOWN(o->hash, RSH, rta_pool);
+
+  if (o->stop && !o->uc)
+  {
+    rfree(o->prune);
+    RTA_UNLOCK;
+
+    if (config->table_debug)
+      log(L_TRACE "All rte_src's for %s pruned, scheduling stop event", o->name);
+
+    rt_done_sources(o);
+  }
+  else
+    RTA_UNLOCK;
 }
 
+void
+rt_init_sources(struct rte_owner *o, const char *name, event_list *list)
+{
+  RTA_LOCK;
+  HASH_INIT(o->hash, rta_pool, RSH_INIT_ORDER);
+  o->hash_key = random_u32();
+  o->uc = 0;
+  o->name = name;
+  o->prune = ev_new_init(rta_pool, rt_prune_sources, o);
+  o->stop = NULL;
+  o->list = list;
+  RTA_UNLOCK;
+}
+
+void
+rt_destroy_sources(struct rte_owner *o, event *done)
+{
+  o->stop = done;
+
+  if (!o->uc)
+  {
+    if (config->table_debug)
+      log(L_TRACE "Source owner %s destroy requested. All rte_src's already pruned, scheduling stop event", o->name);
+
+    RTA_LOCK;
+    rfree(o->prune);
+    RTA_UNLOCK;
+
+    rt_done_sources(o);
+  }
+  else
+    if (config->table_debug)
+      log(L_TRACE "Source owner %s destroy requested. Remaining %u rte_src's to prune.", o->name, o->uc);
+}
 
 /*
  *	Multipath Next Hop
@@ -1328,6 +1407,8 @@ rta_show(struct cli *c, rta *a)
 void
 rta_init(void)
 {
+  attrs_domain = DOMAIN_NEW(attrs, "Attributes");
+
   rta_pool = rp_new(&root_pool, "Attributes");
 
   rta_slab_[0] = sl_new(rta_pool, sizeof(rta));