Hostcache update notification converted to an export hook

Instead of synchronous notifications, we use the asynchronous export
framework to notify also hostcache updates. This allows us to do the
hostcache update and the subsequent next hop update notification without
locking collisions.

2 files changed, 112 insertions, 81 deletions

diff --git a/nest/rt-table.c b/nest/rt-table.c
index d4e0d63c..e23e766e 100644
--- a/nest/rt-table.c
+++ b/nest/rt-table.c
@@ -43,10 +43,10 @@
  * all prefixes that may influence resolving of tracked next hops.
  *
  * When a best route changes in the src table, the hostcache is notified using
- * rt_notify_hostcache(), which immediately checks using the trie whether the
+ * an auxiliary export request, which checks using the trie whether the
  * change is relevant and if it is, then it schedules asynchronous hostcache
  * recomputation. The recomputation is done by rt_update_hostcache() (called
- * from rt_event() of src table), it walks through all hostentries and resolves
+ * as an event of src table), it walks through all hostentries and resolves
  * them (by rt_update_hostentry()). It also updates the trie. If a change in
  * hostentry resolution was found, then it schedules asynchronous nexthop
  * recomputation of associated dst table. That is done by rt_next_hop_update()
@@ -130,8 +130,7 @@ struct rt_export_block {
 };
 
 static void rt_free_hostcache(rtable *tab);
-static void rt_notify_hostcache(rtable *tab, net *net);
-static void rt_update_hostcache(rtable *tab);
+static void rt_update_hostcache(void *tab);
 static void rt_next_hop_update(rtable *tab);
 static inline void rt_next_hop_resolve_rte(rte *r);
 static inline void rt_flowspec_resolve_rte(rte *r, struct channel *c);
@@ -1280,9 +1279,6 @@ rte_announce(rtable *tab, net *net, struct rte_storage *new, struct rte_storage
     if (old_best_valid)
       old_best->rte.sender->stats.pref--;
 
-    if (tab->hostcache)
-      rt_notify_hostcache(tab, net);
-
     if (!EMPTY_LIST(tab->flowspec_links))
       rt_flowspec_notify(tab, net);
   }
@@ -2238,8 +2234,8 @@ void
 rt_dump_hooks(rtable *tab)
 {
   debug("Dump of hooks in routing table <%s>%s\n", tab->name, tab->deleted ? " (deleted)" : "");
-  debug("  nhu_state=%u hcu_scheduled=%u use_count=%d rt_count=%u\n",
-      tab->nhu_state, tab->hcu_scheduled, tab->use_count, tab->rt_count);
+  debug("  nhu_state=%u use_count=%d rt_count=%u\n",
+      tab->nhu_state, tab->use_count, tab->rt_count);
   debug("  last_rt_change=%t gc_time=%t gc_counter=%d prune_state=%u\n",
       tab->last_rt_change, tab->gc_time, tab->gc_counter, tab->prune_state);
 
@@ -2280,16 +2276,6 @@ rt_dump_hooks_all(void)
 }
 
 static inline void
-rt_schedule_hcu(rtable *tab)
-{
-  if (tab->hcu_scheduled)
-    return;
-
-  tab->hcu_scheduled = 1;
-  ev_schedule(tab->rt_event);
-}
-
-static inline void
 rt_schedule_nhu(rtable *tab)
 {
   if (tab->nhu_state == NHU_CLEAN)
@@ -2336,25 +2322,15 @@ rt_event(void *ptr)
   if (tab->export_used)
     rt_export_cleanup(tab);
 
-  if (
-      tab->hcu_corked ||
-      tab->nhu_corked ||
-      (tab->hcu_scheduled || tab->nhu_state) && rt_cork_check(tab->uncork_event)
-      )
+  if (tab->nhu_corked || tab->nhu_state && rt_cork_check(tab->uncork_event))
   {
-    if (!tab->hcu_corked && !tab->nhu_corked)
+    if (!tab->nhu_corked)
       rt_trace(tab, D_STATES, "Next hop updater corked");
 
-    tab->hcu_corked |= tab->hcu_scheduled;
-    tab->hcu_scheduled = 0;
-
     tab->nhu_corked |= tab->nhu_state;
     tab->nhu_state = 0;
   }
 
-  if (tab->hcu_scheduled)
-    rt_update_hostcache(tab);
-
   if (tab->nhu_state)
     rt_next_hop_update(tab);
 
@@ -2369,9 +2345,6 @@ rt_uncork_event(void *ptr)
 {
   rtable *tab = ptr;
 
-  tab->hcu_scheduled |= tab->hcu_corked;
-  tab->hcu_corked = 0;
-
   tab->nhu_state |= tab->nhu_corked;
   tab->nhu_corked = 0;
 
@@ -3719,6 +3692,9 @@ rt_reconfigure(rtable *tab, struct rtable_config *new, struct rtable_config *old
   tab->name = new->name;
   tab->config = new;
 
+  if (tab->hostcache)
+    tab->hostcache->req.trace_routes = new->debug;
+
   tab->cork_threshold = new->cork_threshold;
 
   if (new->cork_threshold.high != old->cork_threshold.high)
@@ -3771,6 +3747,10 @@ rt_commit(struct config *new, struct config *old)
 	  tab->deleted = old;
 	  config_add_obstacle(old);
 	  rt_lock_table(tab);
+
+	  if (tab->hostcache)
+	    rt_stop_export(&tab->hostcache->req, NULL);
+
 	  rt_unlock_table(tab);
 	}
     }
@@ -4056,6 +4036,41 @@ hc_delete_hostentry(struct hostcache *hc, pool *p, struct hostentry *he)
 }
 
 static void
+hc_notify_dump_req(struct rt_export_request *req)
+{
+  debug("  Table %s (%p)\n", req->name, req);
+}
+
+static void
+hc_notify_export_one(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *first)
+{
+  struct hostcache *hc = SKIP_BACK(struct hostcache, req, req);
+
+  /* No interest in this update, mark seen only */
+  if (ev_active(&hc->update) || !trie_match_net(hc->trie, net))
+  {
+    rpe_mark_seen_all(req->hook, first, NULL);
+    return;
+  }
+
+  /* This net may affect some hostentries, check the actual change */
+  rte *o = RTE_VALID_OR_NULL(first->old_best);
+  struct rte_storage *new_best = first->new_best;
+
+  RPE_WALK(first, rpe, NULL)
+  {
+    rpe_mark_seen(req->hook, rpe);
+    new_best = rpe->new_best;
+  }
+
+  /* Yes, something has actually changed. Do the hostcache update.
+   * We don't need any more updates until then. */
+  if (o != RTE_VALID_OR_NULL(new_best))
+    ev_schedule_work(&hc->update);
+}
+
+
+static void
 rt_init_hostcache(rtable *tab)
 {
   struct hostcache *hc = mb_allocz(tab->rp, sizeof(struct hostcache));
@@ -4068,6 +4083,21 @@ rt_init_hostcache(rtable *tab)
   hc->lp = lp_new(tab->rp);
   hc->trie = f_new_trie(hc->lp, 0);
 
+  hc->update = (event) {
+    .hook = rt_update_hostcache,
+    .data = tab,
+  };
+
+  hc->req = (struct rt_export_request) {
+    .name = mb_sprintf(tab->rp, "%s.hcu.notifier", tab->name),
+    .list = &global_work_list,
+    .trace_routes = tab->config->debug,
+    .dump_req = hc_notify_dump_req,
+    .export_one = hc_notify_export_one,
+  };
+
+  rt_request_export(&tab->exporter, &hc->req);
+
   tab->hostcache = hc;
 }
 
@@ -4094,16 +4124,6 @@ rt_free_hostcache(rtable *tab)
   */
 }
 
-static void
-rt_notify_hostcache(rtable *tab, net *net)
-{
-  if (tab->hcu_scheduled)
-    return;
-
-  if (trie_match_net(tab->hostcache->trie, net->n.addr))
-    rt_schedule_hcu(tab);
-}
-
 static int
 if_local_addr(ip_addr a, struct iface *i)
 {
@@ -4200,9 +4220,17 @@ done:
 }
 
 static void
-rt_update_hostcache(rtable *tab)
+rt_update_hostcache(void *data)
 {
+  rtable *tab = data;
   struct hostcache *hc = tab->hostcache;
+
+  if (rt_cork_check(&hc->update))
+  {
+    rt_trace(tab, D_STATES, "Hostcache update corked");
+    return;
+  }
+
   struct hostentry *he;
   node *n, *x;
 
@@ -4222,8 +4250,6 @@ rt_update_hostcache(rtable *tab)
       if (rt_update_hostentry(tab, he))
 	rt_schedule_nhu(he->tab);
     }
-
-  tab->hcu_scheduled = 0;
 }
 
 static struct hostentry *
diff --git a/nest/rt.h b/nest/rt.h
index e7ba55f6..5839ba66 100644
--- a/nest/rt.h
+++ b/nest/rt.h
@@ -27,6 +27,7 @@ struct protocol;
 struct proto;
 struct channel;
 struct rte_src;
+struct hostcache;
 struct symbol;
 struct timer;
 struct filter;
@@ -116,8 +117,6 @@ typedef struct rtable {
   uint gc_counter;			/* Number of operations since last GC */
   byte prune_state;			/* Table prune state, 1 -> scheduled, 2-> running */
   byte prune_trie;			/* Prune prefix trie during next table prune */
-  byte hcu_scheduled;			/* Hostcache update is scheduled */
-  byte hcu_corked;			/* Hostcache update is corked with this state */
   byte nhu_state;			/* Next Hop Update state */
   byte nhu_corked;			/* Next Hop Update is corked with this state */
   byte export_used;			/* Pending Export pruning is scheduled */
@@ -185,43 +184,12 @@ static inline int rt_cork_check(event *e)
 }
 
 
-#define NHU_CLEAN	0
-#define NHU_SCHEDULED	1
-#define NHU_RUNNING	2
-#define NHU_DIRTY	3
-
 typedef struct network {
   struct rte_storage *routes;		/* Available routes for this network */
   struct rt_pending_export *first, *last;
   struct fib_node n;			/* FIB flags reserved for kernel syncer */
 } net;
 
-struct hostcache {
-  slab *slab;				/* Slab holding all hostentries */
-  struct hostentry **hash_table;	/* Hash table for hostentries */
-  unsigned hash_order, hash_shift;
-  unsigned hash_max, hash_min;
-  unsigned hash_items;
-  linpool *lp;				/* Linpool for trie */
-  struct f_trie *trie;			/* Trie of prefixes that might affect hostentries */
-  list hostentries;			/* List of all hostentries */
-  byte update_hostcache;
-};
-
-struct hostentry {
-  node ln;
-  ip_addr addr;				/* IP address of host, part of key */
-  ip_addr link;				/* (link-local) IP address of host, used as gw
-					   if host is directly attached */
-  struct rtable *tab;			/* Dependent table, part of key */
-  struct hostentry *next;		/* Next in hash chain */
-  unsigned hash_key;			/* Hash key */
-  unsigned uc;				/* Use count */
-  ea_list *src;				/* Source attributes */
-  byte nexthop_linkable;		/* Nexthop list is completely non-device */
-  u32 igp_metric;			/* Chosen route IGP metric */
-};
-
 struct rte_storage {
   struct rte_storage *next;		/* Next in chain */
   struct rte rte;			/* Route data */
@@ -394,7 +362,7 @@ struct rt_pending_export *rpe_next(struct rt_pending_export *rpe, struct rte_src
 void rpe_mark_seen(struct rt_export_hook *hook, struct rt_pending_export *rpe);
 
 #define rpe_mark_seen_all(hook, first, src) \
-  RPE_WALK(first, rpe, src) rpe_mark_seen((hook), rpe)
+  RPE_WALK((first), _rpe, (src)) rpe_mark_seen((hook), _rpe)
 
 /* Get pending export seen status */
 int rpe_get_seen(struct rt_export_hook *hook, struct rt_pending_export *rpe);
@@ -412,6 +380,43 @@ int rpe_get_seen(struct rt_export_hook *hook, struct rt_pending_export *rpe);
 #define RIC_REJECT	-1		/* Rejected by protocol */
 #define RIC_DROP	-2		/* Silently dropped by protocol */
 
+/*
+ * Next hop update data structures
+ */
+
+#define NHU_CLEAN	0
+#define NHU_SCHEDULED	1
+#define NHU_RUNNING	2
+#define NHU_DIRTY	3
+
+struct hostentry {
+  node ln;
+  ip_addr addr;				/* IP address of host, part of key */
+  ip_addr link;				/* (link-local) IP address of host, used as gw
+					   if host is directly attached */
+  struct rtable *tab;			/* Dependent table, part of key */
+  struct hostentry *next;		/* Next in hash chain */
+  unsigned hash_key;			/* Hash key */
+  unsigned uc;				/* Use count */
+  ea_list *src;				/* Source attributes */
+  byte nexthop_linkable;		/* Nexthop list is completely non-device */
+  u32 igp_metric;			/* Chosen route IGP metric */
+};
+
+struct hostcache {
+  slab *slab;				/* Slab holding all hostentries */
+  struct hostentry **hash_table;	/* Hash table for hostentries */
+  unsigned hash_order, hash_shift;
+  unsigned hash_max, hash_min;
+  unsigned hash_items;
+  linpool *lp;				/* Linpool for trie */
+  struct f_trie *trie;			/* Trie of prefixes that might affect hostentries */
+  list hostentries;			/* List of all hostentries */
+  event update;
+  struct rt_export_request req;		/* Notifier */
+};
+
+
 #define rte_update  channel_rte_import
 /**
  * rte_update - enter a new update to a routing table