BMP: Support multiple instances of BMP protocol

Add internal BMP functions with plicit bmp_proto *p as first argument,
which allows using TRACE() macro. Keep list of BMP instances and call
internal functions. Old BMP functions are wrappers that call internal
functions for all enabled BMP instances.

Extract End-of-RIB mark into separate function.

Based on patch from Michal Zagorski <mzagorsk@akamai.com>. Thanks!

5 files changed, 139 insertions, 75 deletions

diff --git a/proto/bgp/bgp.c b/proto/bgp/bgp.c
index 17c503c4..54da2253 100644
--- a/proto/bgp/bgp.c
+++ b/proto/bgp/bgp.c
@@ -1715,7 +1715,7 @@ bgp_init(struct proto_config *CF)
   P->rte_igp_metric = bgp_rte_igp_metric;
 
 #ifdef CONFIG_BMP
-  P->rte_update_in_notify = bgp_rte_update_in_notify;
+  P->rte_update_in_notify = bmp_route_monitor_update_in_notify;
 #endif
 
   p->cf = cf;
diff --git a/proto/bgp/bgp.h b/proto/bgp/bgp.h
index 5f8f183d..861f831a 100644
--- a/proto/bgp/bgp.h
+++ b/proto/bgp/bgp.h
@@ -631,11 +631,13 @@ struct rte *bgp_rte_modify_stale(struct rte *r, struct linpool *pool);
 u32 bgp_rte_igp_metric(struct rte *);
 void bgp_rt_notify(struct proto *P, struct channel *C, net *n, rte *new, rte *old);
 int bgp_preexport(struct channel *, struct rte *);
-void bgp_rte_update_in_notify(struct channel *C, const net_addr *n, const struct rte *new, const struct rte_src *src);
 int bgp_get_attr(const struct eattr *e, byte *buf, int buflen);
 void bgp_get_route_info(struct rte *, byte *buf);
 int bgp_total_aigp_metric_(rte *e, u64 *metric, const struct adata **ad);
 
+struct bmp_proto;
+void bgp_bmp_encode_rte(struct bgp_channel *c, struct bmp_proto *bmp, const net_addr *n, const struct rte *new, const struct rte_src *src);
+
 #define BGP_AIGP_METRIC		1
 #define BGP_AIGP_MAX		U64(0xffffffffffffffff)
 
@@ -664,8 +666,8 @@ const char * bgp_error_dsc(unsigned code, unsigned subcode);
 void bgp_log_error(struct bgp_proto *p, u8 class, char *msg, unsigned code, unsigned subcode, byte *data, unsigned len);
 
 void bgp_update_next_hop(struct bgp_export_state *s, eattr *a, ea_list **to);
+byte *bgp_create_end_mark_(struct bgp_channel *c, byte *buf);
 
-byte * bgp_create_end_mark(struct bgp_channel *c, byte *buf);
 
 /* Packet types */
 
diff --git a/proto/bgp/packets.c b/proto/bgp/packets.c
index 63ff955b..0338a887 100644
--- a/proto/bgp/packets.c
+++ b/proto/bgp/packets.c
@@ -2473,11 +2473,10 @@ bgp_bmp_prepare_bgp_hdr(byte *buf, const u16 msg_size, const u8 msg_type)
 }
 
 void
-bgp_rte_update_in_notify(struct channel *C, const net_addr *n,
-			 const struct rte *new, const struct rte_src *src)
+bgp_bmp_encode_rte(struct bgp_channel *c, struct bmp_proto *bmp, const net_addr *n,
+		   const struct rte *new, const struct rte_src *src)
 {
-//  struct bgp_proto *p = (void *) C->proto;
-  struct bgp_channel *c = (void *) C;
+//  struct bgp_proto *p = (void *) c->c.proto;
 
   byte buf[BGP_MAX_EXT_MSG_LENGTH];
   byte *pkt = buf + BGP_HEADER_LENGTH;
@@ -2507,7 +2506,7 @@ bgp_rte_update_in_notify(struct channel *C, const net_addr *n,
     return;
 
   bgp_bmp_prepare_bgp_hdr(buf, end - buf, PKT_UPDATE);
-  bmp_route_monitor_put_update_in_pre_msg(buf, end - buf);
+  bmp_route_monitor_put_update_in_pre_msg(bmp, buf, end - buf);
 }
 
 #endif /* CONFIG_BMP */
@@ -2614,6 +2613,14 @@ bgp_create_mp_end_mark(struct bgp_channel *c, byte *buf)
 }
 
 byte *
+bgp_create_end_mark_(struct bgp_channel *c, byte *buf)
+{
+  return (c->afi == BGP_AF_IPV4) ?
+    bgp_create_ip_end_mark(c, buf):
+    bgp_create_mp_end_mark(c, buf);
+}
+
+static byte *
 bgp_create_end_mark(struct bgp_channel *c, byte *buf)
 {
   struct bgp_proto *p = (void *) c->c.proto;
@@ -2621,9 +2628,7 @@ bgp_create_end_mark(struct bgp_channel *c, byte *buf)
   BGP_TRACE(D_PACKETS, "Sending END-OF-RIB");
   p->stats.tx_updates++;
 
-  return (c->afi == BGP_AF_IPV4) ?
-    bgp_create_ip_end_mark(c, buf):
-    bgp_create_mp_end_mark(c, buf);
+  return bgp_create_end_mark_(c, buf);
 }
 
 static inline void
diff --git a/proto/bmp/bmp.c b/proto/bmp/bmp.c
index 5c261584..aed9d9d6 100644
--- a/proto/bmp/bmp.c
+++ b/proto/bmp/bmp.c
@@ -56,8 +56,8 @@
 #include "nest/iface.h"
 #include "nest/route.h"
 
-// We allow for single instance of BMP protocol
-static struct bmp_proto *g_bmp;
+// List of BMP instances
+static list STATIC_LIST_INIT(bmp_proto_list);
 
 /* BMP Common Header [RFC 7854 - Section 4.1] */
 enum bmp_version {
@@ -214,7 +214,7 @@ struct bmp_data_node {
 };
 
 static void
-bmp_route_monitor_pre_policy_table_in_snapshot(struct bgp_channel *c);
+bmp_route_monitor_pre_policy_table_in_snapshot(struct bmp_proto *p, struct bgp_channel *c);
 
 static void
 bmp_common_hdr_serialize(buffer *stream, const enum bmp_message_type type, const u32 data_size)
@@ -447,16 +447,16 @@ bmp_peer_down_notif_msg_serialize(buffer *stream, const bool is_peer_global,
   bmp_put_data(stream, data, data_size);
 }
 
-void
-bmp_peer_up(const struct bgp_proto *bgp,
+static void
+bmp_peer_up_(struct bmp_proto *p, const struct bgp_proto *bgp,
 	    const byte *tx_open_msg, uint tx_open_length,
 	    const byte *rx_open_msg, uint rx_open_length)
 {
-  struct bmp_proto *p = g_bmp;
-
-  if (!p || !p->started)
+  if (!p->started)
     return;
 
+  TRACE(D_STATES, "Peer up for %s", bgp->p.name);
+
   // struct bmp_peer_map_key key = bmp_peer_map_key_create(bgp->remote_ip, bgp->remote_as);
   // bmp_peer_map_insert(&p->bgp_peers, key, (const byte *) &bgp, sizeof (bgp));
 
@@ -464,11 +464,21 @@ bmp_peer_up(const struct bgp_proto *bgp,
 
   struct bgp_channel *c;
   BGP_WALK_CHANNELS(bgp, c)
-    bmp_route_monitor_pre_policy_table_in_snapshot(c);
+    bmp_route_monitor_pre_policy_table_in_snapshot(p, c);
+}
+
+void
+bmp_peer_up(const struct bgp_proto *bgp,
+	    const byte *tx_open_msg, uint tx_open_length,
+	    const byte *rx_open_msg, uint rx_open_length)
+{
+  struct bmp_proto *p; node *n;
+  WALK_LIST2(p, n, bmp_proto_list, bmp_node)
+      bmp_peer_up_(p, bgp, tx_open_msg, tx_open_length, rx_open_msg, rx_open_length);
 }
 
 static void
-bmp_peer_init(const struct bgp_proto *bgp)
+bmp_peer_init(struct bmp_proto *p, const struct bgp_proto *bgp)
 {
   struct bgp_conn *conn = bgp->conn;
 
@@ -476,8 +486,8 @@ bmp_peer_init(const struct bgp_proto *bgp)
       !conn->local_open_msg || !conn->remote_open_msg)
     return;
 
-  bmp_peer_up(bgp, conn->local_open_msg, conn->local_open_length,
-	      conn->remote_open_msg, conn->remote_open_length);
+  bmp_peer_up_(p, bgp, conn->local_open_msg, conn->local_open_length,
+	       conn->remote_open_msg, conn->remote_open_length);
 }
 
 static const struct birdsock *
@@ -569,12 +579,10 @@ bmp_send_peer_up_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
 }
 
 
-void
-bmp_route_monitor_update_in_pre_begin(void)
+static void
+bmp_route_monitor_update_in_pre_begin_(struct bmp_proto *p)
 {
-  struct bmp_proto *p = g_bmp;
-
-  if (!p || !p->started)
+  if (!p->started)
     return;
 
   if (p->monitoring_rib.in_pre_policy == false)
@@ -592,11 +600,17 @@ bmp_route_monitor_update_in_pre_begin(void)
 }
 
 void
-bmp_route_monitor_put_update_in_pre_msg(const byte *data, const size_t data_size)
+bmp_route_monitor_update_in_pre_begin(void)
 {
-  struct bmp_proto *p = g_bmp;
+  struct bmp_proto *p; node *n;
+  WALK_LIST2(p, n, bmp_proto_list, bmp_node)
+    bmp_route_monitor_update_in_pre_begin_(p);
+}
 
-  if (!p || !p->started)
+void
+bmp_route_monitor_put_update_in_pre_msg(struct bmp_proto *p, const byte *data, const size_t data_size)
+{
+  if (!p->started)
     return;
 
   if (p->monitoring_rib.in_pre_policy == false)
@@ -617,11 +631,20 @@ bmp_route_monitor_put_update_in_pre_msg(const byte *data, const size_t data_size
 }
 
 void
-bmp_route_monitor_update_in_pre_commit(const struct bgp_proto *bgp)
+bmp_route_monitor_update_in_notify(struct channel *C, const net_addr *n,
+				   const struct rte *new, const struct rte_src *src)
 {
-  struct bmp_proto *p = g_bmp;
+  struct bgp_channel *c = (void *) C;
+
+  struct bmp_proto *p; node *nx;
+  WALK_LIST2(p, nx, bmp_proto_list, bmp_node)
+    bgp_bmp_encode_rte(c, p, n, new, src);
+}
 
-  if (!p || !p->started)
+static void
+bmp_route_monitor_update_in_pre_commit_(struct bmp_proto *p, const struct bgp_proto *bgp)
+{
+  if (!p->started)
     return;
 
   if (p->monitoring_rib.in_pre_policy == false)
@@ -670,11 +693,17 @@ bmp_route_monitor_update_in_pre_commit(const struct bgp_proto *bgp)
 }
 
 void
-bmp_route_monitor_update_in_pre_end(void)
+bmp_route_monitor_update_in_pre_commit(const struct bgp_proto *bgp)
 {
-  struct bmp_proto *p = g_bmp;
+  struct bmp_proto *p; node *n;
+  WALK_LIST2(p, n, bmp_proto_list, bmp_node)
+    bmp_route_monitor_update_in_pre_commit_(p, bgp);
+}
 
-  if (!p || !p->started)
+static void
+bmp_route_monitor_update_in_pre_end_(struct bmp_proto *p)
+{
+  if (!p->started)
     return;
 
   if (p->monitoring_rib.in_pre_policy == false)
@@ -692,11 +721,37 @@ bmp_route_monitor_update_in_pre_end(void)
   p->rt_table_in_pre_policy.update_in_progress = false;
 }
 
+void
+bmp_route_monitor_update_in_pre_end(void)
+{
+  struct bmp_proto *p; node *n;
+  WALK_LIST2(p, n, bmp_proto_list, bmp_node)
+    bmp_route_monitor_update_in_pre_end_(p);
+}
+
 static void
-bmp_route_monitor_pre_policy_table_in_snapshot(struct bgp_channel *c)
+bmp_route_monitor_end_of_rib_msg(struct bmp_proto *p, struct bgp_channel *c)
 {
-  struct bmp_proto *p = g_bmp;
+  struct bgp_proto *bgp = (void *) c->c.proto;
+
+  TRACE(D_PACKETS, "Sending END-OF-RIB for %s.%s", bgp->p.name, c->c.name);
+
+  byte rx_end_payload[DEFAULT_MEM_BLOCK_SIZE];
+  byte *pos = bgp_create_end_mark_(c, rx_end_payload + BGP_HEADER_LENGTH);
+  memset(rx_end_payload + BGP_MSG_HDR_MARKER_POS, 0xff,
+	 BGP_MSG_HDR_MARKER_SIZE); // BGP UPDATE MSG marker
+  put_u16(rx_end_payload + BGP_MSG_HDR_LENGTH_POS, pos - rx_end_payload);
+  put_u8(rx_end_payload + BGP_MSG_HDR_TYPE_POS, PKT_UPDATE);
+
+  bmp_route_monitor_update_in_pre_begin_(p);
+  bmp_route_monitor_put_update_in_pre_msg(p, rx_end_payload, pos - rx_end_payload);
+  bmp_route_monitor_update_in_pre_commit_(p, bgp);
+  bmp_route_monitor_update_in_pre_end_(p);
+}
 
+static void
+bmp_route_monitor_pre_policy_table_in_snapshot(struct bmp_proto *p, struct bgp_channel *c)
+{
   if (p->monitoring_rib.in_pre_policy == false)
     return;
 
@@ -713,37 +768,22 @@ bmp_route_monitor_pre_policy_table_in_snapshot(struct bgp_channel *c)
   {
     P = n->routes->sender->proto;
     if (P->proto->class != PROTOCOL_BGP)
-    {
       continue;
-    }
 
-    bmp_route_monitor_update_in_pre_begin();
+    bmp_route_monitor_update_in_pre_begin_(p);
 
     rte *e;
     for (e = n->routes; e; e = e->next)
-    {
-      bgp_rte_update_in_notify(&c->c, n->n.addr, e, e->src);
-    }
+      bgp_bmp_encode_rte(c, p, n->n.addr, e, e->src);
 
-    bmp_route_monitor_update_in_pre_commit((struct bgp_proto *) P);
-    bmp_route_monitor_update_in_pre_end();
+    bmp_route_monitor_update_in_pre_commit_(p, (struct bgp_proto *) P);
+    bmp_route_monitor_update_in_pre_end_(p);
     ++cnt;
   }
   FIB_ITERATE_END;
 
   if (cnt > 0)
-  {
-    bmp_route_monitor_update_in_pre_begin();
-    byte rx_end_payload[DEFAULT_MEM_BLOCK_SIZE];
-    byte *pos = bgp_create_end_mark(c, rx_end_payload + BGP_HEADER_LENGTH);
-    memset(rx_end_payload + BGP_MSG_HDR_MARKER_POS, 0xff,
-             BGP_MSG_HDR_MARKER_SIZE); // BGP UPDATE MSG marker
-    put_u16(rx_end_payload + BGP_MSG_HDR_LENGTH_POS, pos - rx_end_payload);
-    put_u8(rx_end_payload + BGP_MSG_HDR_TYPE_POS, PKT_UPDATE);
-    bmp_route_monitor_put_update_in_pre_msg(rx_end_payload, pos - rx_end_payload);
-    bmp_route_monitor_update_in_pre_commit((struct bgp_proto *) c->c.proto);
-    bmp_route_monitor_update_in_pre_end();
-  }
+    bmp_route_monitor_end_of_rib_msg(p, c);
 }
 
 static void
@@ -765,15 +805,15 @@ bmp_send_peer_down_notif_msg(struct bmp_proto *p, const struct bgp_proto *bgp,
   bmp_buffer_free(&payload);
 }
 
-void
-bmp_peer_down(const struct bgp_proto *bgp, const int err_class,
-	      const byte *msg, size_t msg_length)
+static void
+bmp_peer_down_(struct bmp_proto *p, const struct bgp_proto *bgp,
+	       const int err_class, const byte *msg, size_t msg_length)
 {
-  struct bmp_proto *p = g_bmp;
-
-  if (!p || !p->started)
+  if (!p->started)
     return;
 
+  TRACE(D_STATES, "Peer down for %s", bgp->p.name);
+
   // struct bmp_peer_map_key key = bmp_peer_map_key_create(bgp->remote_ip, bgp->remote_as);
   // bmp_peer_map_remove(&p->bgp_peers, key);
 
@@ -809,6 +849,15 @@ bmp_peer_down(const struct bgp_proto *bgp, const int err_class,
   bmp_buffer_free(&payload);
 }
 
+void
+bmp_peer_down(const struct bgp_proto *bgp, const int err_class,
+	      const byte *msg, size_t msg_length)
+{
+  struct bmp_proto *p; node *n;
+  WALK_LIST2(p, n, bmp_proto_list, bmp_node)
+    bmp_peer_down_(p, bgp, err_class, msg, msg_length);
+}
+
 static void
 bmp_send_termination_msg(struct bmp_proto *p,
   const enum bmp_term_reason reason)
@@ -857,7 +906,7 @@ bmp_startup(struct bmp_proto *p)
   struct proto *peer;
   WALK_LIST(peer, proto_list)
     if ((peer->proto->class == PROTOCOL_BGP) && (peer->proto_state == PS_UP))
-      bmp_peer_init((struct bgp_proto *) peer);
+      bmp_peer_init(p, (struct bgp_proto *) peer);
 
   proto_notify_state(&p->p, PS_UP);
 }
@@ -905,6 +954,8 @@ bmp_connect(struct bmp_proto *p)
   p->sk = sk;
   sk->data = p;
 
+  TRACE(D_EVENTS, "Connecting to %I port %u", sk->daddr, sk->dport);
+
   int rc = sk_open(sk);
 
   if (rc < 0)
@@ -919,6 +970,8 @@ bmp_connected(struct birdsock *sk)
 {
   struct bmp_proto *p = (void *) sk->data;
 
+  TRACE(D_EVENTS, "Connected");
+
   sk->rx_hook = bmp_rx;
   sk->tx_hook = bmp_tx;
   tm_stop(p->connect_retry_timer);
@@ -1025,11 +1078,10 @@ bmp_start(struct proto *P)
   init_list(&p->rt_table_in_pre_policy.update_msg_queue);
   p->started = false;
   p->sock_err = 0;
+  add_tail(&bmp_proto_list, &p->bmp_node);
 
   tm_start(p->connect_retry_timer, CONNECT_INIT_TIME);
 
-  g_bmp = p;
-
   return PS_START;
 }
 
@@ -1045,7 +1097,7 @@ bmp_shutdown(struct proto *P)
   }
 
   p->sock_err = 0;
-  g_bmp = NULL;
+  rem_node(&p->bmp_node);
 
   return PS_DOWN;
 }
diff --git a/proto/bmp/bmp.h b/proto/bmp/bmp.h
index be28468c..258a5089 100644
--- a/proto/bmp/bmp.h
+++ b/proto/bmp/bmp.h
@@ -57,6 +57,7 @@ struct rt_table_info {
 struct bmp_proto {
   struct proto p;                  // Parent proto
   const struct bmp_config *cf;     // Shortcut to BMP configuration
+  node bmp_node;                   // Node in bmp_proto_list
   sock *sk;                        // TCP connection
   event *tx_ev;                    // TX event
   char sys_descr[MIB_II_STR_LEN];  // sysDescr MIB-II [RFC1213] object
@@ -90,20 +91,22 @@ bmp_peer_up(const struct bgp_proto *bgp,
 	    const byte *rx_open_msg, uint rx_open_length);
 
 /**
- * The following 4 functions create BMP Route Monitoring message based on
+ * The following 5 functions create BMP Route Monitoring message based on
  * pre-policy Adj-RIB-In. Composing Route Monitoring message consist of few
  * stages. First of all call bmp_route_monitor_update_in_pre_begin() in order
  * to start composing message. As a second step, call
- * bmp_route_monitor_put_update_in_pre_msg() in order to save BGP UPDATE msg.
- * As a third step call bmp_route_monitor_update_in_pre_commit() in order to
- * send BMP Route Monitoring message to the BMP collector. As a last step,
+ * bmp_route_monitor_update_in_notify() to announce each rte, which internally
+ * calls bmp_route_monitor_put_update_in_pre_msg() in order to save BGP UPDATE
+ * msg. As a third step call bmp_route_monitor_update_in_pre_commit() in order
+ * to send BMP Route Monitoring message to the BMP collector. As a last step,
  * call bmp_route_monitor_update_in_pre_end() in order to release resources.
  */
 void
 bmp_route_monitor_update_in_pre_begin(void);
 
 void
-bmp_route_monitor_put_update_in_pre_msg(const byte *data, const size_t data_size);
+bmp_route_monitor_update_in_notify(struct channel *C, const net_addr *n,
+				   const struct rte *new, const struct rte_src *src);
 
 void
 bmp_route_monitor_update_in_pre_commit(const struct bgp_proto *bgp);
@@ -111,6 +114,9 @@ bmp_route_monitor_update_in_pre_commit(const struct bgp_proto *bgp);
 void
 bmp_route_monitor_update_in_pre_end(void);
 
+void
+bmp_route_monitor_put_update_in_pre_msg(struct bmp_proto *p, const byte *data, const size_t data_size);
+
 /**
  * bmp_peer_down - send notification that BGP peer connection is not in
  * established state
@@ -124,7 +130,6 @@ bmp_peer_down(const struct bgp_proto *bgp, const int err_class, const byte *pkt,
 
 static inline void bmp_peer_up(const struct bgp_proto *bgp UNUSED, const byte *tx_open_msg UNUSED, uint tx_open_length UNUSED, const byte *rx_open_msg UNUSED, uint rx_open_length UNUSED) { }
 static inline void bmp_route_monitor_update_in_pre_begin(void) { }
-static inline void bmp_route_monitor_put_update_in_pre_msg(const byte *data UNUSED, const size_t data_size UNUSED) { }
 static inline void bmp_route_monitor_update_in_pre_commit(const struct bgp_proto *bgp UNUSED) { }
 static inline void bmp_route_monitor_update_in_pre_end(void) { }
 static inline void bmp_peer_down(const struct bgp_proto *bgp UNUSED, const int err_class UNUSED, const byte *pkt UNUSED, size_t pkt_size UNUSED) { }