Decoupling loops from threads to allow fixed thread count

On large configurations, too many threads would spawn with one thread
per loop. Therefore, threads may now run multiple loops at once. The
thread count is configurable and may be changed during run. All threads
are spawned on startup.

This change helps with memory bloating. BIRD filters need large
temporary memory blocks to store their stack and also memory management
keeps its hot page storage per-thread.

Known bugs:
* Thread autobalancing is not yet implemented.
* Low latency loops are executed together with standard loops.

1 files changed, 33 insertions, 13 deletions

diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h
index 29ca96d6..33d529da 100644
--- a/sysdep/unix/io-loop.h
+++ b/sysdep/unix/io-loop.h
@@ -21,26 +21,16 @@ void pipe_kick(struct pipe *);
 
 struct birdloop
 {
+  node n;
+
   pool *pool;
 
   struct timeloop time;
   event_list event_list;
   list sock_list;
-  uint sock_num;
-
-  BUFFER(sock *) poll_sk;
-  BUFFER(struct pollfd) poll_fd;
-  u8 poll_changed;
-  u8 close_scheduled;
+  int sock_num;
 
   uint ping_pending;
-  _Atomic u32 ping_sent;
-  struct pipe wakeup;
-
-  pthread_t thread_id;
-  pthread_attr_t thread_attr;
-
-  struct rcu_birdloop rcu;
 
   uint links;
 
@@ -51,6 +41,36 @@ struct birdloop
   void *stop_data;
 
   struct birdloop *prev_loop;
+
+  struct bird_thread *thread;
+  struct pollfd *pfd;
+
+  u64 total_time_spent_ns;
+};
+
+struct bird_thread
+{
+  node n;
+
+  struct pollfd *pfd;
+  uint pfd_max;
+
+  _Atomic u32 ping_sent;
+  _Atomic u32 run_cleanup;
+  _Atomic u32 poll_changed;
+
+  struct pipe wakeup;
+  event_list priority_events;
+
+  pthread_t thread_id;
+  pthread_attr_t thread_attr;
+
+  struct rcu_thread rcu;
+
+  list loops;
+  pool *pool;
+
+  event cleanup_event;
 };
 
 #endif