Converting the former BFD loop to a universal IO loop and protocol loop.

There is a simple universal IO loop, taking care of events, timers and
sockets. Primarily, one instance of a protocol should use exactly one IO
loop to do all its work, as is now done in BFD.

Contrary to previous versions, the loop is now launched and cleaned by
the nest/proto.c code, allowing for a protocol to just request its own
loop by setting the loop's lock order in config higher than the_bird.

It is not supported nor checked if any protocol changed the requested
lock order in reconfigure. No protocol should do it at all.

12 files changed, 258 insertions, 101 deletions

diff --git a/lib/birdlib.h b/lib/birdlib.h
index 3dc39d19..385bf75c 100644
--- a/lib/birdlib.h
+++ b/lib/birdlib.h
@@ -71,6 +71,7 @@ static inline int u64_cmp(u64 i1, u64 i2)
 /* Macros for gcc attributes */
 
 #define NORET __attribute__((noreturn))
+#define USE_RESULT __atribute__((warn_unused_result))
 #define UNUSED __attribute__((unused))
 #define PACKED __attribute__((packed))
 #define NONNULL(...) __attribute__((nonnull((__VA_ARGS__))))
diff --git a/lib/coro.h b/lib/coro.h
index 51712b36..17ccff89 100644
--- a/lib/coro.h
+++ b/lib/coro.h
@@ -2,7 +2,7 @@
  *	BIRD Coroutines
  *
  *	(c) 2017 Martin Mares <mj@ucw.cz>
- *	(c) 2020 Maria Matejka <mq@jmq.cz>
+ *	(c) 2020-2021 Maria Matejka <mq@jmq.cz>
  *
  *	Can be freely distributed and used under the terms of the GNU GPL.
  */
@@ -22,5 +22,8 @@ struct coroutine;
  */
 struct coroutine *coro_run(pool *, void (*entry)(void *), void *data);
 
+/* Get self. */
+extern _Thread_local struct coroutine *this_coro;
+
 
 #endif
diff --git a/lib/event.c b/lib/event.c
index 273447e0..6c5c8b14 100644
--- a/lib/event.c
+++ b/lib/event.c
@@ -19,8 +19,14 @@
  * events in them and explicitly ask to run them.
  */
 
+#undef LOCAL_DEBUG
+
 #include "nest/bird.h"
 #include "lib/event.h"
+#include "lib/locking.h"
+#include "lib/io-loop.h"
+
+extern _Thread_local struct coroutine *this_coro;
 
 event_list global_event_list;
 event_list global_work_list;
@@ -28,11 +34,16 @@ event_list global_work_list;
 inline void
 ev_postpone(event *e)
 {
+  event_list *el = e->list;
+  if (!el)
+    return;
+
+  ASSERT_DIE(birdloop_inside(el->loop));
+
+  LOCK_DOMAIN(event, el->lock);
   if (ev_active(e))
-    {
-      rem_node(&e->n);
-      e->n.next = NULL;
-    }
+    rem_node(&e->n);
+  UNLOCK_DOMAIN(event, el->lock);
 }
 
 static void
@@ -95,40 +106,25 @@ ev_run(event *e)
  * list @l which can be run by calling ev_run_list().
  */
 inline void
-ev_enqueue(event_list *l, event *e)
+ev_send(event_list *l, event *e)
 {
-  ev_postpone(e);
-  add_tail(l, &e->n);
-}
+  DBG("ev_send(%p, %p)\n", l, e);
+  ASSERT_DIE(e->hook);
+  ASSERT_DIE(!e->list || (e->list == l) || (e->list->loop == l->loop));
 
-/**
- * ev_schedule - schedule an event
- * @e: an event
- *
- * This function schedules an event by enqueueing it to a system-wide
- * event list which is run by the platform dependent code whenever
- * appropriate.
- */
-void
-ev_schedule(event *e)
-{
-  ev_enqueue(&global_event_list, e);
-}
+  e->list = l;
 
-/**
- * ev_schedule_work - schedule a work-event.
- * @e: an event
- *
- * This function schedules an event by enqueueing it to a system-wide work-event
- * list which is run by the platform dependent code whenever appropriate. This
- * is designated for work-events instead of regular events. They are executed
- * less often in order to not clog I/O loop.
- */
-void
-ev_schedule_work(event *e)
-{
-  if (!ev_active(e))
-    add_tail(&global_work_list, &e->n);
+  LOCK_DOMAIN(event, l->lock);
+  if (enlisted(&e->n))
+  {
+    UNLOCK_DOMAIN(event, l->lock);
+    return;
+  }
+
+  add_tail(&l->events, &e->n);
+  UNLOCK_DOMAIN(event, l->lock);
+
+  birdloop_ping(l->loop);
 }
 
 void io_log_event(void *hook, void *data);
@@ -142,35 +138,64 @@ void io_log_event(void *hook, void *data);
 int
 ev_run_list(event_list *l)
 {
+  const _Bool legacy = LEGACY_EVENT_LIST(l);
+
+  if (legacy)
+    ASSERT_THE_BIRD_LOCKED;
+
   node *n;
-  list tmp_list;
 
+  list tmp_list;
   init_list(&tmp_list);
-  add_tail_list(&tmp_list, l);
-  init_list(l);
+
+  /* Move the event list contents to a local list to avoid executing repeatedly added events */
+  LOCK_DOMAIN(event, l->lock);
+  add_tail_list(&tmp_list, &l->events);
+  init_list(&l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+
   WALK_LIST_FIRST(n, tmp_list)
     {
       event *e = SKIP_BACK(event, n, n);
 
-      /* This is ugly hack, we want to log just events executed from the main I/O loop */
-      if ((l == &global_event_list) || (l == &global_work_list))
+      if (legacy)
+      {
+	/* The legacy way of event execution */
 	io_log_event(e->hook, e->data);
-
-      ev_run(e);
+	ev_postpone(e);
+	e->hook(e->data);
+      }
+      else
+      {
+	// io_log_event(e->hook, e->data); /* TODO: add support for event logging in other io loops */
+	ASSERT_DIE(e->list == l);
+	LOCK_DOMAIN(event, l->lock);
+	rem_node(&e->n);
+	UNLOCK_DOMAIN(event, l->lock);
+	e->hook(e->data);
+      }
     }
 
-  return !EMPTY_LIST(*l);
+  LOCK_DOMAIN(event, l->lock);
+  int repeat = ! EMPTY_LIST(l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+  return repeat;
 }
 
 int
 ev_run_list_limited(event_list *l, uint limit)
 {
+  ASSERT_DIE(LEGACY_EVENT_LIST(l));
+  ASSERT_THE_BIRD_LOCKED;
+
   node *n;
   list tmp_list;
 
+  LOCK_DOMAIN(event, l->lock);
   init_list(&tmp_list);
-  add_tail_list(&tmp_list, l);
-  init_list(l);
+  add_tail_list(&tmp_list, &l->events);
+  init_list(&l->events);
+  UNLOCK_DOMAIN(event, l->lock);
 
   WALK_LIST_FIRST(n, tmp_list)
     {
@@ -179,21 +204,23 @@ ev_run_list_limited(event_list *l, uint limit)
       if (!limit)
 	break;
 
-      /* This is ugly hack, we want to log just events executed from the main I/O loop */
-      if ((l == &global_event_list) || (l == &global_work_list))
-	io_log_event(e->hook, e->data);
+      io_log_event(e->hook, e->data);
 
       ev_run(e);
       limit--;
     }
 
+  LOCK_DOMAIN(event, l->lock);
   if (!EMPTY_LIST(tmp_list))
   {
     /* Attach new items after the unprocessed old items */
-    add_tail_list(&tmp_list, l);
-    init_list(l);
-    add_tail_list(l, &tmp_list);
+    add_tail_list(&tmp_list, &l->events);
+    init_list(&l->events);
+    add_tail_list(&l->events, &tmp_list);
   }
 
-  return !EMPTY_LIST(*l);
+  int repeat = ! EMPTY_LIST(l->events);
+  UNLOCK_DOMAIN(event, l->lock);
+
+  return repeat;
 }
diff --git a/lib/event.h b/lib/event.h
index 5f3b78d8..6c358f84 100644
--- a/lib/event.h
+++ b/lib/event.h
@@ -10,33 +10,62 @@
 #define _BIRD_EVENT_H_
 
 #include "lib/resource.h"
+#include "lib/locking.h"
+
+#include <stdatomic.h>
+
+DEFINE_DOMAIN(event);
 
 typedef struct event {
   resource r;
   void (*hook)(void *);
   void *data;
   node n;				/* Internal link */
+  struct event_list *list;		/* List where this event is put in */
 } event;
 
-typedef list event_list;
+typedef struct event_list {
+  list events;
+  pool *pool;
+  struct birdloop *loop;
+  DOMAIN(event) lock;
+} event_list;
 
 extern event_list global_event_list;
 extern event_list global_work_list;
 
 event *ev_new(pool *);
 void ev_run(event *);
-#define ev_init_list(el) init_list(el)
-void ev_enqueue(event_list *, event *);
-void ev_schedule(event *);
-void ev_schedule_work(event *);
+
+static inline void ev_init_list(event_list *el, struct birdloop *loop, const char *name)
+{
+  init_list(&el->events);
+  el->loop = loop;
+  el->lock = DOMAIN_NEW(event, name);
+}
+
+void ev_send(event_list *, event *);
+#define ev_send_loop(l, e) ev_send(birdloop_event_list((l)), (e))
+
+#define ev_schedule(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_event_list, (e)); })
+#define ev_schedule_work(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_work_list, (e)); })
+
 void ev_postpone(event *);
 int ev_run_list(event_list *);
 int ev_run_list_limited(event_list *, uint);
 
+#define LEGACY_EVENT_LIST(l)  (((l) == &global_event_list) || ((l) == &global_work_list))
+
+_Bool birdloop_inside(struct birdloop *loop);
+
 static inline int
 ev_active(event *e)
 {
-  return e->n.next != NULL;
+  if (e->list == NULL)
+    return 0;
+
+  ASSERT_DIE(birdloop_inside(e->list->loop));
+  return enlisted(&e->n);
 }
 
 static inline event*
diff --git a/lib/event_test.c b/lib/event_test.c
index e1215bba..9dda3e2a 100644
--- a/lib/event_test.c
+++ b/lib/event_test.c
@@ -48,14 +48,17 @@ init_event_check_points(void)
     event_check_points[i] = 0;
 }
 
+void resource_sys_init(void);
+
 static int
 t_ev_run_list(void)
 {
   int i;
 
+  resource_sys_init();
   resource_init();
   olock_init();
-  timer_init();
+  birdloop_init();
   io_init();
   rt_init();
   if_init();
@@ -82,7 +85,9 @@ main(int argc, char *argv[])
 {
   bt_init(argc, argv);
 
+  the_bird_lock();
   bt_test_suite(t_ev_run_list, "Schedule and run 3 events in right order.");
+  the_bird_unlock();
 
   return bt_exit_value();
 }
diff --git a/lib/flowspec_test.c b/lib/flowspec_test.c
index ed4afe51..f7f70982 100644
--- a/lib/flowspec_test.c
+++ b/lib/flowspec_test.c
@@ -666,10 +666,13 @@ t_formatting6(void)
   return 1;
 }
 
+void resource_sys_init(void);
+
 int
 main(int argc, char *argv[])
 {
   bt_init(argc, argv);
+  resource_sys_init();
 
   bt_test_suite(t_read_length,  "Testing get NLRI length");
   bt_test_suite(t_write_length, "Testing set NLRI length");
diff --git a/lib/io-loop.h b/lib/io-loop.h
new file mode 100644
index 00000000..25f1b2a3
--- /dev/null
+++ b/lib/io-loop.h
@@ -0,0 +1,54 @@
+/*
+ *	BIRD -- I/O and event loop
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL.
+ */
+
+#ifndef _BIRD_IO_LOOP_H_
+#define _BIRD_IO_LOOP_H_
+
+#include "nest/bird.h"
+#include "lib/lists.h"
+#include "lib/locking.h"
+#include "lib/resource.h"
+#include "lib/event.h"
+#include "lib/socket.h"
+
+void sk_start(sock *s);
+void sk_stop(sock *s);
+void sk_reloop(sock *s, struct birdloop *loop);
+
+extern struct birdloop main_birdloop;
+
+/* Start a new birdloop owned by given pool and domain */
+struct birdloop *birdloop_new(pool *p, uint order, const char *name);
+
+/* Stop the loop. At the end, the @stopped callback is called unlocked in tail
+ * position to finish cleanup. Run birdloop_free() from that callback to free
+ * the loop itself. */
+void birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data);
+void birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data);
+void birdloop_free(struct birdloop *loop);
+
+/* Get birdloop's event list */
+event_list *birdloop_event_list(struct birdloop *loop);
+
+/* Get birdloop's time heap */
+struct timeloop *birdloop_time_loop(struct birdloop *loop);
+
+/* Enter and exit the birdloop */
+void birdloop_enter(struct birdloop *loop);
+void birdloop_leave(struct birdloop *loop);
+
+_Bool birdloop_inside(struct birdloop *loop);
+
+void birdloop_mask_wakeups(struct birdloop *loop);
+void birdloop_unmask_wakeups(struct birdloop *loop);
+
+void birdloop_link(struct birdloop *loop);
+void birdloop_unlink(struct birdloop *loop);
+
+void birdloop_ping(struct birdloop *loop);
+
+void birdloop_init(void);
+#endif /* _BIRD_IO_LOOP_H_ */
diff --git a/lib/lists.h b/lib/lists.h
index 64b4a981..dc49ec8a 100644
--- a/lib/lists.h
+++ b/lib/lists.h
@@ -68,6 +68,18 @@ typedef union list {			/* In fact two overlayed nodes */
 
 #define EMPTY_LIST(list) (!(list).head->next)
 
+static inline _Bool
+enlisted(node *n)
+{
+  switch ((!!n->next) + (!!n->prev))
+  {
+    case 0: return 0;
+    case 2: return 1;
+    case 1: bug("Garbled event list node");
+  }
+
+  bug("Maths is broken. And you should see a new heaven and a new earth: for the first heaven and the first earth had been passed away.");
+}
 
 #ifndef _BIRD_LISTS_C_
 #define LIST_INLINE static inline
diff --git a/lib/locking.h b/lib/locking.h
index eef60154..ab5c06af 100644
--- a/lib/locking.h
+++ b/lib/locking.h
@@ -14,6 +14,9 @@ struct domain_generic;
 /* Here define the global lock order; first to last. */
 struct lock_order {
   struct domain_generic *the_bird;
+  struct domain_generic *proto;
+  struct domain_generic *rtable;
+  struct domain_generic *event;
 };
 
 extern _Thread_local struct lock_order locking_stack;
@@ -21,24 +24,40 @@ extern _Thread_local struct domain_generic **last_locked;
 
 #define DOMAIN(type) struct domain__##type
 #define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; }
+#define DOMAIN_ORDER(type)  OFFSETOF(struct lock_order, type)
 
-#define DOMAIN_NEW(type, name)  (DOMAIN(type)) { .type = domain_new(name, OFFSETOF(struct lock_order, type)) }
+#define DOMAIN_NEW(type, name)  (DOMAIN(type)) { .type = domain_new(name, DOMAIN_ORDER(type)) }
 struct domain_generic *domain_new(const char *name, uint order);
 
+#define DOMAIN_FREE(type, d)	domain_free((d).type)
+void domain_free(struct domain_generic *);
+
 #define DOMAIN_NULL(type)   (DOMAIN(type)) {}
 
 #define LOCK_DOMAIN(type, d)	do_lock(((d).type), &(locking_stack.type))
 #define UNLOCK_DOMAIN(type, d)  do_unlock(((d).type), &(locking_stack.type))
 
+#define DOMAIN_IS_LOCKED(type, d) (((d).type) == (locking_stack.type))
+#define DG_IS_LOCKED(d)	((d) == *(DG_LSP(d)))
+
 /* Internal for locking */
 void do_lock(struct domain_generic *dg, struct domain_generic **lsp);
 void do_unlock(struct domain_generic *dg, struct domain_generic **lsp);
 
+uint dg_order(struct domain_generic *dg);
+
+#define DG_LSP(d)	((struct domain_generic **) (((void *) &locking_stack) + dg_order(d)))
+#define DG_LOCK(d)	do_lock(d, DG_LSP(d))
+#define DG_UNLOCK(d)	do_unlock(d, DG_LSP(d))
+
 /* Use with care. To be removed in near future. */
 DEFINE_DOMAIN(the_bird);
 extern DOMAIN(the_bird) the_bird_domain;
 
 #define the_bird_lock()		LOCK_DOMAIN(the_bird, the_bird_domain)
 #define the_bird_unlock()	UNLOCK_DOMAIN(the_bird, the_bird_domain)
+#define the_bird_locked()	DOMAIN_IS_LOCKED(the_bird, the_bird_domain)
+
+#define ASSERT_THE_BIRD_LOCKED	({ if (!the_bird_locked()) bug("The BIRD lock must be locked here: %s:%d", __FILE__, __LINE__); })
 
 #endif
diff --git a/lib/socket.h b/lib/socket.h
index 96fedeeb..5bdab7f3 100644
--- a/lib/socket.h
+++ b/lib/socket.h
@@ -12,6 +12,7 @@
 #include <errno.h>
 
 #include "lib/resource.h"
+#include "lib/event.h"
 #ifdef HAVE_LIBSSH
 #define LIBSSH_LEGACY_0_4
 #include <libssh/libssh.h>
@@ -79,6 +80,7 @@ typedef struct birdsock {
   const char *password;			/* Password for MD5 authentication */
   const char *err;			/* Error message */
   struct ssh_sock *ssh;			/* Used in SK_SSH */
+  struct event reloop;			/* Reloop event */
 } sock;
 
 sock *sock_new(pool *);			/* Allocate new socket */
@@ -128,6 +130,7 @@ extern int sk_priority_control;		/* Suggested priority for control traffic, shou
 #define SKF_TRUNCATED	0x200	/* Received packet was truncated, set by IO layer */
 #define SKF_HDRINCL	0x400	/* Used internally */
 #define SKF_PKTINFO	0x800	/* Used internally */
+#define SKF_PASSIVE_THREAD  0x1000  /* Child sockets used in thread, do not add to main loop */
 
 /*
  *	Socket types		     SA SP DA DP IF  TTL SendTo	(?=may, -=must not, *=must)
diff --git a/lib/timer.c b/lib/timer.c
index ff1fb5ef..eb7ea690 100644
--- a/lib/timer.c
+++ b/lib/timer.c
@@ -37,15 +37,8 @@
 #include "lib/resource.h"
 #include "lib/timer.h"
 
-
-struct timeloop main_timeloop;
-
-
 #include <pthread.h>
 
-/* Data accessed and modified from proto/bfd/io.c */
-_Thread_local struct timeloop *local_timeloop;
-
 _Atomic btime last_time;
 _Atomic btime real_time;
 
@@ -76,7 +69,7 @@ tm_dump(resource *r)
   if (t->recurrent)
     debug("recur %d, ", t->recurrent);
   if (t->expires)
-    debug("expires in %d ms)\n", (t->expires - current_time()) TO_MS);
+    debug("in loop %p expires in %d ms)\n", t->loop, (t->expires - current_time()) TO_MS);
   else
     debug("inactive)\n");
 }
@@ -99,8 +92,8 @@ tm_new(pool *p)
   return t;
 }
 
-void
-tm_set(timer *t, btime when)
+static void
+tm_set_in_tl(timer *t, btime when, struct timeloop *local_timeloop)
 {
   uint tc = timers_count(local_timeloop);
 
@@ -122,17 +115,17 @@ tm_set(timer *t, btime when)
     HEAP_DECREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
   }
 
-#ifdef CONFIG_BFD
-  /* Hack to notify BFD loops */
-  if ((local_timeloop != &main_timeloop) && (t->index == 1))
-    wakeup_kick_current();
-#endif
+  t->loop = local_timeloop;
+
+  if ((t->index == 1) && (local_timeloop->coro != this_coro))
+    birdloop_ping(local_timeloop->loop);
 }
 
 void
-tm_start(timer *t, btime after)
+tm_set_in(timer *t, btime when, struct birdloop *loop)
 {
-  tm_set(t, current_time() + MAX(after, 0));
+  ASSERT_DIE(birdloop_inside(loop));
+  tm_set_in_tl(t, when, birdloop_time_loop(loop));
 }
 
 void
@@ -141,18 +134,23 @@ tm_stop(timer *t)
   if (!t->expires)
     return;
 
-  uint tc = timers_count(local_timeloop);
+  TLOCK_TIMER_ASSERT(t->loop);
 
-  HEAP_DELETE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
-  BUFFER_POP(local_timeloop->timers);
+  uint tc = timers_count(t->loop);
+
+  HEAP_DELETE(t->loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index);
+  BUFFER_POP(t->loop->timers);
 
   t->index = -1;
   t->expires = 0;
+  t->loop = NULL;
 }
 
 void
 timers_init(struct timeloop *loop, pool *p)
 {
+  TLOCK_TIMER_ASSERT(loop);
+
   BUFFER_INIT(loop->timers, p, 4);
   BUFFER_PUSH(loop->timers) = NULL;
 }
@@ -160,8 +158,10 @@ timers_init(struct timeloop *loop, pool *p)
 void io_log_event(void *hook, void *data);
 
 void
-timers_fire(struct timeloop *loop)
+timers_fire(struct timeloop *loop, int io_log)
 {
+  TLOCK_TIMER_ASSERT(loop);
+
   btime base_time;
   timer *t;
 
@@ -183,26 +183,19 @@ timers_fire(struct timeloop *loop)
       if (t->randomize)
 	when += random() % (t->randomize + 1);
 
-      tm_set(t, when);
+      tm_set_in_tl(t, when, loop);
     }
     else
       tm_stop(t);
 
     /* This is ugly hack, we want to log just timers executed from the main I/O loop */
-    if (loop == &main_timeloop)
+    if (io_log)
       io_log_event(t->hook, t->data);
 
     t->hook(t);
   }
 }
 
-void
-timer_init(void)
-{
-  timers_init(&main_timeloop, &root_pool);
-  local_timeloop = &main_timeloop;
-}
-
 
 /**
  * tm_parse_time - parse a date and time
diff --git a/lib/timer.h b/lib/timer.h
index b201b8c8..04544ace 100644
--- a/lib/timer.h
+++ b/lib/timer.h
@@ -12,6 +12,8 @@
 
 #include "nest/bird.h"
 #include "lib/buffer.h"
+#include "lib/io-loop.h"
+#include "lib/locking.h"
 #include "lib/resource.h"
 
 #include <stdatomic.h>
@@ -29,22 +31,27 @@ typedef struct timer
   uint randomize;			/* Amount of randomization */
   uint recurrent;			/* Timer recurrence */
 
+  struct timeloop *loop;		/* Loop where the timer is active */
+
   int index;
 } timer;
 
 struct timeloop
 {
   BUFFER_(timer *) timers;
+  struct domain_generic *domain;
+  struct birdloop *loop;
+  struct coroutine *coro;
 };
 
+#define TLOCK_TIMER_ASSERT(loop) ASSERT_DIE((loop)->domain && DG_IS_LOCKED((loop)->domain))
+#define TLOCK_LOCAL_ASSERT(loop) ASSERT_DIE(!(loop)->domain || DG_IS_LOCKED((loop)->domain))
+
 static inline uint timers_count(struct timeloop *loop)
-{ return loop->timers.used - 1; }
+{ TLOCK_TIMER_ASSERT(loop); return loop->timers.used - 1; }
 
 static inline timer *timers_first(struct timeloop *loop)
-{ return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; }
-
-extern struct timeloop main_timeloop;
-extern _Thread_local struct timeloop *local_timeloop;
+{ TLOCK_TIMER_ASSERT(loop); return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; }
 
 #define current_time()		atomic_load_explicit(&last_time, memory_order_acquire)
 #define current_real_time()	atomic_load_explicit(&real_time, memory_order_acquire)
@@ -54,10 +61,13 @@ extern _Thread_local struct timeloop *local_timeloop;
 extern btime boot_time;
 
 timer *tm_new(pool *p);
-void tm_set(timer *t, btime when);
-void tm_start(timer *t, btime after);
+#define tm_set(t, when) tm_set_in((t), (when), &main_birdloop)
+#define tm_start(t, after) tm_start_in((t), (after), &main_birdloop)
 void tm_stop(timer *t);
 
+void tm_set_in(timer *t, btime when, struct birdloop *loop);
+#define tm_start_in(t, after, loop) tm_set_in((t), (current_time() + MAX_((after), 0)), loop)
+
 static inline int
 tm_active(timer *t)
 {
@@ -101,9 +111,7 @@ void times_update(void);
 
 /* For I/O loop */
 void timers_init(struct timeloop *loop, pool *p);
-void timers_fire(struct timeloop *loop);
-
-void timer_init(void);
+void timers_fire(struct timeloop *loop, int io_log);
 
 
 struct timeformat {