diff options
38 files changed, 785 insertions, 466 deletions
diff --git a/filter/filter_test.c b/filter/filter_test.c index 63764964..5b24a765 100644 --- a/filter/filter_test.c +++ b/filter/filter_test.c @@ -70,6 +70,7 @@ int main(int argc, char *argv[]) { bt_init(argc, argv); + bt_bird_init(); bt_assert_hook = bt_assert_filter; diff --git a/lib/a-set_test.c b/lib/a-set_test.c index 3280031f..693b8f08 100644 --- a/lib/a-set_test.c +++ b/lib/a-set_test.c @@ -221,6 +221,7 @@ t_set_ec_delete(void) return 1; } + int main(int argc, char *argv[]) { diff --git a/lib/birdlib.h b/lib/birdlib.h index dbc754d4..d55b1a44 100644 --- a/lib/birdlib.h +++ b/lib/birdlib.h @@ -86,6 +86,7 @@ static inline int u64_cmp(u64 i1, u64 i2) /* Macros for gcc attributes */ #define NORET __attribute__((noreturn)) +#define USE_RESULT __atribute__((warn_unused_result)) #define UNUSED __attribute__((unused)) #define PACKED __attribute__((packed)) #define NONNULL(...) __attribute__((nonnull((__VA_ARGS__)))) @@ -2,7 +2,7 @@ * BIRD Coroutines * * (c) 2017 Martin Mares <mj@ucw.cz> - * (c) 2020 Maria Matejka <mq@jmq.cz> + * (c) 2020-2021 Maria Matejka <mq@jmq.cz> * * Can be freely distributed and used under the terms of the GNU GPL. */ @@ -22,5 +22,8 @@ struct coroutine; */ struct coroutine *coro_run(pool *, void (*entry)(void *), void *data); +/* Get self. */ +extern _Thread_local struct coroutine *this_coro; + #endif diff --git a/lib/event.c b/lib/event.c index 33dc00b0..b0abd1d0 100644 --- a/lib/event.c +++ b/lib/event.c @@ -19,8 +19,14 @@ * events in them and explicitly ask to run them. */ +#undef LOCAL_DEBUG + #include "nest/bird.h" #include "lib/event.h" +#include "lib/locking.h" +#include "lib/io-loop.h" + +extern _Thread_local struct coroutine *this_coro; event_list global_event_list; event_list global_work_list; @@ -28,11 +34,16 @@ event_list global_work_list; inline void ev_postpone(event *e) { + event_list *el = e->list; + if (!el) + return; + + ASSERT_DIE(birdloop_inside(el->loop)); + + LOCK_DOMAIN(event, el->lock); if (ev_active(e)) - { - rem_node(&e->n); - e->n.next = NULL; - } + rem_node(&e->n); + UNLOCK_DOMAIN(event, el->lock); } static void @@ -95,40 +106,25 @@ ev_run(event *e) * list @l which can be run by calling ev_run_list(). */ inline void -ev_enqueue(event_list *l, event *e) +ev_send(event_list *l, event *e) { - ev_postpone(e); - add_tail(l, &e->n); -} + DBG("ev_send(%p, %p)\n", l, e); + ASSERT_DIE(e->hook); + ASSERT_DIE(!e->list || (e->list == l) || (e->list->loop == l->loop)); -/** - * ev_schedule - schedule an event - * @e: an event - * - * This function schedules an event by enqueueing it to a system-wide - * event list which is run by the platform dependent code whenever - * appropriate. - */ -void -ev_schedule(event *e) -{ - ev_enqueue(&global_event_list, e); -} + e->list = l; -/** - * ev_schedule_work - schedule a work-event. - * @e: an event - * - * This function schedules an event by enqueueing it to a system-wide work-event - * list which is run by the platform dependent code whenever appropriate. This - * is designated for work-events instead of regular events. They are executed - * less often in order to not clog I/O loop. - */ -void -ev_schedule_work(event *e) -{ - if (!ev_active(e)) - add_tail(&global_work_list, &e->n); + LOCK_DOMAIN(event, l->lock); + if (enlisted(&e->n)) + { + UNLOCK_DOMAIN(event, l->lock); + return; + } + + add_tail(&l->events, &e->n); + UNLOCK_DOMAIN(event, l->lock); + + birdloop_ping(l->loop); } void io_log_event(void *hook, void *data); @@ -142,36 +138,65 @@ void io_log_event(void *hook, void *data); int ev_run_list(event_list *l) { + const _Bool legacy = LEGACY_EVENT_LIST(l); + + if (legacy) + ASSERT_THE_BIRD_LOCKED; + node *n; - list tmp_list; + list tmp_list; init_list(&tmp_list); - add_tail_list(&tmp_list, l); - init_list(l); + + /* Move the event list contents to a local list to avoid executing repeatedly added events */ + LOCK_DOMAIN(event, l->lock); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + UNLOCK_DOMAIN(event, l->lock); + WALK_LIST_FIRST(n, tmp_list) { event *e = SKIP_BACK(event, n, n); - /* This is ugly hack, we want to log just events executed from the main I/O loop */ - if ((l == &global_event_list) || (l == &global_work_list)) + if (legacy) + { + /* The legacy way of event execution */ io_log_event(e->hook, e->data); - - ev_run(e); + ev_postpone(e); + e->hook(e->data); + } + else + { + // io_log_event(e->hook, e->data); /* TODO: add support for event logging in other io loops */ + ASSERT_DIE(e->list == l); + LOCK_DOMAIN(event, l->lock); + rem_node(&e->n); + UNLOCK_DOMAIN(event, l->lock); + e->hook(e->data); + } tmp_flush(); } - return !EMPTY_LIST(*l); + LOCK_DOMAIN(event, l->lock); + int repeat = ! EMPTY_LIST(l->events); + UNLOCK_DOMAIN(event, l->lock); + return repeat; } int ev_run_list_limited(event_list *l, uint limit) { + ASSERT_DIE(LEGACY_EVENT_LIST(l)); + ASSERT_THE_BIRD_LOCKED; + node *n; list tmp_list; + LOCK_DOMAIN(event, l->lock); init_list(&tmp_list); - add_tail_list(&tmp_list, l); - init_list(l); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + UNLOCK_DOMAIN(event, l->lock); WALK_LIST_FIRST(n, tmp_list) { @@ -180,22 +205,24 @@ ev_run_list_limited(event_list *l, uint limit) if (!limit) break; - /* This is ugly hack, we want to log just events executed from the main I/O loop */ - if ((l == &global_event_list) || (l == &global_work_list)) - io_log_event(e->hook, e->data); + io_log_event(e->hook, e->data); ev_run(e); tmp_flush(); limit--; } + LOCK_DOMAIN(event, l->lock); if (!EMPTY_LIST(tmp_list)) { /* Attach new items after the unprocessed old items */ - add_tail_list(&tmp_list, l); - init_list(l); - add_tail_list(l, &tmp_list); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + add_tail_list(&l->events, &tmp_list); } - return !EMPTY_LIST(*l); + int repeat = ! EMPTY_LIST(l->events); + UNLOCK_DOMAIN(event, l->lock); + + return repeat; } diff --git a/lib/event.h b/lib/event.h index 5f3b78d8..6c358f84 100644 --- a/lib/event.h +++ b/lib/event.h @@ -10,33 +10,62 @@ #define _BIRD_EVENT_H_ #include "lib/resource.h" +#include "lib/locking.h" + +#include <stdatomic.h> + +DEFINE_DOMAIN(event); typedef struct event { resource r; void (*hook)(void *); void *data; node n; /* Internal link */ + struct event_list *list; /* List where this event is put in */ } event; -typedef list event_list; +typedef struct event_list { + list events; + pool *pool; + struct birdloop *loop; + DOMAIN(event) lock; +} event_list; extern event_list global_event_list; extern event_list global_work_list; event *ev_new(pool *); void ev_run(event *); -#define ev_init_list(el) init_list(el) -void ev_enqueue(event_list *, event *); -void ev_schedule(event *); -void ev_schedule_work(event *); + +static inline void ev_init_list(event_list *el, struct birdloop *loop, const char *name) +{ + init_list(&el->events); + el->loop = loop; + el->lock = DOMAIN_NEW(event, name); +} + +void ev_send(event_list *, event *); +#define ev_send_loop(l, e) ev_send(birdloop_event_list((l)), (e)) + +#define ev_schedule(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_event_list, (e)); }) +#define ev_schedule_work(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_work_list, (e)); }) + void ev_postpone(event *); int ev_run_list(event_list *); int ev_run_list_limited(event_list *, uint); +#define LEGACY_EVENT_LIST(l) (((l) == &global_event_list) || ((l) == &global_work_list)) + +_Bool birdloop_inside(struct birdloop *loop); + static inline int ev_active(event *e) { - return e->n.next != NULL; + if (e->list == NULL) + return 0; + + ASSERT_DIE(birdloop_inside(e->list->loop)); + return enlisted(&e->n); } static inline event* diff --git a/lib/event_test.c b/lib/event_test.c index 3070327d..5385011a 100644 --- a/lib/event_test.c +++ b/lib/event_test.c @@ -54,7 +54,7 @@ t_ev_run_list(void) int i; olock_init(); - timer_init(); + birdloop_init(); rt_init(); io_init(); if_init(); @@ -81,7 +81,9 @@ main(int argc, char *argv[]) { bt_init(argc, argv); + the_bird_lock(); bt_test_suite(t_ev_run_list, "Schedule and run 3 events in right order."); + the_bird_unlock(); return bt_exit_value(); } diff --git a/lib/io-loop.h b/lib/io-loop.h new file mode 100644 index 00000000..25f1b2a3 --- /dev/null +++ b/lib/io-loop.h @@ -0,0 +1,54 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_IO_LOOP_H_ +#define _BIRD_IO_LOOP_H_ + +#include "nest/bird.h" +#include "lib/lists.h" +#include "lib/locking.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/socket.h" + +void sk_start(sock *s); +void sk_stop(sock *s); +void sk_reloop(sock *s, struct birdloop *loop); + +extern struct birdloop main_birdloop; + +/* Start a new birdloop owned by given pool and domain */ +struct birdloop *birdloop_new(pool *p, uint order, const char *name); + +/* Stop the loop. At the end, the @stopped callback is called unlocked in tail + * position to finish cleanup. Run birdloop_free() from that callback to free + * the loop itself. */ +void birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data); +void birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data); +void birdloop_free(struct birdloop *loop); + +/* Get birdloop's event list */ +event_list *birdloop_event_list(struct birdloop *loop); + +/* Get birdloop's time heap */ +struct timeloop *birdloop_time_loop(struct birdloop *loop); + +/* Enter and exit the birdloop */ +void birdloop_enter(struct birdloop *loop); +void birdloop_leave(struct birdloop *loop); + +_Bool birdloop_inside(struct birdloop *loop); + +void birdloop_mask_wakeups(struct birdloop *loop); +void birdloop_unmask_wakeups(struct birdloop *loop); + +void birdloop_link(struct birdloop *loop); +void birdloop_unlink(struct birdloop *loop); + +void birdloop_ping(struct birdloop *loop); + +void birdloop_init(void); +#endif /* _BIRD_IO_LOOP_H_ */ diff --git a/lib/lists.h b/lib/lists.h index 7e6d5467..86ff59c9 100644 --- a/lib/lists.h +++ b/lib/lists.h @@ -69,6 +69,18 @@ typedef union list { /* In fact two overlayed nodes */ #define EMPTY_LIST(list) (!(list).head->next) +static inline _Bool +enlisted(node *n) +{ + switch ((!!n->next) + (!!n->prev)) + { + case 0: return 0; + case 2: return 1; + case 1: bug("Garbled event list node"); + } + + bug("Maths is broken. And you should see a new heaven and a new earth: for the first heaven and the first earth had been passed away."); +} #ifndef _BIRD_LISTS_C_ #define LIST_INLINE static inline diff --git a/lib/locking.h b/lib/locking.h index eef60154..ab5c06af 100644 --- a/lib/locking.h +++ b/lib/locking.h @@ -14,6 +14,9 @@ struct domain_generic; /* Here define the global lock order; first to last. */ struct lock_order { struct domain_generic *the_bird; + struct domain_generic *proto; + struct domain_generic *rtable; + struct domain_generic *event; }; extern _Thread_local struct lock_order locking_stack; @@ -21,24 +24,40 @@ extern _Thread_local struct domain_generic **last_locked; #define DOMAIN(type) struct domain__##type #define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; } +#define DOMAIN_ORDER(type) OFFSETOF(struct lock_order, type) -#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name, OFFSETOF(struct lock_order, type)) } +#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name, DOMAIN_ORDER(type)) } struct domain_generic *domain_new(const char *name, uint order); +#define DOMAIN_FREE(type, d) domain_free((d).type) +void domain_free(struct domain_generic *); + #define DOMAIN_NULL(type) (DOMAIN(type)) {} #define LOCK_DOMAIN(type, d) do_lock(((d).type), &(locking_stack.type)) #define UNLOCK_DOMAIN(type, d) do_unlock(((d).type), &(locking_stack.type)) +#define DOMAIN_IS_LOCKED(type, d) (((d).type) == (locking_stack.type)) +#define DG_IS_LOCKED(d) ((d) == *(DG_LSP(d))) + /* Internal for locking */ void do_lock(struct domain_generic *dg, struct domain_generic **lsp); void do_unlock(struct domain_generic *dg, struct domain_generic **lsp); +uint dg_order(struct domain_generic *dg); + +#define DG_LSP(d) ((struct domain_generic **) (((void *) &locking_stack) + dg_order(d))) +#define DG_LOCK(d) do_lock(d, DG_LSP(d)) +#define DG_UNLOCK(d) do_unlock(d, DG_LSP(d)) + /* Use with care. To be removed in near future. */ DEFINE_DOMAIN(the_bird); extern DOMAIN(the_bird) the_bird_domain; #define the_bird_lock() LOCK_DOMAIN(the_bird, the_bird_domain) #define the_bird_unlock() UNLOCK_DOMAIN(the_bird, the_bird_domain) +#define the_bird_locked() DOMAIN_IS_LOCKED(the_bird, the_bird_domain) + +#define ASSERT_THE_BIRD_LOCKED ({ if (!the_bird_locked()) bug("The BIRD lock must be locked here: %s:%d", __FILE__, __LINE__); }) #endif diff --git a/lib/resource.c b/lib/resource.c index c31d9889..a33fd214 100644 --- a/lib/resource.c +++ b/lib/resource.c @@ -29,12 +29,6 @@ * is freed upon shutdown of the module. */ -struct pool { - resource r; - list inside; - const char *name; -}; - static void pool_dump(resource *); static void pool_free(resource *); static resource *pool_lookup(resource *, unsigned long); diff --git a/lib/resource.h b/lib/resource.h index 8bb264b1..5ad011ec 100644 --- a/lib/resource.h +++ b/lib/resource.h @@ -40,7 +40,12 @@ struct resclass { /* Generic resource manipulation */ -typedef struct pool pool; +typedef struct pool { + resource r; + list inside; + const char *name; +} pool; + void resource_init(void); pool *rp_new(pool *, const char *); /* Create new pool */ diff --git a/lib/socket.h b/lib/socket.h index 0b6ac589..5c69482e 100644 --- a/lib/socket.h +++ b/lib/socket.h @@ -12,6 +12,7 @@ #include <errno.h> #include "lib/resource.h" +#include "lib/event.h" #ifdef HAVE_LIBSSH #define LIBSSH_LEGACY_0_4 #include <libssh/libssh.h> @@ -79,6 +80,7 @@ typedef struct birdsock { const char *password; /* Password for MD5 authentication */ const char *err; /* Error message */ struct ssh_sock *ssh; /* Used in SK_SSH */ + struct event reloop; /* Reloop event */ } sock; sock *sock_new(pool *); /* Allocate new socket */ @@ -129,6 +131,7 @@ extern int sk_priority_control; /* Suggested priority for control traffic, shou #define SKF_TRUNCATED 0x200 /* Received packet was truncated, set by IO layer */ #define SKF_HDRINCL 0x400 /* Used internally */ #define SKF_PKTINFO 0x800 /* Used internally */ +#define SKF_PASSIVE_THREAD 0x1000 /* Child sockets used in thread, do not add to main loop */ /* * Socket types SA SP DA DP IF TTL SendTo (?=may, -=must not, *=must) diff --git a/lib/timer.c b/lib/timer.c index 295eeaef..c1241ba7 100644 --- a/lib/timer.c +++ b/lib/timer.c @@ -37,15 +37,8 @@ #include "lib/resource.h" #include "lib/timer.h" - -struct timeloop main_timeloop; - - #include <pthread.h> -/* Data accessed and modified from proto/bfd/io.c */ -_Thread_local struct timeloop *local_timeloop; - _Atomic btime last_time; _Atomic btime real_time; @@ -76,7 +69,7 @@ tm_dump(resource *r) if (t->recurrent) debug("recur %d, ", t->recurrent); if (t->expires) - debug("expires in %d ms)\n", (t->expires - current_time()) TO_MS); + debug("in loop %p expires in %d ms)\n", t->loop, (t->expires - current_time()) TO_MS); else debug("inactive)\n"); } @@ -99,8 +92,8 @@ tm_new(pool *p) return t; } -void -tm_set(timer *t, btime when) +static void +tm_set_in_tl(timer *t, btime when, struct timeloop *local_timeloop) { uint tc = timers_count(local_timeloop); @@ -122,17 +115,17 @@ tm_set(timer *t, btime when) HEAP_DECREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); } -#ifdef CONFIG_BFD - /* Hack to notify BFD loops */ - if ((local_timeloop != &main_timeloop) && (t->index == 1)) - wakeup_kick_current(); -#endif + t->loop = local_timeloop; + + if ((t->index == 1) && (local_timeloop->coro != this_coro)) + birdloop_ping(local_timeloop->loop); } void -tm_start(timer *t, btime after) +tm_set_in(timer *t, btime when, struct birdloop *loop) { - tm_set(t, current_time() + MAX(after, 0)); + ASSERT_DIE(birdloop_inside(loop)); + tm_set_in_tl(t, when, birdloop_time_loop(loop)); } void @@ -141,18 +134,23 @@ tm_stop(timer *t) if (!t->expires) return; - uint tc = timers_count(local_timeloop); + TLOCK_TIMER_ASSERT(t->loop); - HEAP_DELETE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); - BUFFER_POP(local_timeloop->timers); + uint tc = timers_count(t->loop); + + HEAP_DELETE(t->loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + BUFFER_POP(t->loop->timers); t->index = -1; t->expires = 0; + t->loop = NULL; } void timers_init(struct timeloop *loop, pool *p) { + TLOCK_TIMER_ASSERT(loop); + BUFFER_INIT(loop->timers, p, 4); BUFFER_PUSH(loop->timers) = NULL; } @@ -160,8 +158,10 @@ timers_init(struct timeloop *loop, pool *p) void io_log_event(void *hook, void *data); void -timers_fire(struct timeloop *loop) +timers_fire(struct timeloop *loop, int io_log) { + TLOCK_TIMER_ASSERT(loop); + btime base_time; timer *t; @@ -183,13 +183,13 @@ timers_fire(struct timeloop *loop) if (t->randomize) when += random() % (t->randomize + 1); - tm_set(t, when); + tm_set_in_tl(t, when, loop); } else tm_stop(t); /* This is ugly hack, we want to log just timers executed from the main I/O loop */ - if (loop == &main_timeloop) + if (io_log) io_log_event(t->hook, t->data); t->hook(t); @@ -197,13 +197,6 @@ timers_fire(struct timeloop *loop) } } -void -timer_init(void) -{ - timers_init(&main_timeloop, &root_pool); - local_timeloop = &main_timeloop; -} - /** * tm_parse_time - parse a date and time diff --git a/lib/timer.h b/lib/timer.h index b201b8c8..04544ace 100644 --- a/lib/timer.h +++ b/lib/timer.h @@ -12,6 +12,8 @@ #include "nest/bird.h" #include "lib/buffer.h" +#include "lib/io-loop.h" +#include "lib/locking.h" #include "lib/resource.h" #include <stdatomic.h> @@ -29,22 +31,27 @@ typedef struct timer uint randomize; /* Amount of randomization */ uint recurrent; /* Timer recurrence */ + struct timeloop *loop; /* Loop where the timer is active */ + int index; } timer; struct timeloop { BUFFER_(timer *) timers; + struct domain_generic *domain; + struct birdloop *loop; + struct coroutine *coro; }; +#define TLOCK_TIMER_ASSERT(loop) ASSERT_DIE((loop)->domain && DG_IS_LOCKED((loop)->domain)) +#define TLOCK_LOCAL_ASSERT(loop) ASSERT_DIE(!(loop)->domain || DG_IS_LOCKED((loop)->domain)) + static inline uint timers_count(struct timeloop *loop) -{ return loop->timers.used - 1; } +{ TLOCK_TIMER_ASSERT(loop); return loop->timers.used - 1; } static inline timer *timers_first(struct timeloop *loop) -{ return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } - -extern struct timeloop main_timeloop; -extern _Thread_local struct timeloop *local_timeloop; +{ TLOCK_TIMER_ASSERT(loop); return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } #define current_time() atomic_load_explicit(&last_time, memory_order_acquire) #define current_real_time() atomic_load_explicit(&real_time, memory_order_acquire) @@ -54,10 +61,13 @@ extern _Thread_local struct timeloop *local_timeloop; extern btime boot_time; timer *tm_new(pool *p); -void tm_set(timer *t, btime when); -void tm_start(timer *t, btime after); +#define tm_set(t, when) tm_set_in((t), (when), &main_birdloop) +#define tm_start(t, after) tm_start_in((t), (after), &main_birdloop) void tm_stop(timer *t); +void tm_set_in(timer *t, btime when, struct birdloop *loop); +#define tm_start_in(t, after, loop) tm_set_in((t), (current_time() + MAX_((after), 0)), loop) + static inline int tm_active(timer *t) { @@ -101,9 +111,7 @@ void times_update(void); /* For I/O loop */ void timers_init(struct timeloop *loop, pool *p); -void timers_fire(struct timeloop *loop); - -void timer_init(void); +void timers_fire(struct timeloop *loop, int io_log); struct timeformat { diff --git a/nest/config.Y b/nest/config.Y index e894f7c4..4c758ea3 100644 --- a/nest/config.Y +++ b/nest/config.Y @@ -472,6 +472,7 @@ proto: dev_proto '}' ; dev_proto_start: proto_start DIRECT { this_proto = proto_config_new(&proto_device, $1); init_list(&DIRECT_CFG->iface_list); + this_proto->late_if_feed = 1; } ; diff --git a/nest/proto.c b/nest/proto.c index 3a80ab0e..e9bced3b 100644 --- a/nest/proto.c +++ b/nest/proto.c @@ -15,6 +15,7 @@ #include "lib/event.h" #include "lib/timer.h" #include "lib/string.h" +#include "lib/coro.h" #include "conf/conf.h" #include "nest/rt.h" #include "nest/iface.h" @@ -57,7 +58,28 @@ static void channel_feed_end(struct channel *c); static void channel_export_stopped(struct rt_export_request *req); static inline int proto_is_done(struct proto *p) -{ return (p->proto_state == PS_DOWN) && (p->active_channels == 0); } +{ return (p->proto_state == PS_DOWN) && proto_is_inactive(p); } + +static inline event_list *proto_event_list(struct proto *p) +{ return p->loop == &main_birdloop ? &global_event_list : birdloop_event_list(p->loop); } + +static inline event_list *proto_work_list(struct proto *p) +{ return p->loop == &main_birdloop ? &global_work_list : birdloop_event_list(p->loop); } + +static inline void proto_send_event(struct proto *p) +{ ev_send(proto_event_list(p), p->event); } + +#define PROTO_ENTER_FROM_MAIN(p) ({ \ + ASSERT_DIE(birdloop_inside(&main_birdloop)); \ + struct birdloop *_loop = (p)->loop; \ + if (_loop != &main_birdloop) birdloop_enter(_loop); \ + _loop; \ + }) + +#define PROTO_LEAVE_FROM_MAIN(loop) ({ if (loop != &main_birdloop) birdloop_leave(loop); }) + +#define PROTO_LOCKED_FROM_MAIN(p) for (struct birdloop *_proto_loop = PROTO_ENTER_FROM_MAIN(p); _proto_loop; PROTO_LEAVE_FROM_MAIN(_proto_loop), (_proto_loop = NULL)) + static inline int channel_is_active(struct channel *c) { return (c->channel_state != CS_DOWN); } @@ -467,6 +489,7 @@ channel_start_export(struct channel *c) c->out_req = (struct rt_export_request) { .name = rn, + .list = proto_work_list(c->proto), .addr = c->out_subprefix, .addr_mode = c->out_subprefix ? TE_ADDR_IN : TE_ADDR_NONE, .trace_routes = c->debug | c->proto->debug, @@ -513,7 +536,7 @@ channel_check_stopped(struct channel *c) return; channel_set_state(c, CS_DOWN); - ev_schedule(c->proto->event); + proto_send_event(c->proto); break; case CS_PAUSE: @@ -633,6 +656,7 @@ channel_setup_in_table(struct channel *c) { c->reload_req = (struct rt_export_request) { .name = mb_sprintf(c->proto->pool, "%s.%s.import", c->proto->name, c->name), + .list = proto_work_list(c->proto), .trace_routes = c->debug | c->proto->debug, .export_bulk = channel_reload_export_bulk, .dump_req = channel_reload_dump_req, @@ -718,7 +742,7 @@ channel_do_down(struct channel *c) /* Schedule protocol shutddown */ if (proto_is_done(c->proto)) - ev_schedule(c->proto->event); + proto_send_event(c->proto); } void @@ -1029,17 +1053,35 @@ proto_configure_channel(struct proto *p, struct channel **pc, struct channel_con return 1; } +static void +proto_cleanup(struct proto *p) +{ + rfree(p->pool); + p->pool = NULL; + + p->active = 0; + proto_log_state_change(p); + proto_rethink_goal(p); +} static void -proto_event(void *ptr) +proto_loop_stopped(void *ptr) { struct proto *p = ptr; - if (p->do_start) - { - if_feed_baby(p); - p->do_start = 0; - } + birdloop_enter(&main_birdloop); + + p->loop = &main_birdloop; + p->event->list = NULL; + proto_cleanup(p); + + birdloop_leave(&main_birdloop); +} + +static void +proto_event(void *ptr) +{ + struct proto *p = ptr; if (p->do_stop) { @@ -1049,14 +1091,10 @@ proto_event(void *ptr) } if (proto_is_done(p)) - { - rfree(p->pool); - p->pool = NULL; - - p->active = 0; - proto_log_state_change(p); - proto_rethink_goal(p); - } + if (p->loop != &main_birdloop) + birdloop_stop_self(p->loop, proto_loop_stopped, p); + else + proto_cleanup(p); } @@ -1097,6 +1135,7 @@ proto_init(struct proto_config *c, node *n) struct protocol *pr = c->protocol; struct proto *p = pr->init(c); + p->loop = &main_birdloop; p->proto_state = PS_DOWN; p->last_state_change = current_time(); p->vrf = c->vrf; @@ -1113,11 +1152,21 @@ proto_init(struct proto_config *c, node *n) static void proto_start(struct proto *p) { - /* Here we cannot use p->cf->name since it won't survive reconfiguration */ - p->pool = rp_new(proto_pool, p->proto->name); + DBG("Kicking %s up\n", p->name); + PD(p, "Starting"); + + p->pool = rp_newf(proto_pool, "Protocol %s", p->cf->name); if (graceful_restart_state == GRS_INIT) p->gr_recovery = 1; + + if (p->cf->loop_order != DOMAIN_ORDER(the_bird)) + p->loop = birdloop_new(p->pool, p->cf->loop_order, p->pool->name); + + p->event->list = proto_event_list(p); + + PROTO_LOCKED_FROM_MAIN(p) + proto_notify_state(p, (p->proto->start ? p->proto->start(p) : PS_UP)); } @@ -1153,6 +1202,7 @@ proto_config_new(struct protocol *pr, int class) cf->class = class; cf->debug = new_config->proto_default_debug; cf->mrtdump = new_config->proto_default_mrtdump; + cf->loop_order = DOMAIN_ORDER(the_bird); init_list(&cf->channels); @@ -1442,11 +1492,20 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty } static void -proto_rethink_goal(struct proto *p) +proto_shutdown(struct proto *p) { - struct protocol *q; - byte goal; + if (p->proto_state == PS_START || p->proto_state == PS_UP) + { + /* Going down */ + DBG("Kicking %s down\n", p->name); + PD(p, "Shutting down"); + proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_DOWN)); + } +} +static void +proto_rethink_goal(struct proto *p) +{ if (p->reconfiguring && !p->active) { struct proto_config *nc = p->cf_new; @@ -1466,32 +1525,12 @@ proto_rethink_goal(struct proto *p) /* Determine what state we want to reach */ if (p->disabled || p->reconfiguring) - goal = PS_DOWN; - else - goal = PS_UP; - - q = p->proto; - if (goal == PS_UP) - { - if (!p->active) - { - /* Going up */ - DBG("Kicking %s up\n", p->name); - PD(p, "Starting"); - proto_start(p); - proto_notify_state(p, (q->start ? q->start(p) : PS_UP)); - } - } - else { - if (p->proto_state == PS_START || p->proto_state == PS_UP) - { - /* Going down */ - DBG("Kicking %s down\n", p->name); - PD(p, "Shutting down"); - proto_notify_state(p, (q->shutdown ? q->shutdown(p) : PS_DOWN)); - } + PROTO_LOCKED_FROM_MAIN(p) + proto_shutdown(p); } + else if (!p->active) + proto_start(p); } struct proto * @@ -1696,7 +1735,7 @@ protos_dump_all(void) #define DPF(x) (p->x ? " " #x : "") debug(" protocol %s (%p) state %s with %d active channels flags: %s%s%s%s%s\n", p->name, p, p_states[p->proto_state], p->active_channels, - DPF(disabled), DPF(active), DPF(do_start), DPF(do_stop), DPF(reconfiguring)); + DPF(disabled), DPF(active), DPF(do_stop), DPF(reconfiguring)); #undef DPF struct channel *c; @@ -1932,8 +1971,8 @@ static inline void proto_do_start(struct proto *p) { p->active = 1; - p->do_start = 1; - ev_schedule(p->event); + if (!p->cf->late_if_feed) + if_feed_baby(p); } static void @@ -1946,6 +1985,9 @@ proto_do_up(struct proto *p) } proto_start_channels(p); + + if (p->cf->late_if_feed) + if_feed_baby(p); } static inline void @@ -1960,9 +2002,6 @@ proto_do_stop(struct proto *p) p->down_sched = 0; p->gr_recovery = 0; - p->do_stop = 1; - ev_schedule(p->event); - if (p->main_source) { rt_unlock_source(p->main_source); @@ -1970,6 +2009,9 @@ proto_do_stop(struct proto *p) } proto_stop_channels(p); + + p->do_stop = 1; + proto_send_event(p); } static void @@ -1980,7 +2022,7 @@ proto_do_down(struct proto *p) /* Shutdown is finished in the protocol event */ if (proto_is_done(p)) - ev_schedule(p->event); + proto_send_event(p); } @@ -2219,7 +2261,7 @@ proto_cmd_disable(struct proto *p, uintptr_t arg, int cnt UNUSED) p->disabled = 1; p->down_code = PDC_CMD_DISABLE; proto_set_message(p, (char *) arg, -1); - proto_rethink_goal(p); + proto_shutdown(p); cli_msg(-9, "%s: disabled", p->name); } @@ -2252,9 +2294,9 @@ proto_cmd_restart(struct proto *p, uintptr_t arg, int cnt UNUSED) p->disabled = 1; p->down_code = PDC_CMD_RESTART; proto_set_message(p, (char *) arg, -1); - proto_rethink_goal(p); + proto_shutdown(p); p->disabled = 0; - proto_rethink_goal(p); + /* After the protocol shuts down, proto_rethink_goal() is run from proto_event. */ cli_msg(-12, "%s: restarted", p->name); } @@ -2329,7 +2371,9 @@ proto_apply_cmd_symbol(const struct symbol *s, void (* cmd)(struct proto *, uint if (s->proto->proto) { - cmd(s->proto->proto, arg, 0); + struct proto *p = s->proto->proto; + PROTO_LOCKED_FROM_MAIN(p) + cmd(p, arg, 0); cli_msg(0, ""); } else @@ -2344,7 +2388,8 @@ proto_apply_cmd_patt(const char *patt, void (* cmd)(struct proto *, uintptr_t, i WALK_LIST(p, proto_list) if (!patt || patmatch(patt, p->name)) - cmd(p, arg, cnt++); + PROTO_LOCKED_FROM_MAIN(p) + cmd(p, arg, cnt++); if (!cnt) cli_msg(8003, "No protocols match"); diff --git a/nest/protocol.h b/nest/protocol.h index 026d42ab..d6224015 100644 --- a/nest/protocol.h +++ b/nest/protocol.h @@ -102,8 +102,10 @@ struct proto_config { u8 net_type; /* Protocol network type (NET_*), 0 for undefined */ u8 disabled; /* Protocol enabled/disabled by default */ u8 vrf_set; /* Related VRF instance (below) is defined */ + u8 late_if_feed; /* Delay interface feed after channels are up */ u32 debug, mrtdump; /* Debugging bitfields, both use D_* constants */ u32 router_id; /* Protocol specific router ID */ + uint loop_order; /* Launch a birdloop on this locking level; use DOMAIN_ORDER(the_bird) for mainloop */ list channels; /* List of channel configs (struct channel_config) */ struct iface *vrf; /* Related VRF instance, NULL if global */ @@ -121,6 +123,7 @@ struct proto { struct proto_config *cf_new; /* Configuration we want to switch to after shutdown (NULL=delete) */ pool *pool; /* Pool containing local objects */ event *event; /* Protocol event */ + struct birdloop *loop; /* BIRDloop running this protocol */ list channels; /* List of channels to rtables (struct channel) */ struct channel *main_channel; /* Primary channel */ @@ -131,12 +134,12 @@ struct proto { u32 debug; /* Debugging flags */ u32 mrtdump; /* MRTDump flags */ uint active_channels; /* Number of active channels */ + uint active_coroutines; /* Number of active coroutines */ byte net_type; /* Protocol network type (NET_*), 0 for undefined */ byte disabled; /* Manually disabled */ byte vrf_set; /* Related VRF instance (above) is defined */ byte proto_state; /* Protocol state machine (PS_*, see below) */ byte active; /* From PS_START to cleanup after PS_STOP */ - byte do_start; /* Start actions are scheduled */ byte do_stop; /* Stop actions are scheduled */ byte reconfiguring; /* We're shutting down due to reconfiguration */ byte gr_recovery; /* Protocol should participate in graceful restart recovery */ @@ -338,6 +341,8 @@ void proto_notify_state(struct proto *p, unsigned state); * as a result of received ROUTE-REFRESH request). */ +static inline int proto_is_inactive(struct proto *p) +{ return (p->active_channels == 0) && (p->active_coroutines == 0); } /* diff --git a/nest/rt-show.c b/nest/rt-show.c index 6dfb85f6..b784bf83 100644 --- a/nest/rt-show.c +++ b/nest/rt-show.c @@ -285,6 +285,7 @@ rt_show_cont(struct rt_show_data *d) d->req = (struct rt_export_request) { .addr = d->addr, .name = "CLI Show Route", + .list = &global_work_list, .export_bulk = rt_show_net_export_bulk, .dump_req = rt_show_dump_req, .log_state_change = rt_show_log_state_change, diff --git a/nest/rt-table.c b/nest/rt-table.c index 85a6faf7..ca092678 100644 --- a/nest/rt-table.c +++ b/nest/rt-table.c @@ -1415,6 +1415,12 @@ rt_next_export(struct rt_export_hook *hook, struct rt_exporter *tab) return tab->first; } +static inline void +rt_send_export_event(struct rt_export_hook *hook) +{ + ev_send(hook->req->list, hook->event); +} + static void rt_announce_exports(timer *tm) { @@ -1426,7 +1432,7 @@ rt_announce_exports(timer *tm) if (atomic_load_explicit(&c->export_state, memory_order_acquire) != TES_READY) continue; - ev_schedule_work(c->event); + rt_send_export_event(c); } } @@ -1479,7 +1485,7 @@ rt_export_hook(void *_data) rte_update_unlock(); } - ev_schedule_work(c->event); + rt_send_export_event(c); } @@ -2097,7 +2103,7 @@ rt_request_export(struct rt_exporter *re, struct rt_export_request *req) /* Regular export */ rt_set_export_state(hook, TES_FEEDING); - ev_schedule_work(hook->event); + rt_send_export_event(hook); } static void @@ -2146,7 +2152,7 @@ rt_stop_export(struct rt_export_request *req, void (*stopped)(struct rt_export_r rt_set_export_state(hook, TES_STOP); /* Run the stopped event */ - ev_schedule(hook->event); + rt_send_export_event(hook); } /** @@ -3727,7 +3733,7 @@ rt_feed_done(struct rt_export_hook *c) rt_set_export_state(c, TES_READY); - ev_schedule_work(c->event); + rt_send_export_event(c); } /** @@ -3756,7 +3762,7 @@ rt_feed_by_fib(void *data) if (max_feed <= 0) { FIB_ITERATE_PUT(fit); - ev_schedule_work(c->event); + rt_send_export_event(c); return; } @@ -17,6 +17,7 @@ #include "lib/type.h" #include "lib/fib.h" #include "lib/route.h" +#include "lib/event.h" #include <stdatomic.h> @@ -238,6 +239,8 @@ struct rt_export_request { u8 trace_routes; u8 addr_mode; /* Network prefilter mode (TE_ADDR_*) */ + event_list *list; /* Where to schedule export events */ + /* There are two methods of export. You can either request feeding every single change * or feeding the whole route feed. In case of regular export, &export_one is preferred. * Anyway, when feeding, &export_bulk is preferred, falling back to &export_one. diff --git a/proto/bfd/Makefile b/proto/bfd/Makefile index dbdc0a09..11d639d7 100644 --- a/proto/bfd/Makefile +++ b/proto/bfd/Makefile @@ -1,4 +1,4 @@ -src := bfd.c io.c packets.c +src := bfd.c packets.c obj := $(src-o-files) $(all-daemon) $(cf-local) diff --git a/proto/bfd/bfd.c b/proto/bfd/bfd.c index 871ecf69..e7d27f74 100644 --- a/proto/bfd/bfd.c +++ b/proto/bfd/bfd.c @@ -113,8 +113,16 @@ #define HASH_IP_EQ(a1,n1,a2,n2) ipa_equal(a1, a2) && n1 == n2 #define HASH_IP_FN(a,n) ipa_hash(a) ^ u32_hash(n) -static list STATIC_LIST_INIT(bfd_proto_list); -static list STATIC_LIST_INIT(bfd_wait_list); +DEFINE_DOMAIN(rtable); +#define BFD_LOCK LOCK_DOMAIN(rtable, bfd_global.lock) +#define BFD_UNLOCK UNLOCK_DOMAIN(rtable, bfd_global.lock) + +static struct { + DOMAIN(rtable) lock; + list wait_list; + list pickup_list; + list proto_list; +} bfd_global; const char *bfd_state_names[] = { "AdminDown", "Down", "Init", "Up" }; @@ -188,7 +196,7 @@ bfd_session_update_tx_interval(struct bfd_session *s) return; /* Set timer relative to last tx_timer event */ - tm_set(s->tx_timer, s->last_tx + tx_int_l); + tm_set_in(s->tx_timer, s->last_tx + tx_int_l, s->ifa->bfd->p.loop); } static void @@ -202,7 +210,7 @@ bfd_session_update_detection_time(struct bfd_session *s, int kick) if (!s->last_rx) return; - tm_set(s->hold_timer, s->last_rx + timeout); + tm_set_in(s->hold_timer, s->last_rx + timeout, s->ifa->bfd->p.loop); } static void @@ -226,7 +234,7 @@ bfd_session_control_tx_timer(struct bfd_session *s, int reset) if (reset || !tm_active(s->tx_timer)) { s->last_tx = 0; - tm_start(s->tx_timer, 0); + tm_start_in(s->tx_timer, 0, s->ifa->bfd->p.loop); } return; @@ -419,7 +427,7 @@ bfd_get_free_id(struct bfd_proto *p) static struct bfd_session * bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *iface, struct bfd_options *opts) { - birdloop_enter(p->loop); + ASSERT_DIE(birdloop_inside(p->p.loop)); struct bfd_iface *ifa = bfd_get_iface(p, local, iface); @@ -454,8 +462,6 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface * TRACE(D_EVENTS, "Session to %I added", s->addr); - birdloop_leave(p->loop); - return s; } @@ -463,38 +469,34 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface * static void bfd_open_session(struct bfd_proto *p, struct bfd_session *s, ip_addr local, struct iface *ifa) { - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); s->opened = 1; bfd_session_control_tx_timer(s); - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); } static void bfd_close_session(struct bfd_proto *p, struct bfd_session *s) { - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); s->opened = 0; bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_PATH_DOWN); bfd_session_control_tx_timer(s); - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); } */ static void -bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) +bfd_remove_session_locked(struct bfd_proto *p, struct bfd_session *s) { - ip_addr ip = s->addr; - /* Caller should ensure that request list is empty */ - birdloop_enter(p->loop); - /* Remove session from notify list if scheduled for notification */ /* No need for bfd_lock_sessions(), we are already protected by birdloop_enter() */ if (NODE_VALID(&s->n)) @@ -508,11 +510,17 @@ bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) HASH_REMOVE(p->session_hash_id, HASH_ID, s); HASH_REMOVE(p->session_hash_ip, HASH_IP, s); - sl_free(s); + TRACE(D_EVENTS, "Session to %I removed", s->addr); - TRACE(D_EVENTS, "Session to %I removed", ip); + sl_free(s); +} - birdloop_leave(p->loop); +static void +bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) +{ + birdloop_enter(p->p.loop); + bfd_remove_session_locked(p, s); + birdloop_leave(p->p.loop); } static void @@ -521,7 +529,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s) if (EMPTY_LIST(s->request_list)) return; - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); struct bfd_request *req = SKIP_BACK(struct bfd_request, n, HEAD(s->request_list)); s->cf = bfd_merge_options(s->ifa->cf, &req->opts); @@ -534,7 +542,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s) bfd_session_control_tx_timer(s, 0); - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); TRACE(D_EVENTS, "Session to %I reconfigured", s->addr); } @@ -627,9 +635,9 @@ bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_con (new->passive != old->passive); /* This should be probably changed to not access ifa->cf from the BFD thread */ - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); ifa->cf = new; - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); } @@ -690,41 +698,68 @@ bfd_add_request(struct bfd_proto *p, struct bfd_request *req) } static void -bfd_submit_request(struct bfd_request *req) +bfd_pickup_requests(void *_data UNUSED) { node *n; + WALK_LIST(n, bfd_global.proto_list) + { + struct bfd_proto *p = SKIP_BACK(struct bfd_proto, bfd_node, n); + birdloop_enter(p->p.loop); + BFD_LOCK; - WALK_LIST(n, bfd_proto_list) - if (bfd_add_request(SKIP_BACK(struct bfd_proto, bfd_node, n), req)) - return; + node *rn, *rnxt; + WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list) + bfd_add_request(p, SKIP_BACK(struct bfd_request, n, rn)); - rem_node(&req->n); - add_tail(&bfd_wait_list, &req->n); - req->session = NULL; - bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0); + BFD_UNLOCK; + birdloop_leave(p->p.loop); + } + + BFD_LOCK; + node *rn, *rnxt; + WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list) + { + rem_node(rn); + add_tail(&bfd_global.wait_list, rn); + bfd_request_notify(SKIP_BACK(struct bfd_request, n, rn), BFD_STATE_ADMIN_DOWN, 0); + } + BFD_UNLOCK; } +static event bfd_pickup_event = { .hook = bfd_pickup_requests }; + static void bfd_take_requests(struct bfd_proto *p) { node *n, *nn; - - WALK_LIST_DELSAFE(n, nn, bfd_wait_list) + BFD_LOCK; + WALK_LIST_DELSAFE(n, nn, bfd_global.wait_list) bfd_add_request(p, SKIP_BACK(struct bfd_request, n, n)); + BFD_UNLOCK; } static void bfd_drop_requests(struct bfd_proto *p) { node *n; - - HASH_WALK(p->session_hash_id, next_id, s) + BFD_LOCK; + HASH_WALK_DELSAFE(p->session_hash_id, next_id, s) { - /* We assume that p is not in bfd_proto_list */ WALK_LIST_FIRST(n, s->request_list) - bfd_submit_request(SKIP_BACK(struct bfd_request, n, n)); + { + struct bfd_request *req = SKIP_BACK(struct bfd_request, n, n); + rem_node(&req->n); + add_tail(&bfd_global.pickup_list, &req->n); + req->session = NULL; + bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0); + } + + ev_send(&global_event_list, &bfd_pickup_event); + + bfd_remove_session_locked(p, s); } HASH_WALK_END; + BFD_UNLOCK; } static struct resclass bfd_request_class; @@ -737,9 +772,6 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local, { struct bfd_request *req = ralloc(p, &bfd_request_class); - /* Hack: self-link req->n, we will call rem_node() on it */ - req->n.prev = req->n.next = &req->n; - req->addr = addr; req->local = local; req->iface = iface; @@ -748,11 +780,16 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local, if (opts) req->opts = *opts; - bfd_submit_request(req); - req->hook = hook; req->data = data; + req->session = NULL; + + BFD_LOCK; + add_tail(&bfd_global.pickup_list, &req->n); + ev_send(&global_event_list, &bfd_pickup_event); + BFD_UNLOCK; + return req; } @@ -1023,10 +1060,10 @@ bfd_start(struct proto *P) struct bfd_proto *p = (struct bfd_proto *) P; struct bfd_config *cf = (struct bfd_config *) (P->cf); - p->loop = birdloop_new(); - p->tpool = rp_new(NULL, "BFD thread root"); pthread_spin_init(&p->lock, PTHREAD_PROCESS_PRIVATE); + p->tpool = rp_new(P->pool, "BFD loop pool"); + p->session_slab = sl_new(P->pool, sizeof(struct bfd_session)); HASH_INIT(p->session_hash_id, P->pool, 8); HASH_INIT(p->session_hash_ip, P->pool, 8); @@ -1036,9 +1073,7 @@ bfd_start(struct proto *P) init_list(&p->notify_list); bfd_notify_init(p); - add_tail(&bfd_proto_list, &p->bfd_node); - - birdloop_enter(p->loop); + add_tail(&bfd_global.proto_list, &p->bfd_node); if (!cf->strict_bind) { @@ -1055,42 +1090,33 @@ bfd_start(struct proto *P) p->rx6_m = bfd_open_rx_sk(p, 1, SK_IPV6); } - birdloop_leave(p->loop); - bfd_take_requests(p); struct bfd_neighbor *n; WALK_LIST(n, cf->neigh_list) bfd_start_neighbor(p, n); - birdloop_start(p->loop); - return PS_UP; } - static int bfd_shutdown(struct proto *P) { struct bfd_proto *p = (struct bfd_proto *) P; - struct bfd_config *cf = (struct bfd_config *) (P->cf); + struct bfd_config *cf = (struct bfd_config *) (p->p.cf); rem_node(&p->bfd_node); - birdloop_stop(p->loop); - - struct bfd_neighbor *n; - WALK_LIST(n, cf->neigh_list) - bfd_stop_neighbor(p, n); + struct bfd_neighbor *bn; + WALK_LIST(bn, cf->neigh_list) + bfd_stop_neighbor(p, bn); bfd_drop_requests(p); - /* FIXME: This is hack */ - birdloop_enter(p->loop); - rfree(p->tpool); - birdloop_leave(p->loop); - - birdloop_free(p->loop); + if (p->rx4_1) sk_stop(p->rx4_1); + if (p->rx4_m) sk_stop(p->rx4_m); + if (p->rx6_1) sk_stop(p->rx6_1); + if (p->rx6_m) sk_stop(p->rx6_m); return PS_DOWN; } @@ -1111,7 +1137,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c) (new->strict_bind != old->strict_bind)) return 0; - birdloop_mask_wakeups(p->loop); + birdloop_mask_wakeups(p->p.loop); WALK_LIST(ifa, p->iface_list) bfd_reconfigure_iface(p, ifa, new); @@ -1125,7 +1151,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c) bfd_reconfigure_neighbors(p, new); - birdloop_unmask_wakeups(p->loop); + birdloop_unmask_wakeups(p->p.loop); return 1; } @@ -1196,4 +1222,9 @@ void bfd_build(void) { proto_build(&proto_bfd); + + bfd_global.lock = DOMAIN_NEW(rtable, "BFD Global"); + init_list(&bfd_global.wait_list); + init_list(&bfd_global.pickup_list); + init_list(&bfd_global.proto_list); } diff --git a/proto/bfd/bfd.h b/proto/bfd/bfd.h index 60b7916c..b9afaf92 100644 --- a/proto/bfd/bfd.h +++ b/proto/bfd/bfd.h @@ -17,12 +17,12 @@ #include "nest/password.h" #include "conf/conf.h" #include "lib/hash.h" +#include "lib/io-loop.h" #include "lib/resource.h" #include "lib/socket.h" #include "lib/string.h" #include "nest/bfd.h" -#include "io.h" #define BFD_CONTROL_PORT 3784 @@ -88,9 +88,11 @@ struct bfd_neighbor struct bfd_proto { struct proto p; - struct birdloop *loop; - pool *tpool; + pthread_spinlock_t lock; + + pool *tpool; + node bfd_node; slab *session_slab; diff --git a/proto/bfd/config.Y b/proto/bfd/config.Y index 70461872..0d6e33fa 100644 --- a/proto/bfd/config.Y +++ b/proto/bfd/config.Y @@ -37,6 +37,7 @@ proto: bfd_proto ; bfd_proto_start: proto_start BFD { this_proto = proto_config_new(&proto_bfd, $1); + this_proto->loop_order = DOMAIN_ORDER(proto); init_list(&BFD_CFG->patt_list); init_list(&BFD_CFG->neigh_list); BFD_CFG->accept_ipv4 = BFD_CFG->accept_ipv6 = 1; diff --git a/proto/bfd/io.h b/proto/bfd/io.h deleted file mode 100644 index ec706e9a..00000000 --- a/proto/bfd/io.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * BIRD -- I/O and event loop - * - * Can be freely distributed and used under the terms of the GNU GPL. - */ - -#ifndef _BIRD_BFD_IO_H_ -#define _BIRD_BFD_IO_H_ - -#include "nest/bird.h" -#include "lib/lists.h" -#include "lib/resource.h" -#include "lib/event.h" -#include "lib/timer.h" -#include "lib/socket.h" - - -void ev2_schedule(event *e); - -void sk_start(sock *s); -void sk_stop(sock *s); - -struct birdloop *birdloop_new(void); -void birdloop_start(struct birdloop *loop); -void birdloop_stop(struct birdloop *loop); -void birdloop_free(struct birdloop *loop); - -void birdloop_enter(struct birdloop *loop); -void birdloop_leave(struct birdloop *loop); -void birdloop_mask_wakeups(struct birdloop *loop); -void birdloop_unmask_wakeups(struct birdloop *loop); - - -#endif /* _BIRD_BFD_IO_H_ */ diff --git a/proto/bfd/packets.c b/proto/bfd/packets.c index 5f10734c..6f0b4eaf 100644 --- a/proto/bfd/packets.c +++ b/proto/bfd/packets.c @@ -412,7 +412,7 @@ bfd_err_hook(sock *sk, int err) sock * bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af) { - sock *sk = sk_new(p->tpool); + sock *sk = sk_new(p->p.pool); sk->type = SK_UDP; sk->subtype = af; sk->sport = !multihop ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; @@ -475,7 +475,7 @@ bfd_open_rx_sk_bound(struct bfd_proto *p, ip_addr local, struct iface *ifa) sock * bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa) { - sock *sk = sk_new(p->tpool); + sock *sk = sk_new(p->p.pool); sk->type = SK_UDP; sk->saddr = local; sk->dport = ifa ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; diff --git a/proto/bgp/bgp.c b/proto/bgp/bgp.c index fb8fa529..3b624a49 100644 --- a/proto/bgp/bgp.c +++ b/proto/bgp/bgp.c @@ -832,6 +832,7 @@ bgp_graceful_restart_feed(struct bgp_channel *c) { c->stale_feed = (struct rt_export_request) { .name = "BGP-GR", + .list = &global_work_list, .trace_routes = c->c.debug | c->c.proto->debug, .dump_req = bgp_graceful_restart_feed_dump_req, .log_state_change = bgp_graceful_restart_feed_log_state_change, diff --git a/sysdep/unix/Makefile b/sysdep/unix/Makefile index a01e8f46..9831dc96 100644 --- a/sysdep/unix/Makefile +++ b/sysdep/unix/Makefile @@ -1,4 +1,4 @@ -src := alloc.c io.c krt.c log.c main.c random.c coroutine.c +src := alloc.c io.c io-loop.c krt.c log.c main.c random.c coroutine.c obj := $(src-o-files) $(all-daemon) $(cf-local) diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index edad6209..a2384ca8 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -97,7 +97,7 @@ alloc_page(void) struct free_page *fp = SKIP_BACK(struct free_page, n, HEAD(fps->pages)); rem_node(&fp->n); if ((--fps->cnt < fps->min) && !shutting_down) - ev_schedule(&fps->cleanup); + ev_send(&global_work_list, &fps->cleanup); bzero(fp, page_size); return fp; @@ -124,7 +124,7 @@ free_page(void *ptr) add_tail(&fps->pages, &fp->n); if ((++fps->cnt > fps->max) && !shutting_down) - ev_schedule(&fps->cleanup); + ev_send(&global_work_list, &fps->cleanup); #endif } diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c index 2eba142c..4758c056 100644 --- a/sysdep/unix/coroutine.c +++ b/sysdep/unix/coroutine.c @@ -21,10 +21,9 @@ #include "lib/resource.h" #include "lib/timer.h" -/* Using a rather big stack for coroutines to allow for stack-local allocations. - * In real world, the kernel doesn't alloc this memory until it is used. - * */ -#define CORO_STACK_SIZE 1048576 +#include "conf/conf.h" + +#define CORO_STACK_SIZE 65536 /* * Implementation of coroutines based on POSIX threads @@ -79,6 +78,11 @@ domain_free(struct domain_generic *dg) xfree(dg); } +uint dg_order(struct domain_generic *dg) +{ + return dg->order; +} + void do_lock(struct domain_generic *dg, struct domain_generic **lsp) { if ((char *) lsp - (char *) &locking_stack != dg->order) @@ -89,7 +93,11 @@ void do_lock(struct domain_generic *dg, struct domain_generic **lsp) if (*lsp) bug("Inconsistent locking stack state on lock"); + btime lock_begin = current_time(); pthread_mutex_lock(&dg->mutex); + btime duration = current_time() - lock_begin; + if (config && (duration > config->watchdog_warning)) + log(L_WARN "Locking of %s took %d ms", dg->name, (int) (duration TO_MS)); if (dg->prev || dg->locked_by) bug("Previous unlock not finished correctly"); @@ -140,11 +148,16 @@ static struct resclass coro_class = { .free = coro_free, }; +_Thread_local struct coroutine *this_coro = NULL; + static void *coro_entry(void *p) { struct coroutine *c = p; + ASSERT_DIE(c->entry); + this_coro = c; + c->entry(c->data); ASSERT_DIE(coro_cleaned_up); diff --git a/proto/bfd/io.c b/sysdep/unix/io-loop.c index 2805e0f2..dfb2ce49 100644 --- a/proto/bfd/io.c +++ b/sysdep/unix/io-loop.c @@ -15,50 +15,47 @@ #include <sys/time.h> #include "nest/bird.h" -#include "proto/bfd/io.h" #include "lib/buffer.h" +#include "lib/coro.h" #include "lib/lists.h" #include "lib/resource.h" #include "lib/event.h" #include "lib/timer.h" #include "lib/socket.h" - -struct birdloop -{ - pool *pool; - pthread_t thread; - pthread_mutex_t mutex; - - u8 stop_called; - u8 poll_active; - u8 wakeup_masked; - int wakeup_fds[2]; - - struct timeloop time; - list event_list; - list sock_list; - uint sock_num; - - BUFFER(sock *) poll_sk; - BUFFER(struct pollfd) poll_fd; - u8 poll_changed; - u8 close_scheduled; -}; - +#include "lib/io-loop.h" +#include "sysdep/unix/io-loop.h" +#include "conf/conf.h" /* * Current thread context */ -static _Thread_local struct birdloop *birdloop_current; +_Thread_local struct birdloop *birdloop_current; +static _Thread_local struct birdloop *birdloop_wakeup_masked; +static _Thread_local uint birdloop_wakeup_masked_count; -static inline void -birdloop_set_current(struct birdloop *loop) +event_list * +birdloop_event_list(struct birdloop *loop) { - birdloop_current = loop; - local_timeloop = loop ? &loop->time : &main_timeloop; + return &loop->event_list; +} + +struct timeloop * +birdloop_time_loop(struct birdloop *loop) +{ + return &loop->time; +} + +_Bool +birdloop_inside(struct birdloop *loop) +{ + for (struct birdloop *c = birdloop_current; c; c = c->prev_loop) + if (loop == c) + return 1; + + return 0; } /* @@ -135,57 +132,17 @@ wakeup_do_kick(struct birdloop *loop) pipe_kick(loop->wakeup_fds[1]); } -static inline void -wakeup_kick(struct birdloop *loop) -{ - if (!loop->wakeup_masked) - wakeup_do_kick(loop); - else - loop->wakeup_masked = 2; -} - -/* For notifications from outside */ -void -wakeup_kick_current(void) -{ - if (birdloop_current && birdloop_current->poll_active) - wakeup_kick(birdloop_current); -} - - -/* - * Events - */ - -static inline uint -events_waiting(struct birdloop *loop) -{ - return !EMPTY_LIST(loop->event_list); -} - -static inline void -events_init(struct birdloop *loop) -{ - init_list(&loop->event_list); -} - -static void -events_fire(struct birdloop *loop) -{ - times_update(); - ev_run_list(&loop->event_list); -} - void -ev2_schedule(event *e) +birdloop_ping(struct birdloop *loop) { - if (birdloop_current->poll_active && EMPTY_LIST(birdloop_current->event_list)) - wakeup_kick(birdloop_current); + u32 ping_sent = atomic_fetch_add_explicit(&loop->ping_sent, 1, memory_order_acq_rel); + if (ping_sent) + return; - if (e->n.next) - rem_node(&e->n); - - add_tail(&birdloop_current->event_list, &e->n); + if (loop == birdloop_wakeup_masked) + birdloop_wakeup_masked_count++; + else + wakeup_do_kick(loop); } @@ -213,13 +170,13 @@ sockets_add(struct birdloop *loop, sock *s) s->index = -1; loop->poll_changed = 1; - if (loop->poll_active) - wakeup_kick(loop); + birdloop_ping(loop); } void sk_start(sock *s) { + ASSERT_DIE(birdloop_current != &main_birdloop); sockets_add(birdloop_current, s); } @@ -230,28 +187,21 @@ sockets_remove(struct birdloop *loop, sock *s) loop->sock_num--; if (s->index >= 0) + { loop->poll_sk.data[s->index] = NULL; - - s->index = -1; - loop->poll_changed = 1; - - /* Wakeup moved to sk_stop() */ + s->index = -1; + loop->poll_changed = 1; + loop->close_scheduled = 1; + birdloop_ping(loop); + } + else + close(s->fd); } void sk_stop(sock *s) { sockets_remove(birdloop_current, s); - - if (birdloop_current->poll_active) - { - birdloop_current->close_scheduled = 1; - wakeup_kick(birdloop_current); - } - else - close(s->fd); - - s->fd = -1; } static inline uint sk_want_events(sock *s) @@ -351,12 +301,15 @@ sockets_fire(struct birdloop *loop) if (pfd->revents & POLLIN) while (e && *psk && (*psk)->rx_hook) - e = sk_read(*psk, 0); + e = sk_read(*psk, pfd->revents); e = 1; if (pfd->revents & POLLOUT) + { + loop->poll_changed = 1; while (e && *psk) e = sk_write(*psk); + } } } @@ -365,106 +318,170 @@ sockets_fire(struct birdloop *loop) * Birdloop */ -static void *birdloop_main(void *arg); +struct birdloop main_birdloop; + +static void birdloop_enter_locked(struct birdloop *loop); + +void +birdloop_init(void) +{ + wakeup_init(&main_birdloop); + + main_birdloop.time.domain = the_bird_domain.the_bird; + main_birdloop.time.loop = &main_birdloop; + + times_update(); + timers_init(&main_birdloop.time, &root_pool); + + birdloop_enter_locked(&main_birdloop); +} + +static void birdloop_main(void *arg); struct birdloop * -birdloop_new(void) +birdloop_new(pool *pp, uint order, const char *name) { - pool *p = rp_new(NULL, "Birdloop root"); + struct domain_generic *dg = domain_new(name, order); + + pool *p = rp_new(pp, name); struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop)); loop->pool = p; - pthread_mutex_init(&loop->mutex, NULL); - wakeup_init(loop); + loop->time.domain = dg; + loop->time.loop = loop; - events_init(loop); + birdloop_enter(loop); + + wakeup_init(loop); + ev_init_list(&loop->event_list, loop, name); timers_init(&loop->time, p); sockets_init(loop); + loop->time.coro = coro_run(p, birdloop_main, loop); + + birdloop_leave(loop); + return loop; } +static void +birdloop_do_stop(struct birdloop *loop, void (*stopped)(void *data), void *data) +{ + loop->stopped = stopped; + loop->stop_data = data; + wakeup_do_kick(loop); +} + void -birdloop_start(struct birdloop *loop) +birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data) { - int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop); - if (rv) - die("pthread_create(): %M", rv); + DG_LOCK(loop->time.domain); + birdloop_do_stop(loop, stopped, data); + DG_UNLOCK(loop->time.domain); } void -birdloop_stop(struct birdloop *loop) +birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data) { - pthread_mutex_lock(&loop->mutex); - loop->stop_called = 1; - wakeup_do_kick(loop); - pthread_mutex_unlock(&loop->mutex); + ASSERT_DIE(loop == birdloop_current); + ASSERT_DIE(DG_IS_LOCKED(loop->time.domain)); - int rv = pthread_join(loop->thread, NULL); - if (rv) - die("pthread_join(): %M", rv); + birdloop_do_stop(loop, stopped, data); } void birdloop_free(struct birdloop *loop) { + ASSERT_DIE(loop->links == 0); + domain_free(loop->time.domain); rfree(loop->pool); } +static void +birdloop_enter_locked(struct birdloop *loop) +{ + ASSERT_DIE(DG_IS_LOCKED(loop->time.domain)); + ASSERT_DIE(!birdloop_inside(loop)); + + /* Store the old context */ + loop->prev_loop = birdloop_current; + + /* Put the new context */ + birdloop_current = loop; +} void birdloop_enter(struct birdloop *loop) { - /* TODO: these functions could save and restore old context */ - pthread_mutex_lock(&loop->mutex); - birdloop_set_current(loop); + DG_LOCK(loop->time.domain); + return birdloop_enter_locked(loop); +} + +static void +birdloop_leave_locked(struct birdloop *loop) +{ + /* Check the current context */ + ASSERT_DIE(birdloop_current == loop); + + /* Restore the old context */ + birdloop_current = loop->prev_loop; } void birdloop_leave(struct birdloop *loop) { - /* TODO: these functions could save and restore old context */ - birdloop_set_current(NULL); - pthread_mutex_unlock(&loop->mutex); + birdloop_leave_locked(loop); + DG_UNLOCK(loop->time.domain); } void birdloop_mask_wakeups(struct birdloop *loop) { - pthread_mutex_lock(&loop->mutex); - loop->wakeup_masked = 1; - pthread_mutex_unlock(&loop->mutex); + ASSERT_DIE(birdloop_wakeup_masked == NULL); + birdloop_wakeup_masked = loop; } void birdloop_unmask_wakeups(struct birdloop *loop) { - pthread_mutex_lock(&loop->mutex); - if (loop->wakeup_masked == 2) + ASSERT_DIE(birdloop_wakeup_masked == loop); + birdloop_wakeup_masked = NULL; + if (birdloop_wakeup_masked_count) wakeup_do_kick(loop); - loop->wakeup_masked = 0; - pthread_mutex_unlock(&loop->mutex); + + birdloop_wakeup_masked_count = 0; } -static void * +void +birdloop_link(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_inside(loop)); + loop->links++; +} + +void +birdloop_unlink(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_inside(loop)); + loop->links--; +} + +static void birdloop_main(void *arg) { struct birdloop *loop = arg; timer *t; int rv, timeout; - birdloop_set_current(loop); + btime loop_begin = current_time(); tmp_init(loop->pool); - pthread_mutex_lock(&loop->mutex); + birdloop_enter(loop); while (1) { - events_fire(loop); - timers_fire(&loop->time); - - times_update(); - if (events_waiting(loop)) + timers_fire(&loop->time, 0); + if (ev_run_list(&loop->event_list)) timeout = 0; else if (t = timers_first(&loop->time)) timeout = (tm_remains(t) TO_MS) + 1; @@ -474,8 +491,11 @@ birdloop_main(void *arg) if (loop->poll_changed) sockets_prepare(loop); - loop->poll_active = 1; - pthread_mutex_unlock(&loop->mutex); + btime duration = current_time() - loop_begin; + if (duration > config->watchdog_warning) + log(L_WARN "I/O loop cycle took %d ms", (int) (duration TO_MS)); + + birdloop_leave(loop); try: rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout); @@ -486,25 +506,32 @@ birdloop_main(void *arg) die("poll: %m"); } - pthread_mutex_lock(&loop->mutex); - loop->poll_active = 0; + birdloop_enter(loop); if (loop->close_scheduled) sockets_close_fds(loop); - if (loop->stop_called) + if (loop->stopped) break; + loop_begin = current_time(); + if (rv) sockets_fire(loop); - timers_fire(&loop->time); + atomic_exchange_explicit(&loop->ping_sent, 0, memory_order_acq_rel); } - loop->stop_called = 0; - pthread_mutex_unlock(&loop->mutex); + /* Flush remaining events */ + ASSERT_DIE(!ev_run_list(&loop->event_list)); + + /* No timers allowed */ + ASSERT_DIE(timers_count(&loop->time) == 0); + ASSERT_DIE(EMPTY_LIST(loop->sock_list)); + ASSERT_DIE(loop->sock_num == 0); - return NULL; + birdloop_leave(loop); + loop->stopped(loop->stop_data); } diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h new file mode 100644 index 00000000..4024b6c5 --- /dev/null +++ b/sysdep/unix/io-loop.h @@ -0,0 +1,35 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_SYSDEP_UNIX_IO_LOOP_H_ +#define _BIRD_SYSDEP_UNIX_IO_LOOP_H_ + +struct birdloop +{ + pool *pool; + + struct timeloop time; + event_list event_list; + list sock_list; + uint sock_num; + + BUFFER(sock *) poll_sk; + BUFFER(struct pollfd) poll_fd; + u8 poll_changed; + u8 close_scheduled; + + _Atomic u32 ping_sent; + int wakeup_fds[2]; + + uint links; + + void (*stopped)(void *data); + void *stop_data; + + struct birdloop *prev_loop; +}; + +#endif diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index a3f1491a..23baffb2 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -43,6 +43,7 @@ #include "conf/conf.h" #include "sysdep/unix/unix.h" +#include "sysdep/unix/io-loop.h" #include CONFIG_INCLUDE_SYSIO_H /* Maximum number of calls of tx handler for one socket in one @@ -800,18 +801,16 @@ sk_free(resource *r) sk_ssh_free(s); #endif - if (s->fd < 0) + if ((s->fd < 0) || (s->flags & SKF_THREAD)) return; - /* FIXME: we should call sk_stop() for SKF_THREAD sockets */ - if (!(s->flags & SKF_THREAD)) - { - if (s == current_sock) - current_sock = sk_next(s); - if (s == stored_sock) - stored_sock = sk_next(s); + if (s == current_sock) + current_sock = sk_next(s); + if (s == stored_sock) + stored_sock = sk_next(s); + + if (enlisted(&s->n)) rem_node(&s->n); - } if (s->type != SK_SSH && s->type != SK_SSH_ACTIVE) close(s->fd); @@ -1104,7 +1103,11 @@ sk_passive_connected(sock *s, int type) return 1; } - sk_insert(t); + if (s->flags & SKF_PASSIVE_THREAD) + t->flags |= SKF_THREAD; + else + sk_insert(t); + sk_alloc_bufs(t); s->rx_hook(t, 0); return 1; @@ -1512,6 +1515,36 @@ sk_open_unix(sock *s, char *name) return 0; } +static void +sk_reloop_hook(void *_vs) +{ + sock *s = _vs; + if (birdloop_inside(&main_birdloop)) + { + s->flags &= ~SKF_THREAD; + sk_insert(s); + } + else + { + s->flags |= SKF_THREAD; + sk_start(s); + } +} + +void +sk_reloop(sock *s, struct birdloop *loop) +{ + if (enlisted(&s->n)) + rem_node(&s->n); + + s->reloop = (event) { + .hook = sk_reloop_hook, + .data = s, + }; + + ev_send_loop(loop, &s->reloop); +} + #define CMSG_RX_SPACE MAX(CMSG4_SPACE_PKTINFO+CMSG4_SPACE_TTL, \ CMSG6_SPACE_PKTINFO+CMSG6_SPACE_TTL) @@ -2164,8 +2197,9 @@ void io_init(void) { init_list(&sock_list); - init_list(&global_event_list); - init_list(&global_work_list); + ev_init_list(&global_event_list, &main_birdloop, "Global event list"); + ev_init_list(&global_work_list, &main_birdloop, "Global work list"); + ev_init_list(&main_birdloop.event_list, &main_birdloop, "Global fast event list"); krt_io_init(); // XXX init_times(); // XXX update_times(); @@ -2179,14 +2213,7 @@ static int short_loops = 0; #define SHORT_LOOP_MAX 10 #define WORK_EVENTS_MAX 10 -static int poll_reload_pipe[2]; - -void -io_loop_reload(void) -{ - char b; - write(poll_reload_pipe[1], &b, 1); -} +void pipe_drain(int fd); void io_loop(void) @@ -2199,21 +2226,19 @@ io_loop(void) int fdmax = 256; struct pollfd *pfd = xmalloc(fdmax * sizeof(struct pollfd)); - if (pipe(poll_reload_pipe) < 0) - die("pipe(poll_reload_pipe) failed: %m"); - watchdog_start1(); for(;;) { times_update(); events = ev_run_list(&global_event_list); events = ev_run_list_limited(&global_work_list, WORK_EVENTS_MAX) || events; - timers_fire(&main_timeloop); + events = ev_run_list(&main_birdloop.event_list) || events; + timers_fire(&main_birdloop.time, 1); io_close_event(); // FIXME poll_tout = (events ? 0 : 3000); /* Time in milliseconds */ - if (t = timers_first(&main_timeloop)) + if (t = timers_first(&main_birdloop.time)) { times_update(); timeout = (tm_remains(t) TO_MS) + 1; @@ -2221,7 +2246,7 @@ io_loop(void) } /* A hack to reload main io_loop() when something has changed asynchronously. */ - pfd[0].fd = poll_reload_pipe[0]; + pfd[0].fd = main_birdloop.wakeup_fds[0]; pfd[0].events = POLLIN; nfds = 1; @@ -2284,9 +2309,9 @@ io_loop(void) /* And finally enter poll() to find active sockets */ watchdog_stop(); - the_bird_unlock(); + birdloop_leave(&main_birdloop); pout = poll(pfd, nfds, poll_tout); - the_bird_lock(); + birdloop_enter(&main_birdloop); watchdog_start(); if (pout < 0) @@ -2300,8 +2325,8 @@ io_loop(void) if (pfd[0].revents & POLLIN) { /* IO loop reload requested */ - char b; - read(poll_reload_pipe[0], &b, 1); + pipe_drain(main_birdloop.wakeup_fds[0]); + atomic_exchange_explicit(&main_birdloop.ping_sent, 0, memory_order_acq_rel); continue; } diff --git a/sysdep/unix/main.c b/sysdep/unix/main.c index 8bc07d94..bf9f2be0 100644 --- a/sysdep/unix/main.c +++ b/sysdep/unix/main.c @@ -878,10 +878,12 @@ main(int argc, char **argv) parse_args(argc, argv); log_switch(1, NULL, NULL); + the_bird_lock(); + random_init(); net_init(); resource_init(); - timer_init(); + birdloop_init(); olock_init(); rt_init(); io_init(); @@ -929,7 +931,6 @@ main(int argc, char **argv) dup2(0, 2); } - the_bird_lock(); main_thread_init(); diff --git a/sysdep/unix/unix.h b/sysdep/unix/unix.h index 313c97c3..ad85d1ea 100644 --- a/sysdep/unix/unix.h +++ b/sysdep/unix/unix.h @@ -106,7 +106,6 @@ extern volatile sig_atomic_t async_shutdown_flag; void io_init(void); void io_loop(void); -void io_loop_reload(void); void io_log_dump(void); int sk_open_unix(struct birdsock *s, char *name); struct rfile *rf_open(struct pool *, const char *name, const char *mode); diff --git a/test/birdtest.c b/test/birdtest.c index ae05d1a5..2ae7b51e 100644 --- a/test/birdtest.c +++ b/test/birdtest.c @@ -21,6 +21,7 @@ #include "test/birdtest.h" #include "lib/string.h" #include "lib/event.h" +#include "lib/io-loop.h" #ifdef HAVE_EXECINFO_H #include <execinfo.h> @@ -121,8 +122,8 @@ bt_init(int argc, char *argv[]) bt_suite_case_begin = bt_suite_begin = bt_begin; resource_init(); - ev_init_list(&global_event_list); - + ev_init_list(&global_event_list, &main_birdloop, "Global event list in unit tests"); + ev_init_list(&global_work_list, &main_birdloop, "Global work list in unit tests"); return; usage: diff --git a/test/bt-utils.c b/test/bt-utils.c index 509b5ed4..3d56292e 100644 --- a/test/bt-utils.c +++ b/test/bt-utils.c @@ -53,6 +53,8 @@ cf_file_read(byte *dest, uint max_len, int fd) return l; } +void resource_sys_init(void); + void bt_bird_init(void) { @@ -60,8 +62,9 @@ bt_bird_init(void) log_init_debug(""); log_switch(bt_verbose != 0, NULL, NULL); + the_bird_lock(); olock_init(); - timer_init(); + birdloop_init(); rt_init(); io_init(); if_init(); @@ -73,6 +76,7 @@ bt_bird_init(void) void bt_bird_cleanup(void) { config = new_config = NULL; + the_bird_unlock(); } static char * |