From 44f26c49f966ca842ff9af55468de0b98c44b73e Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Wed, 6 Oct 2021 15:10:33 +0200 Subject: Special table hooks rectified. * internal tables are now more standalone, having their own import and export hooks * route refresh/reload uses stale counter instead of stale flag, allowing to drop walking the table at the beginning * route modify (by BGP LLGR) is now done by a special refeed hook, reimporting the modified routes directly without filters --- sysdep/unix/krt.c | 256 ++++-------------------------------------------------- sysdep/unix/krt.h | 4 - 2 files changed, 17 insertions(+), 243 deletions(-) (limited to 'sysdep/unix') diff --git a/sysdep/unix/krt.c b/sysdep/unix/krt.c index 40a58442..609ee921 100644 --- a/sysdep/unix/krt.c +++ b/sysdep/unix/krt.c @@ -285,249 +285,24 @@ krt_metric(rte *a) } static inline int -krt_same_key(rte *a, rte *b) +krt_rte_better(rte *a, rte *b) { - return (krt_metric(a) == krt_metric(b)); -} - -static inline int -krt_uptodate(rte *a, rte *b) -{ - return (a->attrs == b->attrs); -} - -static void -krt_learn_announce_update(struct krt_proto *p, rte *e) -{ - rte e0 = { - .attrs = rta_clone(e->attrs), - .src = p->p.main_source, - }; - - rte_update(p->p.main_channel, e->net, &e0, p->p.main_source); -} - -static void -krt_learn_announce_delete(struct krt_proto *p, net_addr *n) -{ - rte_update(p->p.main_channel, n, NULL, p->p.main_source); + return (krt_metric(a) > krt_metric(b)); } /* Called when alien route is discovered during scan */ static void -krt_learn_scan(struct krt_proto *p, rte *e) -{ - net *n = net_get(p->krt_table, e->net); - struct rte_storage *m, **mm; - - struct rte_storage *ee = rte_store(e, n, p->krt_table); - - for(mm = &n->routes; m = *mm; mm = &m->next) - if (krt_same_key(&m->rte, e)) - break; - if (m) - { - if (krt_uptodate(&m->rte, e)) - { - krt_trace_in_rl(&rl_alien, p, e, "[alien] seen"); - rte_free(ee, p->krt_table); - m->rte.pflags |= KRT_REF_SEEN; - } - else - { - krt_trace_in(p, e, "[alien] updated"); - *mm = m->next; - rte_free(m, p->krt_table); - m = NULL; - } - } - else - krt_trace_in(p, e, "[alien] created"); - - if (!m) - { - ee->next = n->routes; - n->routes = ee; - ee->rte.pflags |= KRT_REF_SEEN; - } -} - -static void -krt_learn_prune(struct krt_proto *p) +krt_learn_rte(struct krt_proto *p, rte *e) { - struct fib *fib = &p->krt_table->fib; - struct fib_iterator fit; - - KRT_TRACE(p, D_EVENTS, "Pruning inherited routes"); - - FIB_ITERATE_INIT(&fit, fib); -again: - FIB_ITERATE_START(fib, &fit, net, n) - { - struct rte_storage *e, **ee, *best, **pbest, *old_best; - - /* - * Note that old_best may be NULL even if there was an old best route in - * the previous step, because it might be replaced in krt_learn_scan(). - * But in that case there is a new valid best route. - */ - - old_best = NULL; - best = NULL; - pbest = NULL; - ee = &n->routes; - while (e = *ee) - { - if (e->rte.pflags & KRT_REF_BEST) - old_best = e; - - if (!(e->rte.pflags & KRT_REF_SEEN)) - { - *ee = e->next; - rte_free(e, p->krt_table); - continue; - } - - if (!best || krt_metric(&best->rte) > krt_metric(&e->rte)) - { - best = e; - pbest = ee; - } - - e->rte.pflags &= ~(KRT_REF_SEEN | KRT_REF_BEST); - ee = &e->next; - } - if (!n->routes) - { - DBG("%I/%d: deleting\n", n->n.prefix, n->n.pxlen); - if (old_best) - krt_learn_announce_delete(p, n->n.addr); - - FIB_ITERATE_PUT(&fit); - fib_delete(fib, n); - goto again; - } - - best->rte.pflags |= KRT_REF_BEST; - *pbest = best->next; - best->next = n->routes; - n->routes = best; - - if ((best != old_best) || p->reload) - { - DBG("%I/%d: announcing (metric=%d)\n", n->n.prefix, n->n.pxlen, krt_metric(&best->rte)); - krt_learn_announce_update(p, &best->rte); - } - else - DBG("%I/%d: uptodate (metric=%d)\n", n->n.prefix, n->n.pxlen, krt_metric(&best->rte)); - } - FIB_ITERATE_END; - - p->reload = 0; -} - -static void -krt_learn_async(struct krt_proto *p, rte *e, int new) -{ - net *n = net_get(p->krt_table, e->net); - struct rte_storage *g, **gg, *best, **bestp, *old_best; - - ASSERT(!e->attrs->cached); - e->attrs->pref = p->p.main_channel->preference; - - struct rte_storage *ee = rte_store(e, n, p->krt_table); - - old_best = n->routes; - for(gg=&n->routes; g = *gg; gg = &g->next) - if (krt_same_key(&g->rte, e)) - break; - if (new) - { - if (g) - { - if (krt_uptodate(&g->rte, e)) - { - krt_trace_in(p, e, "[alien async] same"); - rte_free(ee, p->krt_table); - return; - } - krt_trace_in(p, e, "[alien async] updated"); - *gg = g->next; - rte_free(g, p->krt_table); - } - else - krt_trace_in(p, e, "[alien async] created"); - - ee->next = n->routes; - n->routes = ee; - } - else if (!g) - { - krt_trace_in(p, e, "[alien async] delete failed"); - rte_free(ee, p->krt_table); - return; - } - else - { - krt_trace_in(p, e, "[alien async] removed"); - *gg = g->next; - rte_free(ee, p->krt_table); - rte_free(g, p->krt_table); - } - best = n->routes; - bestp = &n->routes; - for(gg=&n->routes; g=*gg; gg=&g->next) - { - if (krt_metric(&best->rte) > krt_metric(&g->rte)) - { - best = g; - bestp = gg; - } - - g->rte.pflags &= ~KRT_REF_BEST; - } - - if (best) - { - best->rte.pflags |= KRT_REF_BEST; - *bestp = best->next; - best->next = n->routes; - n->routes = best; - } - - if (best != old_best) - { - DBG("krt_learn_async: distributing change\n"); - if (best) - krt_learn_announce_update(p, &best->rte); - else - krt_learn_announce_delete(p, n->n.addr); - } + e->src = rt_get_source(&p->p, krt_metric(e)); + rte_update(p->p.main_channel, e->net, e, e->src); } static void krt_learn_init(struct krt_proto *p) { if (KRT_CF->learn) - { - struct rtable_config *cf = mb_allocz(p->p.pool, sizeof(struct rtable_config)); - cf->name = "Inherited"; - cf->addr_type = p->p.net_type; - cf->internal = 1; - - p->krt_table = rt_setup(p->p.pool, cf); - } -} - -static void -krt_dump(struct proto *P) -{ - struct krt_proto *p = (struct krt_proto *) P; - - if (!KRT_CF->learn) - return; - debug("KRT: Table of inheritable routes\n"); - rt_dump(p->krt_table); + channel_setup_in_table(p->p.main_channel, 1); } #endif @@ -547,7 +322,7 @@ rte_feed_count(net *n) { uint count = 0; for (struct rte_storage *e = n->routes; e; e = e->next) - if (rte_is_valid(RTE_OR_NULL(e))) + if (rte_is_valid(RTES_OR_NULL(e))) count++; return count; } @@ -557,7 +332,7 @@ rte_feed_obtain(net *n, rte **feed, uint count) { uint i = 0; for (struct rte_storage *e = n->routes; e; e = e->next) - if (rte_is_valid(RTE_OR_NULL(e))) + if (rte_is_valid(RTES_OR_NULL(e))) { ASSERT_DIE(i < count); feed[i++] = &e->rte; @@ -643,7 +418,7 @@ krt_got_route(struct krt_proto *p, rte *e, s8 src) case KRT_SRC_ALIEN: if (KRT_CF->learn) - krt_learn_scan(p, e); + krt_learn_rte(p, e); else krt_trace_in_rl(&rl_alien, p, e, "[alien] ignored"); return; @@ -712,6 +487,11 @@ static void krt_init_scan(struct krt_proto *p) { bmap_reset(&p->seen_map, 1024); + +#ifdef KRT_ALLOW_LEARN + if (KRT_CF->learn) + channel_refresh_begin(p->p.main_channel); +#endif } static void @@ -739,7 +519,7 @@ krt_prune(struct krt_proto *p) #ifdef KRT_ALLOW_LEARN if (KRT_CF->learn) - krt_learn_prune(p); + channel_refresh_end(p->p.main_channel); #endif if (p->ready) @@ -781,7 +561,7 @@ krt_got_route_async(struct krt_proto *p, rte *e, int new, s8 src) case KRT_SRC_ALIEN: if (KRT_CF->learn) { - krt_learn_async(p, e, new); + krt_learn_rte(p, e); return; } #endif @@ -1027,6 +807,7 @@ krt_init(struct proto_config *CF) p->p.if_notify = krt_if_notify; p->p.reload_routes = krt_reload_routes; p->p.feed_end = krt_feed_end; + p->p.rte_better = krt_rte_better; krt_sys_init(p); return &p->p; @@ -1182,7 +963,4 @@ struct protocol proto_unix_kernel = { .reconfigure = krt_reconfigure, .copy_config = krt_copy_config, .get_attr = krt_get_attr, -#ifdef KRT_ALLOW_LEARN - .dump = krt_dump, -#endif }; diff --git a/sysdep/unix/krt.h b/sysdep/unix/krt.h index f6ad6fde..968c5b16 100644 --- a/sysdep/unix/krt.h +++ b/sysdep/unix/krt.h @@ -51,10 +51,6 @@ struct krt_proto { struct proto p; struct krt_state sys; /* Sysdep state */ -#ifdef KRT_ALLOW_LEARN - struct rtable *krt_table; /* Internal table of inherited routes */ -#endif - #ifndef CONFIG_ALL_TABLES_AT_ONCE timer *scan_timer; #endif -- cgit v1.2.3 From b5061659d3cc011118024861c2f048e67affbd39 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Thu, 4 Feb 2021 15:08:52 +0100 Subject: POSIX threads and thread-local storage is needed for concurrent execution --- aclocal.m4 | 14 ++++++++++++-- configure.ac | 46 +++++++++++++++++++--------------------------- lib/birdlib.h | 4 ---- lib/timer.c | 10 ---------- sysdep/unix/log.c | 11 ----------- 5 files changed, 31 insertions(+), 54 deletions(-) (limited to 'sysdep/unix') diff --git a/aclocal.m4 b/aclocal.m4 index 1613d680..3405b85b 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -1,5 +1,6 @@ dnl ** Additional Autoconf tests for BIRD configure script dnl ** (c) 1999 Martin Mares +dnl ** (c) 2021 Maria Matejka AC_DEFUN([BIRD_CHECK_THREAD_LOCAL], [ @@ -9,14 +10,23 @@ AC_DEFUN([BIRD_CHECK_THREAD_LOCAL], AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [ - _Thread_local static int x = 42; + static _Thread_local int x = 42; ], [] ) ], [bird_cv_thread_local=yes], + [AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM( + [ + static __thread int x = 42; + ], + [] + ) + ], + [bird_cv_thread_local=__thread], [bird_cv_thread_local=no] - ) + )]) ) ]) diff --git a/configure.ac b/configure.ac index 64181d29..5c0cf002 100644 --- a/configure.ac +++ b/configure.ac @@ -36,12 +36,6 @@ AC_ARG_ENABLE([memcheck], [enable_memcheck=yes] ) -AC_ARG_ENABLE([pthreads], - [AS_HELP_STRING([--enable-pthreads], [enable POSIX threads support @<:@try@:>@])], - [], - [enable_pthreads=try] -) - AC_ARG_ENABLE([libssh], [AS_HELP_STRING([--enable-libssh], [enable LibSSH support in RPKI @<:@try@:>@])], [], @@ -125,25 +119,19 @@ if test -z "$GCC" ; then fi BIRD_CHECK_THREAD_LOCAL -if test "$bird_cv_thread_local" = yes ; then - AC_DEFINE([HAVE_THREAD_LOCAL], [1], [Define to 1 if _Thread_local is available]) +if test "$bird_cv_thread_local" = no ; then + AC_MSG_ERROR([This program requires thread local storage.]) +elif test "$bird_cv_thread_local" != yes ; then + AC_DEFINE_UNQUOTED([_Thread_local], [$bird_cv_thread_local], [Legacy _Thread_local]) fi -if test "$enable_pthreads" != no ; then - BIRD_CHECK_PTHREADS +BIRD_CHECK_PTHREADS - if test "$bird_cv_lib_pthreads" = yes ; then - AC_DEFINE([USE_PTHREADS], [1], [Define to 1 if pthreads are enabled]) - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - proto_bfd=bfd - elif test "$enable_pthreads" = yes ; then - AC_MSG_ERROR([POSIX threads not available.]) - fi - - if test "$enable_pthreads" = try ; then - enable_pthreads="$bird_cv_lib_pthreads" - fi +if test "$bird_cv_lib_pthreads" = yes ; then + CFLAGS="$CFLAGS -pthread" + LDFLAGS="$LDFLAGS -pthread" +else + AC_MSG_ERROR([POSIX threads not available.]) fi # This is assumed to be necessary for proper BIRD build @@ -304,8 +292,7 @@ if test "$enable_mpls_kernel" != no ; then fi fi -all_protocols="$proto_bfd babel bgp mrt ospf perf pipe radv rip rpki static" - +all_protocols="bfd babel bgp mrt ospf perf pipe radv rip rpki static" all_protocols=`echo $all_protocols | sed 's/ /,/g'` if test "$with_protocols" = all ; then @@ -351,9 +338,15 @@ case $sysdesc in esac AC_CHECK_HEADERS_ONCE([alloca.h syslog.h]) -AC_CHECK_HEADER([sys/mman.h], [AC_DEFINE([HAVE_MMAP], [1], [Define to 1 if mmap() is available.])]) +AC_CHECK_HEADER([sys/mman.h], [AC_DEFINE([HAVE_MMAP], [1], [Define to 1 if mmap() is available.])], have_mman=no) +AC_CHECK_FUNC([aligned_alloc], [AC_DEFINE([HAVE_ALIGNED_ALLOC], [1], [Define to 1 if aligned_alloc() is available.])], have_aligned_alloc=no) AC_CHECK_MEMBERS([struct sockaddr.sa_len], [], [], [#include ]) +if test "$have_aligned_alloc" = "no" && test "$have_mman" = "no" ; then + AC_MSG_ERROR([No means of aligned alloc found. Need mmap() or aligned_alloc().]) +fi + + AC_C_BIGENDIAN( [AC_DEFINE([CPU_BIG_ENDIAN], [1], [Define to 1 if cpu is big endian])], [AC_DEFINE([CPU_LITTLE_ENDIAN], [1], [Define to 1 if cpu is little endian])], @@ -402,7 +395,7 @@ if test "$enable_debug" = yes ; then fi fi - if test "enable_debug_expensive" = yes ; then + if test "$enable_debug_expensive" = yes ; then AC_DEFINE([ENABLE_EXPENSIVE_CHECKS], [1], [Define to 1 if you want to run expensive consistency checks.]) fi fi @@ -467,7 +460,6 @@ AC_MSG_RESULT([ Object directory: $objdir]) AC_MSG_RESULT([ Iproute2 directory: $iproutedir]) AC_MSG_RESULT([ System configuration: $sysdesc]) AC_MSG_RESULT([ Debugging: $enable_debug]) -AC_MSG_RESULT([ POSIX threads: $enable_pthreads]) AC_MSG_RESULT([ Routing protocols: $protocols]) AC_MSG_RESULT([ LibSSH support in RPKI: $enable_libssh]) AC_MSG_RESULT([ Kernel MPLS support: $enable_mpls_kernel]) diff --git a/lib/birdlib.h b/lib/birdlib.h index 2e642d38..dc8bd00f 100644 --- a/lib/birdlib.h +++ b/lib/birdlib.h @@ -77,10 +77,6 @@ static inline int u64_cmp(u64 i1, u64 i2) #define STATIC_ASSERT(EXP) _Static_assert(EXP, #EXP) #define STATIC_ASSERT_MSG(EXP,MSG) _Static_assert(EXP, MSG) -#ifndef HAVE_THREAD_LOCAL -#define _Thread_local -#endif - /* Microsecond time */ typedef s64 btime; diff --git a/lib/timer.c b/lib/timer.c index 381163d0..f978a0f3 100644 --- a/lib/timer.c +++ b/lib/timer.c @@ -40,8 +40,6 @@ struct timeloop main_timeloop; -#ifdef USE_PTHREADS - #include /* Data accessed and modified from proto/bfd/io.c */ @@ -62,14 +60,6 @@ timeloop_init_current(void) void wakeup_kick_current(void); -#else - -/* Just use main timelooop */ -static inline struct timeloop * timeloop_current(void) { return &main_timeloop; } -static inline void timeloop_init_current(void) { } - -#endif - btime current_time(void) { diff --git a/sysdep/unix/log.c b/sysdep/unix/log.c index 4e9df069..a23903b7 100644 --- a/sysdep/unix/log.c +++ b/sysdep/unix/log.c @@ -36,8 +36,6 @@ static list *current_log_list; static char *current_syslog_name; /* NULL -> syslog closed */ -#ifdef USE_PTHREADS - #include static pthread_mutex_t log_mutex; @@ -48,15 +46,6 @@ static pthread_t main_thread; void main_thread_init(void) { main_thread = pthread_self(); } static int main_thread_self(void) { return pthread_equal(pthread_self(), main_thread); } -#else - -static inline void log_lock(void) { } -static inline void log_unlock(void) { } -void main_thread_init(void) { } -static int main_thread_self(void) { return 1; } - -#endif - #ifdef HAVE_SYSLOG_H #include -- cgit v1.2.3 From 1db83a507a9ae287815d62733d1337074993b433 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Thu, 4 Feb 2021 15:52:42 +0100 Subject: Locking subsystem: Just a global BIRD lock to begin with. --- lib/birdlib.h | 1 + lib/lists.c | 2 +- lib/locking.h | 46 ++++++++++++++++++++++ sysdep/unix/Makefile | 2 +- sysdep/unix/coroutine.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ sysdep/unix/io.c | 3 ++ sysdep/unix/main.c | 3 ++ 7 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 lib/locking.h create mode 100644 sysdep/unix/coroutine.c (limited to 'sysdep/unix') diff --git a/lib/birdlib.h b/lib/birdlib.h index dc8bd00f..3dc39d19 100644 --- a/lib/birdlib.h +++ b/lib/birdlib.h @@ -9,6 +9,7 @@ #ifndef _BIRD_BIRDLIB_H_ #define _BIRD_BIRDLIB_H_ +#include "sysdep/config.h" #include "lib/alloca.h" /* Ugly structure offset handling macros */ diff --git a/lib/lists.c b/lib/lists.c index 58d51073..dc2e4cbb 100644 --- a/lib/lists.c +++ b/lib/lists.c @@ -26,7 +26,7 @@ #define _BIRD_LISTS_C_ -#include "nest/bird.h" +#include "lib/birdlib.h" #include "lib/lists.h" LIST_INLINE int diff --git a/lib/locking.h b/lib/locking.h new file mode 100644 index 00000000..eb1bc8fa --- /dev/null +++ b/lib/locking.h @@ -0,0 +1,46 @@ +/* + * BIRD Library -- Locking + * + * (c) 2020--2021 Maria Matejka + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_LOCKING_H_ +#define _BIRD_LOCKING_H_ + +struct domain_generic; + +/* Here define the global lock order; first to last. */ +struct lock_order { + struct domain_generic *the_bird; +}; + +#define LOCK_ORDER_DEPTH (sizeof(struct lock_order) / sizeof(struct domain_generic *)) + +extern _Thread_local struct lock_order locking_stack; +extern _Thread_local struct domain_generic **last_locked; + +#define DOMAIN(type) struct domain__##type +#define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; } + +#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name) } +struct domain_generic *domain_new(const char *name); + +#define DOMAIN_NULL(type) (DOMAIN(type)) {} + +#define LOCK_DOMAIN(type, d) do_lock(((d).type), &(locking_stack.type)) +#define UNLOCK_DOMAIN(type, d) do_unlock(((d).type), &(locking_stack.type)) + +/* Internal for locking */ +void do_lock(struct domain_generic *dg, struct domain_generic **lsp); +void do_unlock(struct domain_generic *dg, struct domain_generic **lsp); + +/* Use with care. To be removed in near future. */ +DEFINE_DOMAIN(the_bird); +extern DOMAIN(the_bird) the_bird_domain; + +#define the_bird_lock() LOCK_DOMAIN(the_bird, the_bird_domain) +#define the_bird_unlock() UNLOCK_DOMAIN(the_bird, the_bird_domain) + +#endif diff --git a/sysdep/unix/Makefile b/sysdep/unix/Makefile index d0d36b5f..69cf8131 100644 --- a/sysdep/unix/Makefile +++ b/sysdep/unix/Makefile @@ -1,4 +1,4 @@ -src := alloc.c io.c krt.c log.c main.c random.c +src := alloc.c io.c krt.c log.c main.c random.c coroutine.c obj := $(src-o-files) $(all-daemon) $(cf-local) diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c new file mode 100644 index 00000000..05f101fb --- /dev/null +++ b/sysdep/unix/coroutine.c @@ -0,0 +1,102 @@ +/* + * BIRD Coroutines + * + * (c) 2017 Martin Mares + * (c) 2020 Maria Matejka + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#undef LOCAL_DEBUG + +#undef DEBUG_LOCKING + +#include "lib/birdlib.h" +#include "lib/locking.h" +#include "lib/resource.h" + +/* + * Implementation of coroutines based on POSIX threads + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Locking subsystem + */ + +#define ASSERT_NO_LOCK ASSERT_DIE(last_locked == NULL) + +struct domain_generic { + pthread_mutex_t mutex; + struct domain_generic **prev; + struct lock_order *locked_by; + const char *name; +}; + +#define DOMAIN_INIT(_name) { .mutex = PTHREAD_MUTEX_INITIALIZER, .name = _name } + +static struct domain_generic the_bird_domain_gen = DOMAIN_INIT("The BIRD"); + +DOMAIN(the_bird) the_bird_domain = { .the_bird = &the_bird_domain_gen }; + +struct domain_generic * +domain_new(const char *name) +{ + struct domain_generic *dg = xmalloc(sizeof(struct domain_generic)); + *dg = (struct domain_generic) DOMAIN_INIT(name); + return dg; +} + +void +domain_free(struct domain_generic *dg) +{ + pthread_mutex_destroy(&dg->mutex); + xfree(dg); +} + +_Thread_local struct lock_order locking_stack = {}; +_Thread_local struct domain_generic **last_locked = NULL; + +void do_lock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if (lsp <= last_locked) + bug("Trying to lock in a bad order"); + if (*lsp) + bug("Inconsistent locking stack state on lock"); + + pthread_mutex_lock(&dg->mutex); + + if (dg->prev || dg->locked_by) + bug("Previous unlock not finished correctly"); + dg->prev = last_locked; + *lsp = dg; + last_locked = lsp; + dg->locked_by = &locking_stack; +} + +void do_unlock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if (dg->locked_by != &locking_stack) + bug("Inconsistent domain state on unlock"); + if ((last_locked != lsp) || (*lsp != dg)) + bug("Inconsistent locking stack state on unlock"); + dg->locked_by = NULL; + last_locked = dg->prev; + *lsp = NULL; + dg->prev = NULL; + pthread_mutex_unlock(&dg->mutex); +} + diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index 3d67d0a7..29467867 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -36,6 +36,7 @@ #include "lib/resource.h" #include "lib/socket.h" #include "lib/event.h" +#include "lib/locking.h" #include "lib/timer.h" #include "lib/string.h" #include "nest/iface.h" @@ -2263,7 +2264,9 @@ io_loop(void) /* And finally enter poll() to find active sockets */ watchdog_stop(); + the_bird_unlock(); pout = poll(pfd, nfds, poll_tout); + the_bird_lock(); watchdog_start(); if (pout < 0) diff --git a/sysdep/unix/main.c b/sysdep/unix/main.c index 7e8ea0dc..dabfc554 100644 --- a/sysdep/unix/main.c +++ b/sysdep/unix/main.c @@ -28,6 +28,7 @@ #include "lib/resource.h" #include "lib/socket.h" #include "lib/event.h" +#include "lib/locking.h" #include "lib/timer.h" #include "lib/string.h" #include "nest/route.h" @@ -959,6 +960,8 @@ main(int argc, char **argv) dup2(0, 2); } + the_bird_lock(); + main_thread_init(); write_pid_file(); -- cgit v1.2.3 From 1289c1c5eede5b3d015d06b725d30024ccac51bd Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Mon, 8 Feb 2021 09:51:59 +0100 Subject: Coroutines: A simple and lightweight parallel execution framework. --- lib/coro.h | 26 +++++++++++++++++++ sysdep/unix/coroutine.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++ sysdep/unix/io.c | 27 ++++++++++++++++++- sysdep/unix/log.c | 34 +++++++++++++++++++++--- sysdep/unix/unix.h | 1 + 5 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 lib/coro.h (limited to 'sysdep/unix') diff --git a/lib/coro.h b/lib/coro.h new file mode 100644 index 00000000..51712b36 --- /dev/null +++ b/lib/coro.h @@ -0,0 +1,26 @@ +/* + * BIRD Coroutines + * + * (c) 2017 Martin Mares + * (c) 2020 Maria Matejka + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_CORO_H_ +#define _BIRD_CORO_H_ + +#include "lib/resource.h" + +/* A completely opaque coroutine handle. */ +struct coroutine; + +/* Coroutines are independent threads bound to pools. + * You request a coroutine by calling coro_run(). + * It is forbidden to free a running coroutine from outside. + * The running coroutine must free itself by rfree() before returning. + */ +struct coroutine *coro_run(pool *, void (*entry)(void *), void *data); + + +#endif diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c index 05f101fb..71847505 100644 --- a/sysdep/unix/coroutine.c +++ b/sysdep/unix/coroutine.c @@ -17,7 +17,14 @@ #include "lib/birdlib.h" #include "lib/locking.h" +#include "lib/coro.h" #include "lib/resource.h" +#include "lib/timer.h" + +/* Using a rather big stack for coroutines to allow for stack-local allocations. + * In real world, the kernel doesn't alloc this memory until it is used. + * */ +#define CORO_STACK_SIZE 1048576 /* * Implementation of coroutines based on POSIX threads @@ -100,3 +107,65 @@ void do_unlock(struct domain_generic *dg, struct domain_generic **lsp) pthread_mutex_unlock(&dg->mutex); } +/* Coroutines */ +struct coroutine { + resource r; + pthread_t id; + pthread_attr_t attr; + void (*entry)(void *); + void *data; +}; + +static _Thread_local _Bool coro_cleaned_up = 0; + +static void coro_free(resource *r) +{ + struct coroutine *c = (void *) r; + ASSERT_DIE(pthread_equal(pthread_self(), c->id)); + pthread_attr_destroy(&c->attr); + coro_cleaned_up = 1; +} + +static struct resclass coro_class = { + .name = "Coroutine", + .size = sizeof(struct coroutine), + .free = coro_free, +}; + +static void *coro_entry(void *p) +{ + struct coroutine *c = p; + ASSERT_DIE(c->entry); + + c->entry(c->data); + ASSERT_DIE(coro_cleaned_up); + + return NULL; +} + +struct coroutine *coro_run(pool *p, void (*entry)(void *), void *data) +{ + ASSERT_DIE(entry); + ASSERT_DIE(p); + + struct coroutine *c = ralloc(p, &coro_class); + + c->entry = entry; + c->data = data; + + int e = 0; + + if (e = pthread_attr_init(&c->attr)) + die("pthread_attr_init() failed: %M", e); + + if (e = pthread_attr_setstacksize(&c->attr, CORO_STACK_SIZE)) + die("pthread_attr_setstacksize(%u) failed: %M", CORO_STACK_SIZE, e); + + if (e = pthread_attr_setdetachstate(&c->attr, PTHREAD_CREATE_DETACHED)) + die("pthread_attr_setdetachstate(PTHREAD_CREATE_DETACHED) failed: %M", e); + + if (e = pthread_create(&c->id, &c->attr, coro_entry, c)) + die("pthread_create() failed: %M", e); + + return c; +} diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index 29467867..40841ea4 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -2176,6 +2176,15 @@ static int short_loops = 0; #define SHORT_LOOP_MAX 10 #define WORK_EVENTS_MAX 10 +static int poll_reload_pipe[2]; + +void +io_loop_reload(void) +{ + char b; + write(poll_reload_pipe[1], &b, 1); +} + void io_loop(void) { @@ -2187,6 +2196,9 @@ io_loop(void) int fdmax = 256; struct pollfd *pfd = xmalloc(fdmax * sizeof(struct pollfd)); + if (pipe(poll_reload_pipe) < 0) + die("pipe(poll_reload_pipe) failed: %m"); + watchdog_start1(); for(;;) { @@ -2205,7 +2217,12 @@ io_loop(void) poll_tout = MIN(poll_tout, timeout); } - nfds = 0; + /* A hack to reload main io_loop() when something has changed asynchronously. */ + pfd[0].fd = poll_reload_pipe[0]; + pfd[0].events = POLLIN; + + nfds = 1; + WALK_LIST(n, sock_list) { pfd[nfds] = (struct pollfd) { .fd = -1 }; /* everything other set to 0 by this */ @@ -2277,6 +2294,14 @@ io_loop(void) } if (pout) { + if (pfd[0].revents & POLLIN) + { + /* IO loop reload requested */ + char b; + read(poll_reload_pipe[0], &b, 1); + continue; + } + times_update(&main_timeloop); /* guaranteed to be non-empty */ diff --git a/sysdep/unix/log.c b/sysdep/unix/log.c index a23903b7..dc2b14b3 100644 --- a/sysdep/unix/log.c +++ b/sysdep/unix/log.c @@ -15,6 +15,7 @@ * user's manual. */ +#include #include #include #include @@ -35,6 +36,10 @@ static FILE *dbgf; static list *current_log_list; static char *current_syslog_name; /* NULL -> syslog closed */ +static _Atomic uint max_coro_id = ATOMIC_VAR_INIT(1); +static _Thread_local uint this_coro_id; + +#define THIS_CORO_ID (this_coro_id ?: (this_coro_id = atomic_fetch_add_explicit(&max_coro_id, 1, memory_order_acq_rel))) #include @@ -178,7 +183,7 @@ log_commit(int class, buffer *buf) l->pos += msg_len; } - fprintf(l->fh, "%s <%s> ", tbuf, class_names[class]); + fprintf(l->fh, "%s [%04x] <%s> ", tbuf, THIS_CORO_ID, class_names[class]); } fputs(buf->start, l->fh); fputc('\n', l->fh); @@ -288,6 +293,8 @@ die(const char *msg, ...) exit(1); } +static struct timespec dbg_time_start; + /** * debug - write to debug output * @msg: a printf-like message @@ -300,12 +307,33 @@ debug(const char *msg, ...) { #define MAX_DEBUG_BUFSIZE 16384 va_list args; - char buf[MAX_DEBUG_BUFSIZE]; + char buf[MAX_DEBUG_BUFSIZE], *pos = buf; + int max = MAX_DEBUG_BUFSIZE; va_start(args, msg); if (dbgf) { - if (bvsnprintf(buf, MAX_DEBUG_BUFSIZE, msg, args) < 0) + struct timespec dbg_time; + clock_gettime(CLOCK_MONOTONIC, &dbg_time); + uint nsec; + uint sec; + + if (dbg_time.tv_nsec > dbg_time_start.tv_nsec) + { + nsec = dbg_time.tv_nsec - dbg_time_start.tv_nsec; + sec = dbg_time.tv_sec - dbg_time_start.tv_sec; + } + else + { + nsec = 1000000000 + dbg_time.tv_nsec - dbg_time_start.tv_nsec; + sec = dbg_time.tv_sec - dbg_time_start.tv_sec - 1; + } + + int n = bsnprintf(pos, max, "%u.%09u: [%04x] ", sec, nsec, THIS_CORO_ID); + pos += n; + max -= n; + + if (bvsnprintf(pos, max, msg, args) < 0) bug("Extremely long debug output, split it."); fputs(buf, dbgf); diff --git a/sysdep/unix/unix.h b/sysdep/unix/unix.h index ad85d1ea..313c97c3 100644 --- a/sysdep/unix/unix.h +++ b/sysdep/unix/unix.h @@ -106,6 +106,7 @@ extern volatile sig_atomic_t async_shutdown_flag; void io_init(void); void io_loop(void); +void io_loop_reload(void); void io_log_dump(void); int sk_open_unix(struct birdsock *s, char *name); struct rfile *rf_open(struct pool *, const char *name, const char *mode); -- cgit v1.2.3 From df3264f51ff38c9366398564a9d342a26bc83f37 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Mon, 24 May 2021 13:41:23 +0200 Subject: Lock position checking allows for safe lock unions --- lib/locking.h | 6 ++---- sysdep/unix/coroutine.c | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) (limited to 'sysdep/unix') diff --git a/lib/locking.h b/lib/locking.h index eb1bc8fa..eef60154 100644 --- a/lib/locking.h +++ b/lib/locking.h @@ -16,16 +16,14 @@ struct lock_order { struct domain_generic *the_bird; }; -#define LOCK_ORDER_DEPTH (sizeof(struct lock_order) / sizeof(struct domain_generic *)) - extern _Thread_local struct lock_order locking_stack; extern _Thread_local struct domain_generic **last_locked; #define DOMAIN(type) struct domain__##type #define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; } -#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name) } -struct domain_generic *domain_new(const char *name); +#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name, OFFSETOF(struct lock_order, type)) } +struct domain_generic *domain_new(const char *name, uint order); #define DOMAIN_NULL(type) (DOMAIN(type)) {} diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c index 71847505..2eba142c 100644 --- a/sysdep/unix/coroutine.c +++ b/sysdep/unix/coroutine.c @@ -44,26 +44,31 @@ * Locking subsystem */ +_Thread_local struct lock_order locking_stack = {}; +_Thread_local struct domain_generic **last_locked = NULL; + #define ASSERT_NO_LOCK ASSERT_DIE(last_locked == NULL) struct domain_generic { pthread_mutex_t mutex; + uint order; struct domain_generic **prev; struct lock_order *locked_by; const char *name; }; -#define DOMAIN_INIT(_name) { .mutex = PTHREAD_MUTEX_INITIALIZER, .name = _name } +#define DOMAIN_INIT(_name, _order) { .mutex = PTHREAD_MUTEX_INITIALIZER, .name = _name, .order = _order } -static struct domain_generic the_bird_domain_gen = DOMAIN_INIT("The BIRD"); +static struct domain_generic the_bird_domain_gen = DOMAIN_INIT("The BIRD", OFFSETOF(struct lock_order, the_bird)); DOMAIN(the_bird) the_bird_domain = { .the_bird = &the_bird_domain_gen }; struct domain_generic * -domain_new(const char *name) +domain_new(const char *name, uint order) { + ASSERT_DIE(order < sizeof(struct lock_order)); struct domain_generic *dg = xmalloc(sizeof(struct domain_generic)); - *dg = (struct domain_generic) DOMAIN_INIT(name); + *dg = (struct domain_generic) DOMAIN_INIT(name, order); return dg; } @@ -74,11 +79,11 @@ domain_free(struct domain_generic *dg) xfree(dg); } -_Thread_local struct lock_order locking_stack = {}; -_Thread_local struct domain_generic **last_locked = NULL; - void do_lock(struct domain_generic *dg, struct domain_generic **lsp) { + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to lock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + if (lsp <= last_locked) bug("Trying to lock in a bad order"); if (*lsp) @@ -96,6 +101,9 @@ void do_lock(struct domain_generic *dg, struct domain_generic **lsp) void do_unlock(struct domain_generic *dg, struct domain_generic **lsp) { + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to unlock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + if (dg->locked_by != &locking_stack) bug("Inconsistent domain state on unlock"); if ((last_locked != lsp) || (*lsp != dg)) -- cgit v1.2.3 From a2af807357875f866f149b0f6db4d463d4533204 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Mon, 28 Jun 2021 15:43:45 +0200 Subject: Debug messages with timestamps. On most of current hardware, getting monotonic clock is fast enough to get it and write for each debug message. --- sysdep/unix/log.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'sysdep/unix') diff --git a/sysdep/unix/log.c b/sysdep/unix/log.c index dc2b14b3..f48588b6 100644 --- a/sysdep/unix/log.c +++ b/sysdep/unix/log.c @@ -439,6 +439,8 @@ done: void log_init_debug(char *f) { + clock_gettime(CLOCK_MONOTONIC, &dbg_time_start); + if (dbgf && dbgf != stderr) fclose(dbgf); if (!f) -- cgit v1.2.3 From c84ed603714db2c42a781f8dbb5b3fd540ff689f Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Fri, 18 Jun 2021 18:23:41 +0200 Subject: Moved BFD IO loop out of BFD as we want to use it as socket-io coroutine --- proto/bfd/Makefile | 4 +- proto/bfd/bfd.h | 2 +- proto/bfd/io.c | 508 -------------------------------------------------- proto/bfd/io.h | 34 ---- sysdep/unix/Makefile | 2 +- sysdep/unix/io-loop.c | 508 ++++++++++++++++++++++++++++++++++++++++++++++++++ sysdep/unix/io-loop.h | 34 ++++ 7 files changed, 546 insertions(+), 546 deletions(-) delete mode 100644 proto/bfd/io.c delete mode 100644 proto/bfd/io.h create mode 100644 sysdep/unix/io-loop.c create mode 100644 sysdep/unix/io-loop.h (limited to 'sysdep/unix') diff --git a/proto/bfd/Makefile b/proto/bfd/Makefile index 402122fc..267dff98 100644 --- a/proto/bfd/Makefile +++ b/proto/bfd/Makefile @@ -1,6 +1,6 @@ -src := bfd.c io.c packets.c +src := bfd.c packets.c obj := $(src-o-files) $(all-daemon) $(cf-local) -tests_objs := $(tests_objs) $(src-o-files) \ No newline at end of file +tests_objs := $(tests_objs) $(src-o-files) diff --git a/proto/bfd/bfd.h b/proto/bfd/bfd.h index 91fdaa60..9d4cbbf8 100644 --- a/proto/bfd/bfd.h +++ b/proto/bfd/bfd.h @@ -22,7 +22,7 @@ #include "lib/string.h" #include "nest/bfd.h" -#include "io.h" +#include "sysdep/unix/io-loop.h" #define BFD_CONTROL_PORT 3784 diff --git a/proto/bfd/io.c b/proto/bfd/io.c deleted file mode 100644 index c5f1e024..00000000 --- a/proto/bfd/io.c +++ /dev/null @@ -1,508 +0,0 @@ -/* - * BIRD -- I/O and event loop - * - * Can be freely distributed and used under the terms of the GNU GPL. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nest/bird.h" -#include "proto/bfd/io.h" - -#include "lib/buffer.h" -#include "lib/lists.h" -#include "lib/resource.h" -#include "lib/event.h" -#include "lib/timer.h" -#include "lib/socket.h" - - -struct birdloop -{ - pool *pool; - pthread_t thread; - pthread_mutex_t mutex; - - u8 stop_called; - u8 poll_active; - u8 wakeup_masked; - int wakeup_fds[2]; - - struct timeloop time; - list event_list; - list sock_list; - uint sock_num; - - BUFFER(sock *) poll_sk; - BUFFER(struct pollfd) poll_fd; - u8 poll_changed; - u8 close_scheduled; -}; - - -/* - * Current thread context - */ - -static _Thread_local struct birdloop *birdloop_current; - -static inline void -birdloop_set_current(struct birdloop *loop) -{ - birdloop_current = loop; - local_timeloop = loop ? &loop->time : &main_timeloop; -} - -/* - * Wakeup code for birdloop - */ - -static void -pipe_new(int *pfds) -{ - int rv = pipe(pfds); - if (rv < 0) - die("pipe: %m"); - - if (fcntl(pfds[0], F_SETFL, O_NONBLOCK) < 0) - die("fcntl(O_NONBLOCK): %m"); - - if (fcntl(pfds[1], F_SETFL, O_NONBLOCK) < 0) - die("fcntl(O_NONBLOCK): %m"); -} - -void -pipe_drain(int fd) -{ - char buf[64]; - int rv; - - try: - rv = read(fd, buf, 64); - if (rv < 0) - { - if (errno == EINTR) - goto try; - if (errno == EAGAIN) - return; - die("wakeup read: %m"); - } - if (rv == 64) - goto try; -} - -void -pipe_kick(int fd) -{ - u64 v = 1; - int rv; - - try: - rv = write(fd, &v, sizeof(u64)); - if (rv < 0) - { - if (errno == EINTR) - goto try; - if (errno == EAGAIN) - return; - die("wakeup write: %m"); - } -} - -static inline void -wakeup_init(struct birdloop *loop) -{ - pipe_new(loop->wakeup_fds); -} - -static inline void -wakeup_drain(struct birdloop *loop) -{ - pipe_drain(loop->wakeup_fds[0]); -} - -static inline void -wakeup_do_kick(struct birdloop *loop) -{ - pipe_kick(loop->wakeup_fds[1]); -} - -static inline void -wakeup_kick(struct birdloop *loop) -{ - if (!loop->wakeup_masked) - wakeup_do_kick(loop); - else - loop->wakeup_masked = 2; -} - -/* For notifications from outside */ -void -wakeup_kick_current(void) -{ - if (birdloop_current && birdloop_current->poll_active) - wakeup_kick(birdloop_current); -} - - -/* - * Events - */ - -static inline uint -events_waiting(struct birdloop *loop) -{ - return !EMPTY_LIST(loop->event_list); -} - -static inline void -events_init(struct birdloop *loop) -{ - init_list(&loop->event_list); -} - -static void -events_fire(struct birdloop *loop) -{ - times_update(); - ev_run_list(&loop->event_list); -} - -void -ev2_schedule(event *e) -{ - if (birdloop_current->poll_active && EMPTY_LIST(birdloop_current->event_list)) - wakeup_kick(birdloop_current); - - if (e->n.next) - rem_node(&e->n); - - add_tail(&birdloop_current->event_list, &e->n); -} - - -/* - * Sockets - */ - -static void -sockets_init(struct birdloop *loop) -{ - init_list(&loop->sock_list); - loop->sock_num = 0; - - BUFFER_INIT(loop->poll_sk, loop->pool, 4); - BUFFER_INIT(loop->poll_fd, loop->pool, 4); - loop->poll_changed = 1; /* add wakeup fd */ -} - -static void -sockets_add(struct birdloop *loop, sock *s) -{ - add_tail(&loop->sock_list, &s->n); - loop->sock_num++; - - s->index = -1; - loop->poll_changed = 1; - - if (loop->poll_active) - wakeup_kick(loop); -} - -void -sk_start(sock *s) -{ - sockets_add(birdloop_current, s); -} - -static void -sockets_remove(struct birdloop *loop, sock *s) -{ - rem_node(&s->n); - loop->sock_num--; - - if (s->index >= 0) - loop->poll_sk.data[s->index] = NULL; - - s->index = -1; - loop->poll_changed = 1; - - /* Wakeup moved to sk_stop() */ -} - -void -sk_stop(sock *s) -{ - sockets_remove(birdloop_current, s); - - if (birdloop_current->poll_active) - { - birdloop_current->close_scheduled = 1; - wakeup_kick(birdloop_current); - } - else - close(s->fd); - - s->fd = -1; -} - -static inline uint sk_want_events(sock *s) -{ return (s->rx_hook ? POLLIN : 0) | ((s->ttx != s->tpos) ? POLLOUT : 0); } - -/* -FIXME: this should be called from sock code - -static void -sockets_update(struct birdloop *loop, sock *s) -{ - if (s->index >= 0) - loop->poll_fd.data[s->index].events = sk_want_events(s); -} -*/ - -static void -sockets_prepare(struct birdloop *loop) -{ - BUFFER_SET(loop->poll_sk, loop->sock_num + 1); - BUFFER_SET(loop->poll_fd, loop->sock_num + 1); - - struct pollfd *pfd = loop->poll_fd.data; - sock **psk = loop->poll_sk.data; - uint i = 0; - node *n; - - WALK_LIST(n, loop->sock_list) - { - sock *s = SKIP_BACK(sock, n, n); - - ASSERT(i < loop->sock_num); - - s->index = i; - *psk = s; - pfd->fd = s->fd; - pfd->events = sk_want_events(s); - pfd->revents = 0; - - pfd++; - psk++; - i++; - } - - ASSERT(i == loop->sock_num); - - /* Add internal wakeup fd */ - *psk = NULL; - pfd->fd = loop->wakeup_fds[0]; - pfd->events = POLLIN; - pfd->revents = 0; - - loop->poll_changed = 0; -} - -static void -sockets_close_fds(struct birdloop *loop) -{ - struct pollfd *pfd = loop->poll_fd.data; - sock **psk = loop->poll_sk.data; - int poll_num = loop->poll_fd.used - 1; - - int i; - for (i = 0; i < poll_num; i++) - if (psk[i] == NULL) - close(pfd[i].fd); - - loop->close_scheduled = 0; -} - -int sk_read(sock *s, int revents); -int sk_write(sock *s); - -static void -sockets_fire(struct birdloop *loop) -{ - struct pollfd *pfd = loop->poll_fd.data; - sock **psk = loop->poll_sk.data; - int poll_num = loop->poll_fd.used - 1; - - times_update(); - - /* Last fd is internal wakeup fd */ - if (pfd[poll_num].revents & POLLIN) - wakeup_drain(loop); - - int i; - for (i = 0; i < poll_num; pfd++, psk++, i++) - { - int e = 1; - - if (! pfd->revents) - continue; - - if (pfd->revents & POLLNVAL) - die("poll: invalid fd %d", pfd->fd); - - if (pfd->revents & POLLIN) - while (e && *psk && (*psk)->rx_hook) - e = sk_read(*psk, 0); - - e = 1; - if (pfd->revents & POLLOUT) - while (e && *psk) - e = sk_write(*psk); - } -} - - -/* - * Birdloop - */ - -static void *birdloop_main(void *arg); - -struct birdloop * -birdloop_new(void) -{ - pool *p = rp_new(NULL, "Birdloop root"); - struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop)); - loop->pool = p; - pthread_mutex_init(&loop->mutex, NULL); - - wakeup_init(loop); - - events_init(loop); - timers_init(&loop->time, p); - sockets_init(loop); - - return loop; -} - -void -birdloop_start(struct birdloop *loop) -{ - int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop); - if (rv) - die("pthread_create(): %M", rv); -} - -void -birdloop_stop(struct birdloop *loop) -{ - pthread_mutex_lock(&loop->mutex); - loop->stop_called = 1; - wakeup_do_kick(loop); - pthread_mutex_unlock(&loop->mutex); - - int rv = pthread_join(loop->thread, NULL); - if (rv) - die("pthread_join(): %M", rv); -} - -void -birdloop_free(struct birdloop *loop) -{ - rfree(loop->pool); -} - - -void -birdloop_enter(struct birdloop *loop) -{ - /* TODO: these functions could save and restore old context */ - pthread_mutex_lock(&loop->mutex); - birdloop_set_current(loop); -} - -void -birdloop_leave(struct birdloop *loop) -{ - /* TODO: these functions could save and restore old context */ - birdloop_set_current(NULL); - pthread_mutex_unlock(&loop->mutex); -} - -void -birdloop_mask_wakeups(struct birdloop *loop) -{ - pthread_mutex_lock(&loop->mutex); - loop->wakeup_masked = 1; - pthread_mutex_unlock(&loop->mutex); -} - -void -birdloop_unmask_wakeups(struct birdloop *loop) -{ - pthread_mutex_lock(&loop->mutex); - if (loop->wakeup_masked == 2) - wakeup_do_kick(loop); - loop->wakeup_masked = 0; - pthread_mutex_unlock(&loop->mutex); -} - -static void * -birdloop_main(void *arg) -{ - struct birdloop *loop = arg; - timer *t; - int rv, timeout; - - birdloop_set_current(loop); - - pthread_mutex_lock(&loop->mutex); - while (1) - { - events_fire(loop); - timers_fire(&loop->time); - - times_update(); - if (events_waiting(loop)) - timeout = 0; - else if (t = timers_first(&loop->time)) - timeout = (tm_remains(t) TO_MS) + 1; - else - timeout = -1; - - if (loop->poll_changed) - sockets_prepare(loop); - - loop->poll_active = 1; - pthread_mutex_unlock(&loop->mutex); - - try: - rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout); - if (rv < 0) - { - if (errno == EINTR || errno == EAGAIN) - goto try; - die("poll: %m"); - } - - pthread_mutex_lock(&loop->mutex); - loop->poll_active = 0; - - if (loop->close_scheduled) - sockets_close_fds(loop); - - if (loop->stop_called) - break; - - if (rv) - sockets_fire(loop); - - timers_fire(&loop->time); - } - - loop->stop_called = 0; - pthread_mutex_unlock(&loop->mutex); - - return NULL; -} - - diff --git a/proto/bfd/io.h b/proto/bfd/io.h deleted file mode 100644 index ec706e9a..00000000 --- a/proto/bfd/io.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * BIRD -- I/O and event loop - * - * Can be freely distributed and used under the terms of the GNU GPL. - */ - -#ifndef _BIRD_BFD_IO_H_ -#define _BIRD_BFD_IO_H_ - -#include "nest/bird.h" -#include "lib/lists.h" -#include "lib/resource.h" -#include "lib/event.h" -#include "lib/timer.h" -#include "lib/socket.h" - - -void ev2_schedule(event *e); - -void sk_start(sock *s); -void sk_stop(sock *s); - -struct birdloop *birdloop_new(void); -void birdloop_start(struct birdloop *loop); -void birdloop_stop(struct birdloop *loop); -void birdloop_free(struct birdloop *loop); - -void birdloop_enter(struct birdloop *loop); -void birdloop_leave(struct birdloop *loop); -void birdloop_mask_wakeups(struct birdloop *loop); -void birdloop_unmask_wakeups(struct birdloop *loop); - - -#endif /* _BIRD_BFD_IO_H_ */ diff --git a/sysdep/unix/Makefile b/sysdep/unix/Makefile index 69cf8131..07f454ab 100644 --- a/sysdep/unix/Makefile +++ b/sysdep/unix/Makefile @@ -1,4 +1,4 @@ -src := alloc.c io.c krt.c log.c main.c random.c coroutine.c +src := alloc.c io.c io-loop.c krt.c log.c main.c random.c coroutine.c obj := $(src-o-files) $(all-daemon) $(cf-local) diff --git a/sysdep/unix/io-loop.c b/sysdep/unix/io-loop.c new file mode 100644 index 00000000..a15d866a --- /dev/null +++ b/sysdep/unix/io-loop.c @@ -0,0 +1,508 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nest/bird.h" +#include "sysdep/unix/io-loop.h" + +#include "lib/buffer.h" +#include "lib/lists.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/timer.h" +#include "lib/socket.h" + + +struct birdloop +{ + pool *pool; + pthread_t thread; + pthread_mutex_t mutex; + + u8 stop_called; + u8 poll_active; + u8 wakeup_masked; + int wakeup_fds[2]; + + struct timeloop time; + list event_list; + list sock_list; + uint sock_num; + + BUFFER(sock *) poll_sk; + BUFFER(struct pollfd) poll_fd; + u8 poll_changed; + u8 close_scheduled; +}; + + +/* + * Current thread context + */ + +static _Thread_local struct birdloop *birdloop_current; + +static inline void +birdloop_set_current(struct birdloop *loop) +{ + birdloop_current = loop; + local_timeloop = loop ? &loop->time : &main_timeloop; +} + +/* + * Wakeup code for birdloop + */ + +static void +pipe_new(int *pfds) +{ + int rv = pipe(pfds); + if (rv < 0) + die("pipe: %m"); + + if (fcntl(pfds[0], F_SETFL, O_NONBLOCK) < 0) + die("fcntl(O_NONBLOCK): %m"); + + if (fcntl(pfds[1], F_SETFL, O_NONBLOCK) < 0) + die("fcntl(O_NONBLOCK): %m"); +} + +void +pipe_drain(int fd) +{ + char buf[64]; + int rv; + + try: + rv = read(fd, buf, 64); + if (rv < 0) + { + if (errno == EINTR) + goto try; + if (errno == EAGAIN) + return; + die("wakeup read: %m"); + } + if (rv == 64) + goto try; +} + +void +pipe_kick(int fd) +{ + u64 v = 1; + int rv; + + try: + rv = write(fd, &v, sizeof(u64)); + if (rv < 0) + { + if (errno == EINTR) + goto try; + if (errno == EAGAIN) + return; + die("wakeup write: %m"); + } +} + +static inline void +wakeup_init(struct birdloop *loop) +{ + pipe_new(loop->wakeup_fds); +} + +static inline void +wakeup_drain(struct birdloop *loop) +{ + pipe_drain(loop->wakeup_fds[0]); +} + +static inline void +wakeup_do_kick(struct birdloop *loop) +{ + pipe_kick(loop->wakeup_fds[1]); +} + +static inline void +wakeup_kick(struct birdloop *loop) +{ + if (!loop->wakeup_masked) + wakeup_do_kick(loop); + else + loop->wakeup_masked = 2; +} + +/* For notifications from outside */ +void +wakeup_kick_current(void) +{ + if (birdloop_current && birdloop_current->poll_active) + wakeup_kick(birdloop_current); +} + + +/* + * Events + */ + +static inline uint +events_waiting(struct birdloop *loop) +{ + return !EMPTY_LIST(loop->event_list); +} + +static inline void +events_init(struct birdloop *loop) +{ + init_list(&loop->event_list); +} + +static void +events_fire(struct birdloop *loop) +{ + times_update(); + ev_run_list(&loop->event_list); +} + +void +ev2_schedule(event *e) +{ + if (birdloop_current->poll_active && EMPTY_LIST(birdloop_current->event_list)) + wakeup_kick(birdloop_current); + + if (e->n.next) + rem_node(&e->n); + + add_tail(&birdloop_current->event_list, &e->n); +} + + +/* + * Sockets + */ + +static void +sockets_init(struct birdloop *loop) +{ + init_list(&loop->sock_list); + loop->sock_num = 0; + + BUFFER_INIT(loop->poll_sk, loop->pool, 4); + BUFFER_INIT(loop->poll_fd, loop->pool, 4); + loop->poll_changed = 1; /* add wakeup fd */ +} + +static void +sockets_add(struct birdloop *loop, sock *s) +{ + add_tail(&loop->sock_list, &s->n); + loop->sock_num++; + + s->index = -1; + loop->poll_changed = 1; + + if (loop->poll_active) + wakeup_kick(loop); +} + +void +sk_start(sock *s) +{ + sockets_add(birdloop_current, s); +} + +static void +sockets_remove(struct birdloop *loop, sock *s) +{ + rem_node(&s->n); + loop->sock_num--; + + if (s->index >= 0) + loop->poll_sk.data[s->index] = NULL; + + s->index = -1; + loop->poll_changed = 1; + + /* Wakeup moved to sk_stop() */ +} + +void +sk_stop(sock *s) +{ + sockets_remove(birdloop_current, s); + + if (birdloop_current->poll_active) + { + birdloop_current->close_scheduled = 1; + wakeup_kick(birdloop_current); + } + else + close(s->fd); + + s->fd = -1; +} + +static inline uint sk_want_events(sock *s) +{ return (s->rx_hook ? POLLIN : 0) | ((s->ttx != s->tpos) ? POLLOUT : 0); } + +/* +FIXME: this should be called from sock code + +static void +sockets_update(struct birdloop *loop, sock *s) +{ + if (s->index >= 0) + loop->poll_fd.data[s->index].events = sk_want_events(s); +} +*/ + +static void +sockets_prepare(struct birdloop *loop) +{ + BUFFER_SET(loop->poll_sk, loop->sock_num + 1); + BUFFER_SET(loop->poll_fd, loop->sock_num + 1); + + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + uint i = 0; + node *n; + + WALK_LIST(n, loop->sock_list) + { + sock *s = SKIP_BACK(sock, n, n); + + ASSERT(i < loop->sock_num); + + s->index = i; + *psk = s; + pfd->fd = s->fd; + pfd->events = sk_want_events(s); + pfd->revents = 0; + + pfd++; + psk++; + i++; + } + + ASSERT(i == loop->sock_num); + + /* Add internal wakeup fd */ + *psk = NULL; + pfd->fd = loop->wakeup_fds[0]; + pfd->events = POLLIN; + pfd->revents = 0; + + loop->poll_changed = 0; +} + +static void +sockets_close_fds(struct birdloop *loop) +{ + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + int poll_num = loop->poll_fd.used - 1; + + int i; + for (i = 0; i < poll_num; i++) + if (psk[i] == NULL) + close(pfd[i].fd); + + loop->close_scheduled = 0; +} + +int sk_read(sock *s, int revents); +int sk_write(sock *s); + +static void +sockets_fire(struct birdloop *loop) +{ + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + int poll_num = loop->poll_fd.used - 1; + + times_update(); + + /* Last fd is internal wakeup fd */ + if (pfd[poll_num].revents & POLLIN) + wakeup_drain(loop); + + int i; + for (i = 0; i < poll_num; pfd++, psk++, i++) + { + int e = 1; + + if (! pfd->revents) + continue; + + if (pfd->revents & POLLNVAL) + die("poll: invalid fd %d", pfd->fd); + + if (pfd->revents & POLLIN) + while (e && *psk && (*psk)->rx_hook) + e = sk_read(*psk, 0); + + e = 1; + if (pfd->revents & POLLOUT) + while (e && *psk) + e = sk_write(*psk); + } +} + + +/* + * Birdloop + */ + +static void *birdloop_main(void *arg); + +struct birdloop * +birdloop_new(void) +{ + pool *p = rp_new(NULL, "Birdloop root"); + struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop)); + loop->pool = p; + pthread_mutex_init(&loop->mutex, NULL); + + wakeup_init(loop); + + events_init(loop); + timers_init(&loop->time, p); + sockets_init(loop); + + return loop; +} + +void +birdloop_start(struct birdloop *loop) +{ + int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop); + if (rv) + die("pthread_create(): %M", rv); +} + +void +birdloop_stop(struct birdloop *loop) +{ + pthread_mutex_lock(&loop->mutex); + loop->stop_called = 1; + wakeup_do_kick(loop); + pthread_mutex_unlock(&loop->mutex); + + int rv = pthread_join(loop->thread, NULL); + if (rv) + die("pthread_join(): %M", rv); +} + +void +birdloop_free(struct birdloop *loop) +{ + rfree(loop->pool); +} + + +void +birdloop_enter(struct birdloop *loop) +{ + /* TODO: these functions could save and restore old context */ + pthread_mutex_lock(&loop->mutex); + birdloop_set_current(loop); +} + +void +birdloop_leave(struct birdloop *loop) +{ + /* TODO: these functions could save and restore old context */ + birdloop_set_current(NULL); + pthread_mutex_unlock(&loop->mutex); +} + +void +birdloop_mask_wakeups(struct birdloop *loop) +{ + pthread_mutex_lock(&loop->mutex); + loop->wakeup_masked = 1; + pthread_mutex_unlock(&loop->mutex); +} + +void +birdloop_unmask_wakeups(struct birdloop *loop) +{ + pthread_mutex_lock(&loop->mutex); + if (loop->wakeup_masked == 2) + wakeup_do_kick(loop); + loop->wakeup_masked = 0; + pthread_mutex_unlock(&loop->mutex); +} + +static void * +birdloop_main(void *arg) +{ + struct birdloop *loop = arg; + timer *t; + int rv, timeout; + + birdloop_set_current(loop); + + pthread_mutex_lock(&loop->mutex); + while (1) + { + events_fire(loop); + timers_fire(&loop->time); + + times_update(); + if (events_waiting(loop)) + timeout = 0; + else if (t = timers_first(&loop->time)) + timeout = (tm_remains(t) TO_MS) + 1; + else + timeout = -1; + + if (loop->poll_changed) + sockets_prepare(loop); + + loop->poll_active = 1; + pthread_mutex_unlock(&loop->mutex); + + try: + rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout); + if (rv < 0) + { + if (errno == EINTR || errno == EAGAIN) + goto try; + die("poll: %m"); + } + + pthread_mutex_lock(&loop->mutex); + loop->poll_active = 0; + + if (loop->close_scheduled) + sockets_close_fds(loop); + + if (loop->stop_called) + break; + + if (rv) + sockets_fire(loop); + + timers_fire(&loop->time); + } + + loop->stop_called = 0; + pthread_mutex_unlock(&loop->mutex); + + return NULL; +} + + diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h new file mode 100644 index 00000000..d858b04e --- /dev/null +++ b/sysdep/unix/io-loop.h @@ -0,0 +1,34 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_IO_LOOP_H_ +#define _BIRD_IO_LOOP_H_ + +#include "nest/bird.h" +#include "lib/lists.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/timer.h" +#include "lib/socket.h" + + +void ev2_schedule(event *e); + +void sk_start(sock *s); +void sk_stop(sock *s); + +struct birdloop *birdloop_new(void); +void birdloop_start(struct birdloop *loop); +void birdloop_stop(struct birdloop *loop); +void birdloop_free(struct birdloop *loop); + +void birdloop_enter(struct birdloop *loop); +void birdloop_leave(struct birdloop *loop); +void birdloop_mask_wakeups(struct birdloop *loop); +void birdloop_unmask_wakeups(struct birdloop *loop); + + +#endif /* _BIRD_IO_LOOP_H_ */ -- cgit v1.2.3 From a4451535c69b8f934523905a8131ae2f16be2146 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Wed, 4 Aug 2021 22:48:51 +0200 Subject: Unified time for whole BIRD In previous versions, every thread used its own time structures, effectively leading to different time in every thread and strange logging messages. The time processing code now uses global atomic variables to keep current time available for fast concurrent reading and safe updates. --- lib/timer.c | 29 +++++------------- lib/timer.h | 14 ++++----- proto/bfd/io.c | 8 ++--- sysdep/unix/io.c | 90 ++++++++++++++++++++++-------------------------------- sysdep/unix/main.c | 1 + 5 files changed, 56 insertions(+), 86 deletions(-) (limited to 'sysdep/unix') diff --git a/lib/timer.c b/lib/timer.c index 6efcadb4..ff1fb5ef 100644 --- a/lib/timer.c +++ b/lib/timer.c @@ -32,6 +32,7 @@ #include "nest/bird.h" +#include "lib/coro.h" #include "lib/heap.h" #include "lib/resource.h" #include "lib/timer.h" @@ -45,22 +46,10 @@ struct timeloop main_timeloop; /* Data accessed and modified from proto/bfd/io.c */ _Thread_local struct timeloop *local_timeloop; -void wakeup_kick_current(void); - -btime -current_time(void) -{ - return local_timeloop->last_time; -} +_Atomic btime last_time; +_Atomic btime real_time; -btime -current_real_time(void) -{ - if (!local_timeloop->real_time) - times_update_real_time(local_timeloop); - - return local_timeloop->real_time; -} +void wakeup_kick_current(void); #define TIMER_LESS(a,b) ((a)->expires < (b)->expires) @@ -164,8 +153,6 @@ tm_stop(timer *t) void timers_init(struct timeloop *loop, pool *p) { - times_init(loop); - BUFFER_INIT(loop->timers, p, 4); BUFFER_PUSH(loop->timers) = NULL; } @@ -178,8 +165,8 @@ timers_fire(struct timeloop *loop) btime base_time; timer *t; - times_update(loop); - base_time = loop->last_time; + times_update(); + base_time = current_time(); while (t = timers_first(loop)) { @@ -190,8 +177,8 @@ timers_fire(struct timeloop *loop) { btime when = t->expires + t->recurrent; - if (when <= loop->last_time) - when = loop->last_time + t->recurrent; + if (when <= base_time) + when = base_time + t->recurrent; if (t->randomize) when += random() % (t->randomize + 1); diff --git a/lib/timer.h b/lib/timer.h index bc568ee6..b201b8c8 100644 --- a/lib/timer.h +++ b/lib/timer.h @@ -14,6 +14,10 @@ #include "lib/buffer.h" #include "lib/resource.h" +#include + +extern _Atomic btime last_time; +extern _Atomic btime real_time; typedef struct timer { @@ -31,8 +35,6 @@ typedef struct timer struct timeloop { BUFFER_(timer *) timers; - btime last_time; - btime real_time; }; static inline uint timers_count(struct timeloop *loop) @@ -44,8 +46,8 @@ static inline timer *timers_first(struct timeloop *loop) extern struct timeloop main_timeloop; extern _Thread_local struct timeloop *local_timeloop; -btime current_time(void); -btime current_real_time(void); +#define current_time() atomic_load_explicit(&last_time, memory_order_acquire) +#define current_real_time() atomic_load_explicit(&real_time, memory_order_acquire) //#define now (current_time() TO_S) //#define now_real (current_real_time() TO_S) @@ -95,9 +97,7 @@ tm_start_max(timer *t, btime after) } /* In sysdep code */ -void times_init(struct timeloop *loop); -void times_update(struct timeloop *loop); -void times_update_real_time(struct timeloop *loop); +void times_update(void); /* For I/O loop */ void timers_init(struct timeloop *loop, pool *p); diff --git a/proto/bfd/io.c b/proto/bfd/io.c index 8fdc84fb..c5f1e024 100644 --- a/proto/bfd/io.c +++ b/proto/bfd/io.c @@ -172,7 +172,7 @@ events_init(struct birdloop *loop) static void events_fire(struct birdloop *loop) { - times_update(&loop->time); + times_update(); ev_run_list(&loop->event_list); } @@ -332,7 +332,7 @@ sockets_fire(struct birdloop *loop) sock **psk = loop->poll_sk.data; int poll_num = loop->poll_fd.used - 1; - times_update(&loop->time); + times_update(); /* Last fd is internal wakeup fd */ if (pfd[poll_num].revents & POLLIN) @@ -365,7 +365,7 @@ sockets_fire(struct birdloop *loop) * Birdloop */ -static void * birdloop_main(void *arg); +static void *birdloop_main(void *arg); struct birdloop * birdloop_new(void) @@ -461,7 +461,7 @@ birdloop_main(void *arg) events_fire(loop); timers_fire(&loop->time); - times_update(&loop->time); + times_update(); if (events_waiting(loop)) timeout = 0; else if (t = timers_first(&loop->time)) diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index 40841ea4..90bb5d64 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -123,55 +123,50 @@ rf_fileno(struct rfile *f) btime boot_time; + void -times_init(struct timeloop *loop) +times_update(void) { struct timespec ts; int rv; + btime old_time = current_time(); + btime old_real_time = current_real_time(); + rv = clock_gettime(CLOCK_MONOTONIC, &ts); if (rv < 0) die("Monotonic clock is missing"); if ((ts.tv_sec < 0) || (((u64) ts.tv_sec) > ((u64) 1 << 40))) log(L_WARN "Monotonic clock is crazy"); - - loop->last_time = ts.tv_sec S + ts.tv_nsec NS; - loop->real_time = 0; -} - -void -times_update(struct timeloop *loop) -{ - struct timespec ts; - int rv; - - rv = clock_gettime(CLOCK_MONOTONIC, &ts); - if (rv < 0) - die("clock_gettime: %m"); - + btime new_time = ts.tv_sec S + ts.tv_nsec NS; - if (new_time < loop->last_time) + if (new_time < old_time) log(L_ERR "Monotonic clock is broken"); - loop->last_time = new_time; - loop->real_time = 0; -} - -void -times_update_real_time(struct timeloop *loop) -{ - struct timespec ts; - int rv; - rv = clock_gettime(CLOCK_REALTIME, &ts); if (rv < 0) die("clock_gettime: %m"); - loop->real_time = ts.tv_sec S + ts.tv_nsec NS; -} + btime new_real_time = ts.tv_sec S + ts.tv_nsec NS; + if (!atomic_compare_exchange_strong_explicit( + &last_time, + &old_time, + new_time, + memory_order_acq_rel, + memory_order_relaxed)) + DBG("Time update collision: last_time"); + + if (!atomic_compare_exchange_strong_explicit( + &real_time, + &old_real_time, + new_real_time, + memory_order_acq_rel, + memory_order_relaxed)) + DBG("Time update collision: real_time"); +} /** * DOC: Sockets @@ -2017,30 +2012,17 @@ struct event_log_entry static struct event_log_entry event_log[EVENT_LOG_LENGTH]; static struct event_log_entry *event_open; static int event_log_pos, event_log_num, watchdog_active; -static btime last_time; +static btime last_io_time; static btime loop_time; static void io_update_time(void) { - struct timespec ts; - int rv; - - /* - * This is third time-tracking procedure (after update_times() above and - * times_update() in BFD), dedicated to internal event log and latency - * tracking. Hopefully, we consolidate these sometimes. - */ - - rv = clock_gettime(CLOCK_MONOTONIC, &ts); - if (rv < 0) - die("clock_gettime: %m"); - - last_time = ts.tv_sec S + ts.tv_nsec NS; + last_io_time = current_time(); if (event_open) { - event_open->duration = last_time - event_open->timestamp; + event_open->duration = last_io_time - event_open->timestamp; if (event_open->duration > config->latency_limit) log(L_WARN "Event 0x%p 0x%p took %d ms", @@ -2069,7 +2051,7 @@ io_log_event(void *hook, void *data) en->hook = hook; en->data = data; - en->timestamp = last_time; + en->timestamp = last_io_time; en->duration = 0; event_log_num++; @@ -2097,14 +2079,14 @@ io_log_dump(void) struct event_log_entry *en = event_log + (event_log_pos + i) % EVENT_LOG_LENGTH; if (en->hook) log(L_DEBUG " Event 0x%p 0x%p at %8d for %d ms", en->hook, en->data, - (int) ((last_time - en->timestamp) TO_MS), (int) (en->duration TO_MS)); + (int) ((last_io_time - en->timestamp) TO_MS), (int) (en->duration TO_MS)); } } void watchdog_sigalrm(int sig UNUSED) { - /* Update last_time and duration, but skip latency check */ + /* Update last_io_time and duration, but skip latency check */ config->latency_limit = 0xffffffff; io_update_time(); @@ -2117,7 +2099,7 @@ watchdog_start1(void) { io_update_time(); - loop_time = last_time; + loop_time = last_io_time; } static inline void @@ -2125,7 +2107,7 @@ watchdog_start(void) { io_update_time(); - loop_time = last_time; + loop_time = last_io_time; event_log_num = 0; if (config->watchdog_timeout) @@ -2146,7 +2128,7 @@ watchdog_stop(void) watchdog_active = 0; } - btime duration = last_time - loop_time; + btime duration = last_io_time - loop_time; if (duration > config->watchdog_warning) log(L_WARN "I/O loop cycle took %d ms for %d events", (int) (duration TO_MS), event_log_num); @@ -2202,7 +2184,7 @@ io_loop(void) watchdog_start1(); for(;;) { - times_update(&main_timeloop); + times_update(); events = ev_run_list(&global_event_list); events = ev_run_list_limited(&global_work_list, WORK_EVENTS_MAX) || events; timers_fire(&main_timeloop); @@ -2212,7 +2194,7 @@ io_loop(void) poll_tout = (events ? 0 : 3000); /* Time in milliseconds */ if (t = timers_first(&main_timeloop)) { - times_update(&main_timeloop); + times_update(); timeout = (tm_remains(t) TO_MS) + 1; poll_tout = MIN(poll_tout, timeout); } @@ -2302,7 +2284,7 @@ io_loop(void) continue; } - times_update(&main_timeloop); + times_update(); /* guaranteed to be non-empty */ current_sock = SKIP_BACK(sock, n, HEAD(sock_list)); diff --git a/sysdep/unix/main.c b/sysdep/unix/main.c index dabfc554..d35424ff 100644 --- a/sysdep/unix/main.c +++ b/sysdep/unix/main.c @@ -903,6 +903,7 @@ main(int argc, char **argv) dmalloc_debug(0x2f03d00); #endif + times_update(); resource_sys_init(); parse_args(argc, argv); log_switch(1, NULL, NULL); -- cgit v1.2.3 From 94eb0858c2b938549d9d1703c872c6149901e7dd Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Sat, 19 Jun 2021 20:50:18 +0200 Subject: Converting the former BFD loop to a universal IO loop and protocol loop. There is a simple universal IO loop, taking care of events, timers and sockets. Primarily, one instance of a protocol should use exactly one IO loop to do all its work, as is now done in BFD. Contrary to previous versions, the loop is now launched and cleaned by the nest/proto.c code, allowing for a protocol to just request its own loop by setting the loop's lock order in config higher than the_bird. It is not supported nor checked if any protocol changed the requested lock order in reconfigure. No protocol should do it at all. --- filter/filter_test.c | 1 + lib/birdlib.h | 1 + lib/coro.h | 5 +- lib/event.c | 131 ++++++++++++-------- lib/event.h | 41 ++++++- lib/event_test.c | 7 +- lib/flowspec_test.c | 3 + lib/io-loop.h | 54 +++++++++ lib/lists.h | 12 ++ lib/locking.h | 21 +++- lib/socket.h | 3 + lib/timer.c | 53 ++++---- lib/timer.h | 28 +++-- nest/a-path_test.c | 3 + nest/a-set_test.c | 4 + nest/config.Y | 1 + nest/proto.c | 173 +++++++++++++++++--------- nest/protocol.h | 7 +- nest/route.h | 3 + nest/rt-table.c | 18 ++- proto/bfd/bfd.c | 166 ++++++++++++++----------- proto/bfd/bfd.h | 8 +- proto/bfd/config.Y | 1 + proto/bfd/packets.c | 4 +- proto/bgp/bgp.c | 1 + sysdep/unix/coroutine.c | 21 +++- sysdep/unix/io-loop.c | 313 ++++++++++++++++++++++++++---------------------- sysdep/unix/io-loop.h | 41 +++---- sysdep/unix/io.c | 85 ++++++++----- sysdep/unix/main.c | 5 +- sysdep/unix/unix.h | 1 - test/bt-utils.c | 7 +- 32 files changed, 780 insertions(+), 442 deletions(-) create mode 100644 lib/io-loop.h (limited to 'sysdep/unix') diff --git a/filter/filter_test.c b/filter/filter_test.c index 7e4af092..2a0b5431 100644 --- a/filter/filter_test.c +++ b/filter/filter_test.c @@ -72,6 +72,7 @@ int main(int argc, char *argv[]) { bt_init(argc, argv); + bt_bird_init(); bt_assert_hook = bt_assert_filter; diff --git a/lib/birdlib.h b/lib/birdlib.h index 3dc39d19..385bf75c 100644 --- a/lib/birdlib.h +++ b/lib/birdlib.h @@ -71,6 +71,7 @@ static inline int u64_cmp(u64 i1, u64 i2) /* Macros for gcc attributes */ #define NORET __attribute__((noreturn)) +#define USE_RESULT __atribute__((warn_unused_result)) #define UNUSED __attribute__((unused)) #define PACKED __attribute__((packed)) #define NONNULL(...) __attribute__((nonnull((__VA_ARGS__)))) diff --git a/lib/coro.h b/lib/coro.h index 51712b36..17ccff89 100644 --- a/lib/coro.h +++ b/lib/coro.h @@ -2,7 +2,7 @@ * BIRD Coroutines * * (c) 2017 Martin Mares - * (c) 2020 Maria Matejka + * (c) 2020-2021 Maria Matejka * * Can be freely distributed and used under the terms of the GNU GPL. */ @@ -22,5 +22,8 @@ struct coroutine; */ struct coroutine *coro_run(pool *, void (*entry)(void *), void *data); +/* Get self. */ +extern _Thread_local struct coroutine *this_coro; + #endif diff --git a/lib/event.c b/lib/event.c index 273447e0..6c5c8b14 100644 --- a/lib/event.c +++ b/lib/event.c @@ -19,8 +19,14 @@ * events in them and explicitly ask to run them. */ +#undef LOCAL_DEBUG + #include "nest/bird.h" #include "lib/event.h" +#include "lib/locking.h" +#include "lib/io-loop.h" + +extern _Thread_local struct coroutine *this_coro; event_list global_event_list; event_list global_work_list; @@ -28,11 +34,16 @@ event_list global_work_list; inline void ev_postpone(event *e) { + event_list *el = e->list; + if (!el) + return; + + ASSERT_DIE(birdloop_inside(el->loop)); + + LOCK_DOMAIN(event, el->lock); if (ev_active(e)) - { - rem_node(&e->n); - e->n.next = NULL; - } + rem_node(&e->n); + UNLOCK_DOMAIN(event, el->lock); } static void @@ -95,40 +106,25 @@ ev_run(event *e) * list @l which can be run by calling ev_run_list(). */ inline void -ev_enqueue(event_list *l, event *e) +ev_send(event_list *l, event *e) { - ev_postpone(e); - add_tail(l, &e->n); -} + DBG("ev_send(%p, %p)\n", l, e); + ASSERT_DIE(e->hook); + ASSERT_DIE(!e->list || (e->list == l) || (e->list->loop == l->loop)); -/** - * ev_schedule - schedule an event - * @e: an event - * - * This function schedules an event by enqueueing it to a system-wide - * event list which is run by the platform dependent code whenever - * appropriate. - */ -void -ev_schedule(event *e) -{ - ev_enqueue(&global_event_list, e); -} + e->list = l; -/** - * ev_schedule_work - schedule a work-event. - * @e: an event - * - * This function schedules an event by enqueueing it to a system-wide work-event - * list which is run by the platform dependent code whenever appropriate. This - * is designated for work-events instead of regular events. They are executed - * less often in order to not clog I/O loop. - */ -void -ev_schedule_work(event *e) -{ - if (!ev_active(e)) - add_tail(&global_work_list, &e->n); + LOCK_DOMAIN(event, l->lock); + if (enlisted(&e->n)) + { + UNLOCK_DOMAIN(event, l->lock); + return; + } + + add_tail(&l->events, &e->n); + UNLOCK_DOMAIN(event, l->lock); + + birdloop_ping(l->loop); } void io_log_event(void *hook, void *data); @@ -142,35 +138,64 @@ void io_log_event(void *hook, void *data); int ev_run_list(event_list *l) { + const _Bool legacy = LEGACY_EVENT_LIST(l); + + if (legacy) + ASSERT_THE_BIRD_LOCKED; + node *n; - list tmp_list; + list tmp_list; init_list(&tmp_list); - add_tail_list(&tmp_list, l); - init_list(l); + + /* Move the event list contents to a local list to avoid executing repeatedly added events */ + LOCK_DOMAIN(event, l->lock); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + UNLOCK_DOMAIN(event, l->lock); + WALK_LIST_FIRST(n, tmp_list) { event *e = SKIP_BACK(event, n, n); - /* This is ugly hack, we want to log just events executed from the main I/O loop */ - if ((l == &global_event_list) || (l == &global_work_list)) + if (legacy) + { + /* The legacy way of event execution */ io_log_event(e->hook, e->data); - - ev_run(e); + ev_postpone(e); + e->hook(e->data); + } + else + { + // io_log_event(e->hook, e->data); /* TODO: add support for event logging in other io loops */ + ASSERT_DIE(e->list == l); + LOCK_DOMAIN(event, l->lock); + rem_node(&e->n); + UNLOCK_DOMAIN(event, l->lock); + e->hook(e->data); + } } - return !EMPTY_LIST(*l); + LOCK_DOMAIN(event, l->lock); + int repeat = ! EMPTY_LIST(l->events); + UNLOCK_DOMAIN(event, l->lock); + return repeat; } int ev_run_list_limited(event_list *l, uint limit) { + ASSERT_DIE(LEGACY_EVENT_LIST(l)); + ASSERT_THE_BIRD_LOCKED; + node *n; list tmp_list; + LOCK_DOMAIN(event, l->lock); init_list(&tmp_list); - add_tail_list(&tmp_list, l); - init_list(l); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + UNLOCK_DOMAIN(event, l->lock); WALK_LIST_FIRST(n, tmp_list) { @@ -179,21 +204,23 @@ ev_run_list_limited(event_list *l, uint limit) if (!limit) break; - /* This is ugly hack, we want to log just events executed from the main I/O loop */ - if ((l == &global_event_list) || (l == &global_work_list)) - io_log_event(e->hook, e->data); + io_log_event(e->hook, e->data); ev_run(e); limit--; } + LOCK_DOMAIN(event, l->lock); if (!EMPTY_LIST(tmp_list)) { /* Attach new items after the unprocessed old items */ - add_tail_list(&tmp_list, l); - init_list(l); - add_tail_list(l, &tmp_list); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + add_tail_list(&l->events, &tmp_list); } - return !EMPTY_LIST(*l); + int repeat = ! EMPTY_LIST(l->events); + UNLOCK_DOMAIN(event, l->lock); + + return repeat; } diff --git a/lib/event.h b/lib/event.h index 5f3b78d8..6c358f84 100644 --- a/lib/event.h +++ b/lib/event.h @@ -10,33 +10,62 @@ #define _BIRD_EVENT_H_ #include "lib/resource.h" +#include "lib/locking.h" + +#include + +DEFINE_DOMAIN(event); typedef struct event { resource r; void (*hook)(void *); void *data; node n; /* Internal link */ + struct event_list *list; /* List where this event is put in */ } event; -typedef list event_list; +typedef struct event_list { + list events; + pool *pool; + struct birdloop *loop; + DOMAIN(event) lock; +} event_list; extern event_list global_event_list; extern event_list global_work_list; event *ev_new(pool *); void ev_run(event *); -#define ev_init_list(el) init_list(el) -void ev_enqueue(event_list *, event *); -void ev_schedule(event *); -void ev_schedule_work(event *); + +static inline void ev_init_list(event_list *el, struct birdloop *loop, const char *name) +{ + init_list(&el->events); + el->loop = loop; + el->lock = DOMAIN_NEW(event, name); +} + +void ev_send(event_list *, event *); +#define ev_send_loop(l, e) ev_send(birdloop_event_list((l)), (e)) + +#define ev_schedule(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_event_list, (e)); }) +#define ev_schedule_work(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_work_list, (e)); }) + void ev_postpone(event *); int ev_run_list(event_list *); int ev_run_list_limited(event_list *, uint); +#define LEGACY_EVENT_LIST(l) (((l) == &global_event_list) || ((l) == &global_work_list)) + +_Bool birdloop_inside(struct birdloop *loop); + static inline int ev_active(event *e) { - return e->n.next != NULL; + if (e->list == NULL) + return 0; + + ASSERT_DIE(birdloop_inside(e->list->loop)); + return enlisted(&e->n); } static inline event* diff --git a/lib/event_test.c b/lib/event_test.c index e1215bba..9dda3e2a 100644 --- a/lib/event_test.c +++ b/lib/event_test.c @@ -48,14 +48,17 @@ init_event_check_points(void) event_check_points[i] = 0; } +void resource_sys_init(void); + static int t_ev_run_list(void) { int i; + resource_sys_init(); resource_init(); olock_init(); - timer_init(); + birdloop_init(); io_init(); rt_init(); if_init(); @@ -82,7 +85,9 @@ main(int argc, char *argv[]) { bt_init(argc, argv); + the_bird_lock(); bt_test_suite(t_ev_run_list, "Schedule and run 3 events in right order."); + the_bird_unlock(); return bt_exit_value(); } diff --git a/lib/flowspec_test.c b/lib/flowspec_test.c index ed4afe51..f7f70982 100644 --- a/lib/flowspec_test.c +++ b/lib/flowspec_test.c @@ -666,10 +666,13 @@ t_formatting6(void) return 1; } +void resource_sys_init(void); + int main(int argc, char *argv[]) { bt_init(argc, argv); + resource_sys_init(); bt_test_suite(t_read_length, "Testing get NLRI length"); bt_test_suite(t_write_length, "Testing set NLRI length"); diff --git a/lib/io-loop.h b/lib/io-loop.h new file mode 100644 index 00000000..25f1b2a3 --- /dev/null +++ b/lib/io-loop.h @@ -0,0 +1,54 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_IO_LOOP_H_ +#define _BIRD_IO_LOOP_H_ + +#include "nest/bird.h" +#include "lib/lists.h" +#include "lib/locking.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/socket.h" + +void sk_start(sock *s); +void sk_stop(sock *s); +void sk_reloop(sock *s, struct birdloop *loop); + +extern struct birdloop main_birdloop; + +/* Start a new birdloop owned by given pool and domain */ +struct birdloop *birdloop_new(pool *p, uint order, const char *name); + +/* Stop the loop. At the end, the @stopped callback is called unlocked in tail + * position to finish cleanup. Run birdloop_free() from that callback to free + * the loop itself. */ +void birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data); +void birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data); +void birdloop_free(struct birdloop *loop); + +/* Get birdloop's event list */ +event_list *birdloop_event_list(struct birdloop *loop); + +/* Get birdloop's time heap */ +struct timeloop *birdloop_time_loop(struct birdloop *loop); + +/* Enter and exit the birdloop */ +void birdloop_enter(struct birdloop *loop); +void birdloop_leave(struct birdloop *loop); + +_Bool birdloop_inside(struct birdloop *loop); + +void birdloop_mask_wakeups(struct birdloop *loop); +void birdloop_unmask_wakeups(struct birdloop *loop); + +void birdloop_link(struct birdloop *loop); +void birdloop_unlink(struct birdloop *loop); + +void birdloop_ping(struct birdloop *loop); + +void birdloop_init(void); +#endif /* _BIRD_IO_LOOP_H_ */ diff --git a/lib/lists.h b/lib/lists.h index 64b4a981..dc49ec8a 100644 --- a/lib/lists.h +++ b/lib/lists.h @@ -68,6 +68,18 @@ typedef union list { /* In fact two overlayed nodes */ #define EMPTY_LIST(list) (!(list).head->next) +static inline _Bool +enlisted(node *n) +{ + switch ((!!n->next) + (!!n->prev)) + { + case 0: return 0; + case 2: return 1; + case 1: bug("Garbled event list node"); + } + + bug("Maths is broken. And you should see a new heaven and a new earth: for the first heaven and the first earth had been passed away."); +} #ifndef _BIRD_LISTS_C_ #define LIST_INLINE static inline diff --git a/lib/locking.h b/lib/locking.h index eef60154..ab5c06af 100644 --- a/lib/locking.h +++ b/lib/locking.h @@ -14,6 +14,9 @@ struct domain_generic; /* Here define the global lock order; first to last. */ struct lock_order { struct domain_generic *the_bird; + struct domain_generic *proto; + struct domain_generic *rtable; + struct domain_generic *event; }; extern _Thread_local struct lock_order locking_stack; @@ -21,24 +24,40 @@ extern _Thread_local struct domain_generic **last_locked; #define DOMAIN(type) struct domain__##type #define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; } +#define DOMAIN_ORDER(type) OFFSETOF(struct lock_order, type) -#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name, OFFSETOF(struct lock_order, type)) } +#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name, DOMAIN_ORDER(type)) } struct domain_generic *domain_new(const char *name, uint order); +#define DOMAIN_FREE(type, d) domain_free((d).type) +void domain_free(struct domain_generic *); + #define DOMAIN_NULL(type) (DOMAIN(type)) {} #define LOCK_DOMAIN(type, d) do_lock(((d).type), &(locking_stack.type)) #define UNLOCK_DOMAIN(type, d) do_unlock(((d).type), &(locking_stack.type)) +#define DOMAIN_IS_LOCKED(type, d) (((d).type) == (locking_stack.type)) +#define DG_IS_LOCKED(d) ((d) == *(DG_LSP(d))) + /* Internal for locking */ void do_lock(struct domain_generic *dg, struct domain_generic **lsp); void do_unlock(struct domain_generic *dg, struct domain_generic **lsp); +uint dg_order(struct domain_generic *dg); + +#define DG_LSP(d) ((struct domain_generic **) (((void *) &locking_stack) + dg_order(d))) +#define DG_LOCK(d) do_lock(d, DG_LSP(d)) +#define DG_UNLOCK(d) do_unlock(d, DG_LSP(d)) + /* Use with care. To be removed in near future. */ DEFINE_DOMAIN(the_bird); extern DOMAIN(the_bird) the_bird_domain; #define the_bird_lock() LOCK_DOMAIN(the_bird, the_bird_domain) #define the_bird_unlock() UNLOCK_DOMAIN(the_bird, the_bird_domain) +#define the_bird_locked() DOMAIN_IS_LOCKED(the_bird, the_bird_domain) + +#define ASSERT_THE_BIRD_LOCKED ({ if (!the_bird_locked()) bug("The BIRD lock must be locked here: %s:%d", __FILE__, __LINE__); }) #endif diff --git a/lib/socket.h b/lib/socket.h index 96fedeeb..5bdab7f3 100644 --- a/lib/socket.h +++ b/lib/socket.h @@ -12,6 +12,7 @@ #include #include "lib/resource.h" +#include "lib/event.h" #ifdef HAVE_LIBSSH #define LIBSSH_LEGACY_0_4 #include @@ -79,6 +80,7 @@ typedef struct birdsock { const char *password; /* Password for MD5 authentication */ const char *err; /* Error message */ struct ssh_sock *ssh; /* Used in SK_SSH */ + struct event reloop; /* Reloop event */ } sock; sock *sock_new(pool *); /* Allocate new socket */ @@ -128,6 +130,7 @@ extern int sk_priority_control; /* Suggested priority for control traffic, shou #define SKF_TRUNCATED 0x200 /* Received packet was truncated, set by IO layer */ #define SKF_HDRINCL 0x400 /* Used internally */ #define SKF_PKTINFO 0x800 /* Used internally */ +#define SKF_PASSIVE_THREAD 0x1000 /* Child sockets used in thread, do not add to main loop */ /* * Socket types SA SP DA DP IF TTL SendTo (?=may, -=must not, *=must) diff --git a/lib/timer.c b/lib/timer.c index ff1fb5ef..eb7ea690 100644 --- a/lib/timer.c +++ b/lib/timer.c @@ -37,15 +37,8 @@ #include "lib/resource.h" #include "lib/timer.h" - -struct timeloop main_timeloop; - - #include -/* Data accessed and modified from proto/bfd/io.c */ -_Thread_local struct timeloop *local_timeloop; - _Atomic btime last_time; _Atomic btime real_time; @@ -76,7 +69,7 @@ tm_dump(resource *r) if (t->recurrent) debug("recur %d, ", t->recurrent); if (t->expires) - debug("expires in %d ms)\n", (t->expires - current_time()) TO_MS); + debug("in loop %p expires in %d ms)\n", t->loop, (t->expires - current_time()) TO_MS); else debug("inactive)\n"); } @@ -99,8 +92,8 @@ tm_new(pool *p) return t; } -void -tm_set(timer *t, btime when) +static void +tm_set_in_tl(timer *t, btime when, struct timeloop *local_timeloop) { uint tc = timers_count(local_timeloop); @@ -122,17 +115,17 @@ tm_set(timer *t, btime when) HEAP_DECREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); } -#ifdef CONFIG_BFD - /* Hack to notify BFD loops */ - if ((local_timeloop != &main_timeloop) && (t->index == 1)) - wakeup_kick_current(); -#endif + t->loop = local_timeloop; + + if ((t->index == 1) && (local_timeloop->coro != this_coro)) + birdloop_ping(local_timeloop->loop); } void -tm_start(timer *t, btime after) +tm_set_in(timer *t, btime when, struct birdloop *loop) { - tm_set(t, current_time() + MAX(after, 0)); + ASSERT_DIE(birdloop_inside(loop)); + tm_set_in_tl(t, when, birdloop_time_loop(loop)); } void @@ -141,18 +134,23 @@ tm_stop(timer *t) if (!t->expires) return; - uint tc = timers_count(local_timeloop); + TLOCK_TIMER_ASSERT(t->loop); - HEAP_DELETE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); - BUFFER_POP(local_timeloop->timers); + uint tc = timers_count(t->loop); + + HEAP_DELETE(t->loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + BUFFER_POP(t->loop->timers); t->index = -1; t->expires = 0; + t->loop = NULL; } void timers_init(struct timeloop *loop, pool *p) { + TLOCK_TIMER_ASSERT(loop); + BUFFER_INIT(loop->timers, p, 4); BUFFER_PUSH(loop->timers) = NULL; } @@ -160,8 +158,10 @@ timers_init(struct timeloop *loop, pool *p) void io_log_event(void *hook, void *data); void -timers_fire(struct timeloop *loop) +timers_fire(struct timeloop *loop, int io_log) { + TLOCK_TIMER_ASSERT(loop); + btime base_time; timer *t; @@ -183,26 +183,19 @@ timers_fire(struct timeloop *loop) if (t->randomize) when += random() % (t->randomize + 1); - tm_set(t, when); + tm_set_in_tl(t, when, loop); } else tm_stop(t); /* This is ugly hack, we want to log just timers executed from the main I/O loop */ - if (loop == &main_timeloop) + if (io_log) io_log_event(t->hook, t->data); t->hook(t); } } -void -timer_init(void) -{ - timers_init(&main_timeloop, &root_pool); - local_timeloop = &main_timeloop; -} - /** * tm_parse_time - parse a date and time diff --git a/lib/timer.h b/lib/timer.h index b201b8c8..04544ace 100644 --- a/lib/timer.h +++ b/lib/timer.h @@ -12,6 +12,8 @@ #include "nest/bird.h" #include "lib/buffer.h" +#include "lib/io-loop.h" +#include "lib/locking.h" #include "lib/resource.h" #include @@ -29,22 +31,27 @@ typedef struct timer uint randomize; /* Amount of randomization */ uint recurrent; /* Timer recurrence */ + struct timeloop *loop; /* Loop where the timer is active */ + int index; } timer; struct timeloop { BUFFER_(timer *) timers; + struct domain_generic *domain; + struct birdloop *loop; + struct coroutine *coro; }; +#define TLOCK_TIMER_ASSERT(loop) ASSERT_DIE((loop)->domain && DG_IS_LOCKED((loop)->domain)) +#define TLOCK_LOCAL_ASSERT(loop) ASSERT_DIE(!(loop)->domain || DG_IS_LOCKED((loop)->domain)) + static inline uint timers_count(struct timeloop *loop) -{ return loop->timers.used - 1; } +{ TLOCK_TIMER_ASSERT(loop); return loop->timers.used - 1; } static inline timer *timers_first(struct timeloop *loop) -{ return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } - -extern struct timeloop main_timeloop; -extern _Thread_local struct timeloop *local_timeloop; +{ TLOCK_TIMER_ASSERT(loop); return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } #define current_time() atomic_load_explicit(&last_time, memory_order_acquire) #define current_real_time() atomic_load_explicit(&real_time, memory_order_acquire) @@ -54,10 +61,13 @@ extern _Thread_local struct timeloop *local_timeloop; extern btime boot_time; timer *tm_new(pool *p); -void tm_set(timer *t, btime when); -void tm_start(timer *t, btime after); +#define tm_set(t, when) tm_set_in((t), (when), &main_birdloop) +#define tm_start(t, after) tm_start_in((t), (after), &main_birdloop) void tm_stop(timer *t); +void tm_set_in(timer *t, btime when, struct birdloop *loop); +#define tm_start_in(t, after, loop) tm_set_in((t), (current_time() + MAX_((after), 0)), loop) + static inline int tm_active(timer *t) { @@ -101,9 +111,7 @@ void times_update(void); /* For I/O loop */ void timers_init(struct timeloop *loop, pool *p); -void timers_fire(struct timeloop *loop); - -void timer_init(void); +void timers_fire(struct timeloop *loop, int io_log); struct timeformat { diff --git a/nest/a-path_test.c b/nest/a-path_test.c index 9ed0a786..2e6e4956 100644 --- a/nest/a-path_test.c +++ b/nest/a-path_test.c @@ -204,10 +204,13 @@ t_as_path_converting(void) } #endif +void resource_sys_init(void); + int main(int argc, char *argv[]) { bt_init(argc, argv); + resource_sys_init(); bt_test_suite(t_as_path_match, "Testing AS path matching and some a-path utilities."); bt_test_suite(t_path_format, "Testing formating as path into byte buffer"); diff --git a/nest/a-set_test.c b/nest/a-set_test.c index 96b6a727..efd1b67d 100644 --- a/nest/a-set_test.c +++ b/nest/a-set_test.c @@ -240,10 +240,14 @@ t_set_ec_delete(void) return 1; } + +void resource_sys_init(void); + int main(int argc, char *argv[]) { bt_init(argc, argv); + resource_sys_init(); bt_test_suite(t_set_int_contains, "Testing sets of integers: contains, get_data"); bt_test_suite(t_set_int_format, "Testing sets of integers: format"); diff --git a/nest/config.Y b/nest/config.Y index 92f1aad2..6e7689ed 100644 --- a/nest/config.Y +++ b/nest/config.Y @@ -434,6 +434,7 @@ proto: dev_proto '}' ; dev_proto_start: proto_start DIRECT { this_proto = proto_config_new(&proto_device, $1); init_list(&DIRECT_CFG->iface_list); + this_proto->late_if_feed = 1; } ; diff --git a/nest/proto.c b/nest/proto.c index fae0647a..ac0fb232 100644 --- a/nest/proto.c +++ b/nest/proto.c @@ -15,6 +15,7 @@ #include "lib/event.h" #include "lib/timer.h" #include "lib/string.h" +#include "lib/coro.h" #include "conf/conf.h" #include "nest/route.h" #include "nest/iface.h" @@ -58,7 +59,28 @@ static void channel_feed_end(struct channel *c); static void channel_export_stopped(struct rt_export_request *req); static inline int proto_is_done(struct proto *p) -{ return (p->proto_state == PS_DOWN) && (p->active_channels == 0); } +{ return (p->proto_state == PS_DOWN) && proto_is_inactive(p); } + +static inline event_list *proto_event_list(struct proto *p) +{ return p->loop == &main_birdloop ? &global_event_list : birdloop_event_list(p->loop); } + +static inline event_list *proto_work_list(struct proto *p) +{ return p->loop == &main_birdloop ? &global_work_list : birdloop_event_list(p->loop); } + +static inline void proto_send_event(struct proto *p) +{ ev_send(proto_event_list(p), p->event); } + +#define PROTO_ENTER_FROM_MAIN(p) ({ \ + ASSERT_DIE(birdloop_inside(&main_birdloop)); \ + struct birdloop *_loop = (p)->loop; \ + if (_loop != &main_birdloop) birdloop_enter(_loop); \ + _loop; \ + }) + +#define PROTO_LEAVE_FROM_MAIN(loop) ({ if (loop != &main_birdloop) birdloop_leave(loop); }) + +#define PROTO_LOCKED_FROM_MAIN(p) for (struct birdloop *_proto_loop = PROTO_ENTER_FROM_MAIN(p); _proto_loop; PROTO_LEAVE_FROM_MAIN(_proto_loop), (_proto_loop = NULL)) + static inline int channel_is_active(struct channel *c) { return (c->channel_state != CS_DOWN); } @@ -473,6 +495,7 @@ channel_start_export(struct channel *c) c->out_req = (struct rt_export_request) { .name = rn, + .list = proto_work_list(c->proto), .trace_routes = c->debug | c->proto->debug, .dump_req = channel_dump_export_req, .log_state_change = channel_export_log_state_change, @@ -517,7 +540,7 @@ channel_check_stopped(struct channel *c) return; channel_set_state(c, CS_DOWN); - ev_schedule(c->proto->event); + proto_send_event(c->proto); break; case CS_PAUSE: @@ -853,6 +876,7 @@ channel_setup_in_table(struct channel *c, int best) }; c->in_table->get = (struct rt_export_request) { .name = cat->name, + .list = proto_work_list(c->proto), .trace_routes = c->debug | c->proto->debug, .dump_req = channel_in_get_dump_req, .log_state_change = channel_get_log_state_change, @@ -895,6 +919,7 @@ channel_setup_out_table(struct channel *c) }; c->out_table->get = (struct rt_export_request) { .name = cat->name, + .list = proto_work_list(c->proto), .trace_routes = c->debug | c->proto->debug, .dump_req = channel_out_get_dump_req, .log_state_change = channel_get_log_state_change, @@ -997,7 +1022,7 @@ channel_do_down(struct channel *c) /* Schedule protocol shutddown */ if (proto_is_done(c->proto)) - ev_schedule(c->proto->event); + proto_send_event(c->proto); } void @@ -1085,9 +1110,12 @@ channel_request_table_feeding(struct channel *c) void channel_request_feeding(struct channel *c) { + if (c->gr_wait || !c->proto->rt_notify) + return; + CD(c, "Refeed requested"); - ASSERT(c->out_req.hook); + ASSERT_DIE(c->out_req.hook); if (c->out_table) channel_aux_request_refeed(c->out_table); @@ -1331,17 +1359,35 @@ proto_configure_channel(struct proto *p, struct channel **pc, struct channel_con return 1; } +static void +proto_cleanup(struct proto *p) +{ + rfree(p->pool); + p->pool = NULL; + + p->active = 0; + proto_log_state_change(p); + proto_rethink_goal(p); +} static void -proto_event(void *ptr) +proto_loop_stopped(void *ptr) { struct proto *p = ptr; - if (p->do_start) - { - if_feed_baby(p); - p->do_start = 0; - } + birdloop_enter(&main_birdloop); + + p->loop = &main_birdloop; + p->event->list = NULL; + proto_cleanup(p); + + birdloop_leave(&main_birdloop); +} + +static void +proto_event(void *ptr) +{ + struct proto *p = ptr; if (p->do_stop) { @@ -1351,14 +1397,10 @@ proto_event(void *ptr) } if (proto_is_done(p)) - { - rfree(p->pool); - p->pool = NULL; - - p->active = 0; - proto_log_state_change(p); - proto_rethink_goal(p); - } + if (p->loop != &main_birdloop) + birdloop_stop_self(p->loop, proto_loop_stopped, p); + else + proto_cleanup(p); } @@ -1399,6 +1441,7 @@ proto_init(struct proto_config *c, node *n) struct protocol *pr = c->protocol; struct proto *p = pr->init(c); + p->loop = &main_birdloop; p->proto_state = PS_DOWN; p->last_state_change = current_time(); p->vrf = c->vrf; @@ -1415,11 +1458,27 @@ proto_init(struct proto_config *c, node *n) static void proto_start(struct proto *p) { - /* Here we cannot use p->cf->name since it won't survive reconfiguration */ - p->pool = rp_new(proto_pool, p->proto->name); + DBG("Kicking %s up\n", p->name); + PD(p, "Starting"); + + int ns = strlen("Protocol ") + strlen(p->cf->name) + 1; + void *nb = mb_alloc(proto_pool, ns); + ASSERT_DIE(ns - 1 == bsnprintf(nb, ns, "Protocol %s", p->cf->name)); + + p->pool = rp_new(proto_pool, nb); if (graceful_restart_state == GRS_INIT) p->gr_recovery = 1; + + if (p->cf->loop_order != DOMAIN_ORDER(the_bird)) + p->loop = birdloop_new(p->pool, p->cf->loop_order, nb); + + p->event->list = proto_event_list(p); + + mb_move(nb, p->pool); + + PROTO_LOCKED_FROM_MAIN(p) + proto_notify_state(p, (p->proto->start ? p->proto->start(p) : PS_UP)); } @@ -1455,6 +1514,7 @@ proto_config_new(struct protocol *pr, int class) cf->class = class; cf->debug = new_config->proto_default_debug; cf->mrtdump = new_config->proto_default_mrtdump; + cf->loop_order = DOMAIN_ORDER(the_bird); init_list(&cf->channels); @@ -1744,11 +1804,20 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty } static void -proto_rethink_goal(struct proto *p) +proto_shutdown(struct proto *p) { - struct protocol *q; - byte goal; + if (p->proto_state == PS_START || p->proto_state == PS_UP) + { + /* Going down */ + DBG("Kicking %s down\n", p->name); + PD(p, "Shutting down"); + proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_DOWN)); + } +} +static void +proto_rethink_goal(struct proto *p) +{ if (p->reconfiguring && !p->active) { struct proto_config *nc = p->cf_new; @@ -1768,32 +1837,12 @@ proto_rethink_goal(struct proto *p) /* Determine what state we want to reach */ if (p->disabled || p->reconfiguring) - goal = PS_DOWN; - else - goal = PS_UP; - - q = p->proto; - if (goal == PS_UP) - { - if (!p->active) - { - /* Going up */ - DBG("Kicking %s up\n", p->name); - PD(p, "Starting"); - proto_start(p); - proto_notify_state(p, (q->start ? q->start(p) : PS_UP)); - } - } - else { - if (p->proto_state == PS_START || p->proto_state == PS_UP) - { - /* Going down */ - DBG("Kicking %s down\n", p->name); - PD(p, "Shutting down"); - proto_notify_state(p, (q->shutdown ? q->shutdown(p) : PS_DOWN)); - } + PROTO_LOCKED_FROM_MAIN(p) + proto_shutdown(p); } + else if (!p->active) + proto_start(p); } struct proto * @@ -1998,7 +2047,7 @@ protos_dump_all(void) #define DPF(x) (p->x ? " " #x : "") debug(" protocol %s (%p) state %s with %d active channels flags: %s%s%s%s\n", p->name, p, p_states[p->proto_state], p->active_channels, - DPF(disabled), DPF(active), DPF(do_start), DPF(do_stop), DPF(reconfiguring)); + DPF(disabled), DPF(active), DPF(do_stop), DPF(reconfiguring)); #undef DPF struct channel *c; @@ -2286,8 +2335,8 @@ static inline void proto_do_start(struct proto *p) { p->active = 1; - p->do_start = 1; - ev_schedule(p->event); + if (!p->cf->late_if_feed) + if_feed_baby(p); } static void @@ -2300,6 +2349,9 @@ proto_do_up(struct proto *p) } proto_start_channels(p); + + if (p->cf->late_if_feed) + if_feed_baby(p); } static inline void @@ -2314,9 +2366,6 @@ proto_do_stop(struct proto *p) p->down_sched = 0; p->gr_recovery = 0; - p->do_stop = 1; - ev_schedule(p->event); - if (p->main_source) { rt_unlock_source(p->main_source); @@ -2324,6 +2373,9 @@ proto_do_stop(struct proto *p) } proto_stop_channels(p); + + p->do_stop = 1; + proto_send_event(p); } static void @@ -2334,7 +2386,7 @@ proto_do_down(struct proto *p) /* Shutdown is finished in the protocol event */ if (proto_is_done(p)) - ev_schedule(p->event); + proto_send_event(p); } @@ -2573,7 +2625,7 @@ proto_cmd_disable(struct proto *p, uintptr_t arg, int cnt UNUSED) p->disabled = 1; p->down_code = PDC_CMD_DISABLE; proto_set_message(p, (char *) arg, -1); - proto_rethink_goal(p); + proto_shutdown(p); cli_msg(-9, "%s: disabled", p->name); } @@ -2606,9 +2658,9 @@ proto_cmd_restart(struct proto *p, uintptr_t arg, int cnt UNUSED) p->disabled = 1; p->down_code = PDC_CMD_RESTART; proto_set_message(p, (char *) arg, -1); - proto_rethink_goal(p); + proto_shutdown(p); p->disabled = 0; - proto_rethink_goal(p); + /* After the protocol shuts down, proto_rethink_goal() is run from proto_event. */ cli_msg(-12, "%s: restarted", p->name); } @@ -2683,7 +2735,9 @@ proto_apply_cmd_symbol(const struct symbol *s, void (* cmd)(struct proto *, uint if (s->proto->proto) { - cmd(s->proto->proto, arg, 0); + struct proto *p = s->proto->proto; + PROTO_LOCKED_FROM_MAIN(p) + cmd(p, arg, 0); cli_msg(0, ""); } else @@ -2698,7 +2752,8 @@ proto_apply_cmd_patt(const char *patt, void (* cmd)(struct proto *, uintptr_t, i WALK_LIST(p, proto_list) if (!patt || patmatch(patt, p->name)) - cmd(p, arg, cnt++); + PROTO_LOCKED_FROM_MAIN(p) + cmd(p, arg, cnt++); if (!cnt) cli_msg(8003, "No protocols match"); diff --git a/nest/protocol.h b/nest/protocol.h index f9996b18..2fbd607c 100644 --- a/nest/protocol.h +++ b/nest/protocol.h @@ -120,8 +120,10 @@ struct proto_config { u8 net_type; /* Protocol network type (NET_*), 0 for undefined */ u8 disabled; /* Protocol enabled/disabled by default */ u8 vrf_set; /* Related VRF instance (below) is defined */ + u8 late_if_feed; /* Delay interface feed after channels are up */ u32 debug, mrtdump; /* Debugging bitfields, both use D_* constants */ u32 router_id; /* Protocol specific router ID */ + uint loop_order; /* Launch a birdloop on this locking level; use DOMAIN_ORDER(the_bird) for mainloop */ list channels; /* List of channel configs (struct channel_config) */ struct iface *vrf; /* Related VRF instance, NULL if global */ @@ -139,6 +141,7 @@ struct proto { struct proto_config *cf_new; /* Configuration we want to switch to after shutdown (NULL=delete) */ pool *pool; /* Pool containing local objects */ event *event; /* Protocol event */ + struct birdloop *loop; /* BIRDloop running this protocol */ list channels; /* List of channels to rtables (struct channel) */ struct channel *main_channel; /* Primary channel */ @@ -149,12 +152,12 @@ struct proto { u32 debug; /* Debugging flags */ u32 mrtdump; /* MRTDump flags */ uint active_channels; /* Number of active channels */ + uint active_coroutines; /* Number of active coroutines */ byte net_type; /* Protocol network type (NET_*), 0 for undefined */ byte disabled; /* Manually disabled */ byte vrf_set; /* Related VRF instance (above) is defined */ byte proto_state; /* Protocol state machine (PS_*, see below) */ byte active; /* From PS_START to cleanup after PS_STOP */ - byte do_start; /* Start actions are scheduled */ byte do_stop; /* Stop actions are scheduled */ byte reconfiguring; /* We're shutting down due to reconfiguration */ byte gr_recovery; /* Protocol should participate in graceful restart recovery */ @@ -356,6 +359,8 @@ void proto_notify_state(struct proto *p, unsigned state); * as a result of received ROUTE-REFRESH request). */ +static inline int proto_is_inactive(struct proto *p) +{ return (p->active_channels == 0) && (p->active_coroutines == 0); } /* diff --git a/nest/route.h b/nest/route.h index d6474e09..7c5fd02f 100644 --- a/nest/route.h +++ b/nest/route.h @@ -11,6 +11,7 @@ #define _BIRD_ROUTE_H_ #include "lib/lists.h" +#include "lib/event.h" #include "lib/bitmap.h" #include "lib/resource.h" #include "lib/net.h" @@ -329,6 +330,8 @@ struct rt_export_request { char *name; u8 trace_routes; + event_list *list; /* Where to schedule export events */ + /* There are two methods of export. You can either request feeding every single change * or feeding the whole route feed. In case of regular export, &export_one is preferred. * Anyway, when feeding, &export_bulk is preferred, falling back to &export_one. diff --git a/nest/rt-table.c b/nest/rt-table.c index c049101a..9c12ef56 100644 --- a/nest/rt-table.c +++ b/nest/rt-table.c @@ -1105,6 +1105,12 @@ rt_next_export(struct rt_export_hook *hook, rtable *tab) return tab->first_export; } +static inline void +rt_send_export_event(struct rt_export_hook *hook) +{ + ev_send(hook->req->list, hook->event); +} + static void rt_announce_exports(timer *tm) { @@ -1116,7 +1122,7 @@ rt_announce_exports(timer *tm) if (atomic_load_explicit(&c->export_state, memory_order_acquire) != TES_READY) continue; - ev_schedule_work(c->event); + rt_send_export_event(c); } } @@ -1169,7 +1175,7 @@ rt_export_hook(void *_data) rte_update_unlock(); } - ev_schedule_work(c->event); + rt_send_export_event(c); } @@ -1732,7 +1738,7 @@ rt_request_export(rtable *tab, struct rt_export_request *req) DBG("New export hook %p req %p in table %s uc=%u\n", hook, req, tab->name, tab->use_count); hook->event = ev_new_init(p, rt_feed_channel, hook); - ev_schedule_work(hook->event); + rt_send_export_event(hook); rt_set_export_state(hook, TES_FEEDING); } @@ -1754,7 +1760,7 @@ rt_stop_export(struct rt_export_request *req, void (*stopped)(struct rt_export_r hook->event->hook = rt_export_stopped; hook->stopped = stopped; - ev_schedule(hook->event); + rt_send_export_event(hook); rt_set_export_state(hook, TES_STOP); } @@ -2869,7 +2875,7 @@ rt_feed_channel(void *data) if (max_feed <= 0) { FIB_ITERATE_PUT(fit); - ev_schedule_work(c->event); + rt_send_export_event(c); return; } @@ -2904,7 +2910,7 @@ rt_feed_channel(void *data) FIB_ITERATE_END; c->event->hook = rt_export_hook; - ev_schedule_work(c->event); + rt_send_export_event(c); rt_set_export_state(c, TES_READY); } diff --git a/proto/bfd/bfd.c b/proto/bfd/bfd.c index dac184c5..3964c267 100644 --- a/proto/bfd/bfd.c +++ b/proto/bfd/bfd.c @@ -113,8 +113,16 @@ #define HASH_IP_EQ(a1,n1,a2,n2) ipa_equal(a1, a2) && n1 == n2 #define HASH_IP_FN(a,n) ipa_hash(a) ^ u32_hash(n) -static list bfd_proto_list; -static list bfd_wait_list; +DEFINE_DOMAIN(rtable); +#define BFD_LOCK LOCK_DOMAIN(rtable, bfd_global.lock) +#define BFD_UNLOCK UNLOCK_DOMAIN(rtable, bfd_global.lock) + +static struct { + DOMAIN(rtable) lock; + list wait_list; + list pickup_list; + list proto_list; +} bfd_global; const char *bfd_state_names[] = { "AdminDown", "Down", "Init", "Up" }; @@ -188,7 +196,7 @@ bfd_session_update_tx_interval(struct bfd_session *s) return; /* Set timer relative to last tx_timer event */ - tm_set(s->tx_timer, s->last_tx + tx_int_l); + tm_set_in(s->tx_timer, s->last_tx + tx_int_l, s->ifa->bfd->p.loop); } static void @@ -202,7 +210,7 @@ bfd_session_update_detection_time(struct bfd_session *s, int kick) if (!s->last_rx) return; - tm_set(s->hold_timer, s->last_rx + timeout); + tm_set_in(s->hold_timer, s->last_rx + timeout, s->ifa->bfd->p.loop); } static void @@ -226,7 +234,7 @@ bfd_session_control_tx_timer(struct bfd_session *s, int reset) if (reset || !tm_active(s->tx_timer)) { s->last_tx = 0; - tm_start(s->tx_timer, 0); + tm_start_in(s->tx_timer, 0, s->ifa->bfd->p.loop); } return; @@ -419,7 +427,7 @@ bfd_get_free_id(struct bfd_proto *p) static struct bfd_session * bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *iface, struct bfd_options *opts) { - birdloop_enter(p->loop); + ASSERT_DIE(birdloop_inside(p->p.loop)); struct bfd_iface *ifa = bfd_get_iface(p, local, iface); @@ -454,8 +462,6 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface * TRACE(D_EVENTS, "Session to %I added", s->addr); - birdloop_leave(p->loop); - return s; } @@ -463,38 +469,34 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface * static void bfd_open_session(struct bfd_proto *p, struct bfd_session *s, ip_addr local, struct iface *ifa) { - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); s->opened = 1; bfd_session_control_tx_timer(s); - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); } static void bfd_close_session(struct bfd_proto *p, struct bfd_session *s) { - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); s->opened = 0; bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_PATH_DOWN); bfd_session_control_tx_timer(s); - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); } */ static void -bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) +bfd_remove_session_locked(struct bfd_proto *p, struct bfd_session *s) { - ip_addr ip = s->addr; - /* Caller should ensure that request list is empty */ - birdloop_enter(p->loop); - /* Remove session from notify list if scheduled for notification */ /* No need for bfd_lock_sessions(), we are already protected by birdloop_enter() */ if (NODE_VALID(&s->n)) @@ -508,11 +510,17 @@ bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) HASH_REMOVE(p->session_hash_id, HASH_ID, s); HASH_REMOVE(p->session_hash_ip, HASH_IP, s); - sl_free(p->session_slab, s); + TRACE(D_EVENTS, "Session to %I removed", s->addr); - TRACE(D_EVENTS, "Session to %I removed", ip); + sl_free(p->session_slab, s); +} - birdloop_leave(p->loop); +static void +bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) +{ + birdloop_enter(p->p.loop); + bfd_remove_session_locked(p, s); + birdloop_leave(p->p.loop); } static void @@ -521,7 +529,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s) if (EMPTY_LIST(s->request_list)) return; - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); struct bfd_request *req = SKIP_BACK(struct bfd_request, n, HEAD(s->request_list)); s->cf = bfd_merge_options(s->ifa->cf, &req->opts); @@ -534,7 +542,7 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s) bfd_session_control_tx_timer(s, 0); - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); TRACE(D_EVENTS, "Session to %I reconfigured", s->addr); } @@ -618,9 +626,9 @@ bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_con (new->passive != old->passive); /* This should be probably changed to not access ifa->cf from the BFD thread */ - birdloop_enter(p->loop); + birdloop_enter(p->p.loop); ifa->cf = new; - birdloop_leave(p->loop); + birdloop_leave(p->p.loop); } @@ -681,41 +689,68 @@ bfd_add_request(struct bfd_proto *p, struct bfd_request *req) } static void -bfd_submit_request(struct bfd_request *req) +bfd_pickup_requests(void *_data UNUSED) { node *n; + WALK_LIST(n, bfd_global.proto_list) + { + struct bfd_proto *p = SKIP_BACK(struct bfd_proto, bfd_node, n); + birdloop_enter(p->p.loop); + BFD_LOCK; - WALK_LIST(n, bfd_proto_list) - if (bfd_add_request(SKIP_BACK(struct bfd_proto, bfd_node, n), req)) - return; + node *rn, *rnxt; + WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list) + bfd_add_request(p, SKIP_BACK(struct bfd_request, n, rn)); - rem_node(&req->n); - add_tail(&bfd_wait_list, &req->n); - req->session = NULL; - bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0); + BFD_UNLOCK; + birdloop_leave(p->p.loop); + } + + BFD_LOCK; + node *rn, *rnxt; + WALK_LIST_DELSAFE(rn, rnxt, bfd_global.pickup_list) + { + rem_node(rn); + add_tail(&bfd_global.wait_list, rn); + bfd_request_notify(SKIP_BACK(struct bfd_request, n, rn), BFD_STATE_ADMIN_DOWN, 0); + } + BFD_UNLOCK; } +static event bfd_pickup_event = { .hook = bfd_pickup_requests }; + static void bfd_take_requests(struct bfd_proto *p) { node *n, *nn; - - WALK_LIST_DELSAFE(n, nn, bfd_wait_list) + BFD_LOCK; + WALK_LIST_DELSAFE(n, nn, bfd_global.wait_list) bfd_add_request(p, SKIP_BACK(struct bfd_request, n, n)); + BFD_UNLOCK; } static void bfd_drop_requests(struct bfd_proto *p) { node *n; - - HASH_WALK(p->session_hash_id, next_id, s) + BFD_LOCK; + HASH_WALK_DELSAFE(p->session_hash_id, next_id, s) { - /* We assume that p is not in bfd_proto_list */ WALK_LIST_FIRST(n, s->request_list) - bfd_submit_request(SKIP_BACK(struct bfd_request, n, n)); + { + struct bfd_request *req = SKIP_BACK(struct bfd_request, n, n); + rem_node(&req->n); + add_tail(&bfd_global.pickup_list, &req->n); + req->session = NULL; + bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0); + } + + ev_send(&global_event_list, &bfd_pickup_event); + + bfd_remove_session_locked(p, s); } HASH_WALK_END; + BFD_UNLOCK; } static struct resclass bfd_request_class; @@ -728,9 +763,6 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local, { struct bfd_request *req = ralloc(p, &bfd_request_class); - /* Hack: self-link req->n, we will call rem_node() on it */ - req->n.prev = req->n.next = &req->n; - req->addr = addr; req->local = local; req->iface = iface; @@ -739,11 +771,16 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local, if (opts) req->opts = *opts; - bfd_submit_request(req); - req->hook = hook; req->data = data; + req->session = NULL; + + BFD_LOCK; + add_tail(&bfd_global.pickup_list, &req->n); + ev_send(&global_event_list, &bfd_pickup_event); + BFD_UNLOCK; + return req; } @@ -1001,8 +1038,10 @@ bfd_notify_init(struct bfd_proto *p) void bfd_init_all(void) { - init_list(&bfd_proto_list); - init_list(&bfd_wait_list); + bfd_global.lock = DOMAIN_NEW(rtable, "BFD Global"); + init_list(&bfd_global.wait_list); + init_list(&bfd_global.pickup_list); + init_list(&bfd_global.proto_list); } static struct proto * @@ -1021,10 +1060,10 @@ bfd_start(struct proto *P) struct bfd_proto *p = (struct bfd_proto *) P; struct bfd_config *cf = (struct bfd_config *) (P->cf); - p->loop = birdloop_new(); - p->tpool = rp_new(NULL, "BFD thread root"); pthread_spin_init(&p->lock, PTHREAD_PROCESS_PRIVATE); + p->tpool = rp_new(P->pool, "BFD loop pool"); + p->session_slab = sl_new(P->pool, sizeof(struct bfd_session)); HASH_INIT(p->session_hash_id, P->pool, 8); HASH_INIT(p->session_hash_ip, P->pool, 8); @@ -1034,9 +1073,7 @@ bfd_start(struct proto *P) init_list(&p->notify_list); bfd_notify_init(p); - add_tail(&bfd_proto_list, &p->bfd_node); - - birdloop_enter(p->loop); + add_tail(&bfd_global.proto_list, &p->bfd_node); if (cf->accept_ipv4 && cf->accept_direct) p->rx4_1 = bfd_open_rx_sk(p, 0, SK_IPV4); @@ -1050,42 +1087,33 @@ bfd_start(struct proto *P) if (cf->accept_ipv6 && cf->accept_multihop) p->rx6_m = bfd_open_rx_sk(p, 1, SK_IPV6); - birdloop_leave(p->loop); - bfd_take_requests(p); struct bfd_neighbor *n; WALK_LIST(n, cf->neigh_list) bfd_start_neighbor(p, n); - birdloop_start(p->loop); - return PS_UP; } - static int bfd_shutdown(struct proto *P) { struct bfd_proto *p = (struct bfd_proto *) P; - struct bfd_config *cf = (struct bfd_config *) (P->cf); + struct bfd_config *cf = (struct bfd_config *) (p->p.cf); rem_node(&p->bfd_node); - birdloop_stop(p->loop); - - struct bfd_neighbor *n; - WALK_LIST(n, cf->neigh_list) - bfd_stop_neighbor(p, n); + struct bfd_neighbor *bn; + WALK_LIST(bn, cf->neigh_list) + bfd_stop_neighbor(p, bn); bfd_drop_requests(p); - /* FIXME: This is hack */ - birdloop_enter(p->loop); - rfree(p->tpool); - birdloop_leave(p->loop); - - birdloop_free(p->loop); + if (p->rx4_1) sk_stop(p->rx4_1); + if (p->rx4_m) sk_stop(p->rx4_m); + if (p->rx6_1) sk_stop(p->rx6_1); + if (p->rx6_m) sk_stop(p->rx6_m); return PS_DOWN; } @@ -1105,7 +1133,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c) (new->accept_multihop != old->accept_multihop)) return 0; - birdloop_mask_wakeups(p->loop); + birdloop_mask_wakeups(p->p.loop); WALK_LIST(ifa, p->iface_list) bfd_reconfigure_iface(p, ifa, new); @@ -1119,7 +1147,7 @@ bfd_reconfigure(struct proto *P, struct proto_config *c) bfd_reconfigure_neighbors(p, new); - birdloop_unmask_wakeups(p->loop); + birdloop_unmask_wakeups(p->p.loop); return 1; } diff --git a/proto/bfd/bfd.h b/proto/bfd/bfd.h index 9d4cbbf8..8430064b 100644 --- a/proto/bfd/bfd.h +++ b/proto/bfd/bfd.h @@ -17,12 +17,12 @@ #include "nest/password.h" #include "conf/conf.h" #include "lib/hash.h" +#include "lib/io-loop.h" #include "lib/resource.h" #include "lib/socket.h" #include "lib/string.h" #include "nest/bfd.h" -#include "sysdep/unix/io-loop.h" #define BFD_CONTROL_PORT 3784 @@ -87,9 +87,11 @@ struct bfd_neighbor struct bfd_proto { struct proto p; - struct birdloop *loop; - pool *tpool; + pthread_spinlock_t lock; + + pool *tpool; + node bfd_node; slab *session_slab; diff --git a/proto/bfd/config.Y b/proto/bfd/config.Y index df1cba42..ed5479fb 100644 --- a/proto/bfd/config.Y +++ b/proto/bfd/config.Y @@ -36,6 +36,7 @@ proto: bfd_proto ; bfd_proto_start: proto_start BFD { this_proto = proto_config_new(&proto_bfd, $1); + this_proto->loop_order = DOMAIN_ORDER(proto); init_list(&BFD_CFG->patt_list); init_list(&BFD_CFG->neigh_list); BFD_CFG->accept_ipv4 = BFD_CFG->accept_ipv6 = 1; diff --git a/proto/bfd/packets.c b/proto/bfd/packets.c index 7618e20f..37d77f37 100644 --- a/proto/bfd/packets.c +++ b/proto/bfd/packets.c @@ -410,7 +410,7 @@ bfd_err_hook(sock *sk, int err) sock * bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af) { - sock *sk = sk_new(p->tpool); + sock *sk = sk_new(p->p.pool); sk->type = SK_UDP; sk->subtype = af; sk->sport = !multihop ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; @@ -441,7 +441,7 @@ bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af) sock * bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa) { - sock *sk = sk_new(p->tpool); + sock *sk = sk_new(p->p.pool); sk->type = SK_UDP; sk->saddr = local; sk->dport = ifa ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; diff --git a/proto/bgp/bgp.c b/proto/bgp/bgp.c index 35e9ea59..96df671f 100644 --- a/proto/bgp/bgp.c +++ b/proto/bgp/bgp.c @@ -853,6 +853,7 @@ bgp_graceful_restart_feed(struct bgp_channel *c) { c->stale_feed = (struct rt_export_request) { .name = "BGP-GR", + .list = &global_work_list, .trace_routes = c->c.debug | c->c.proto->debug, .dump_req = bgp_graceful_restart_feed_dump_req, .log_state_change = bgp_graceful_restart_feed_log_state_change, diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c index 2eba142c..4758c056 100644 --- a/sysdep/unix/coroutine.c +++ b/sysdep/unix/coroutine.c @@ -21,10 +21,9 @@ #include "lib/resource.h" #include "lib/timer.h" -/* Using a rather big stack for coroutines to allow for stack-local allocations. - * In real world, the kernel doesn't alloc this memory until it is used. - * */ -#define CORO_STACK_SIZE 1048576 +#include "conf/conf.h" + +#define CORO_STACK_SIZE 65536 /* * Implementation of coroutines based on POSIX threads @@ -79,6 +78,11 @@ domain_free(struct domain_generic *dg) xfree(dg); } +uint dg_order(struct domain_generic *dg) +{ + return dg->order; +} + void do_lock(struct domain_generic *dg, struct domain_generic **lsp) { if ((char *) lsp - (char *) &locking_stack != dg->order) @@ -89,7 +93,11 @@ void do_lock(struct domain_generic *dg, struct domain_generic **lsp) if (*lsp) bug("Inconsistent locking stack state on lock"); + btime lock_begin = current_time(); pthread_mutex_lock(&dg->mutex); + btime duration = current_time() - lock_begin; + if (config && (duration > config->watchdog_warning)) + log(L_WARN "Locking of %s took %d ms", dg->name, (int) (duration TO_MS)); if (dg->prev || dg->locked_by) bug("Previous unlock not finished correctly"); @@ -140,11 +148,16 @@ static struct resclass coro_class = { .free = coro_free, }; +_Thread_local struct coroutine *this_coro = NULL; + static void *coro_entry(void *p) { struct coroutine *c = p; + ASSERT_DIE(c->entry); + this_coro = c; + c->entry(c->data); ASSERT_DIE(coro_cleaned_up); diff --git a/sysdep/unix/io-loop.c b/sysdep/unix/io-loop.c index a15d866a..275d38a0 100644 --- a/sysdep/unix/io-loop.c +++ b/sysdep/unix/io-loop.c @@ -15,50 +15,47 @@ #include #include "nest/bird.h" -#include "sysdep/unix/io-loop.h" #include "lib/buffer.h" +#include "lib/coro.h" #include "lib/lists.h" #include "lib/resource.h" #include "lib/event.h" #include "lib/timer.h" #include "lib/socket.h" - -struct birdloop -{ - pool *pool; - pthread_t thread; - pthread_mutex_t mutex; - - u8 stop_called; - u8 poll_active; - u8 wakeup_masked; - int wakeup_fds[2]; - - struct timeloop time; - list event_list; - list sock_list; - uint sock_num; - - BUFFER(sock *) poll_sk; - BUFFER(struct pollfd) poll_fd; - u8 poll_changed; - u8 close_scheduled; -}; - +#include "lib/io-loop.h" +#include "sysdep/unix/io-loop.h" +#include "conf/conf.h" /* * Current thread context */ -static _Thread_local struct birdloop *birdloop_current; +_Thread_local struct birdloop *birdloop_current; +static _Thread_local struct birdloop *birdloop_wakeup_masked; +static _Thread_local uint birdloop_wakeup_masked_count; -static inline void -birdloop_set_current(struct birdloop *loop) +event_list * +birdloop_event_list(struct birdloop *loop) { - birdloop_current = loop; - local_timeloop = loop ? &loop->time : &main_timeloop; + return &loop->event_list; +} + +struct timeloop * +birdloop_time_loop(struct birdloop *loop) +{ + return &loop->time; +} + +_Bool +birdloop_inside(struct birdloop *loop) +{ + for (struct birdloop *c = birdloop_current; c; c = c->prev_loop) + if (loop == c) + return 1; + + return 0; } /* @@ -135,57 +132,17 @@ wakeup_do_kick(struct birdloop *loop) pipe_kick(loop->wakeup_fds[1]); } -static inline void -wakeup_kick(struct birdloop *loop) -{ - if (!loop->wakeup_masked) - wakeup_do_kick(loop); - else - loop->wakeup_masked = 2; -} - -/* For notifications from outside */ void -wakeup_kick_current(void) +birdloop_ping(struct birdloop *loop) { - if (birdloop_current && birdloop_current->poll_active) - wakeup_kick(birdloop_current); -} - - -/* - * Events - */ + u32 ping_sent = atomic_fetch_add_explicit(&loop->ping_sent, 1, memory_order_acq_rel); + if (ping_sent) + return; -static inline uint -events_waiting(struct birdloop *loop) -{ - return !EMPTY_LIST(loop->event_list); -} - -static inline void -events_init(struct birdloop *loop) -{ - init_list(&loop->event_list); -} - -static void -events_fire(struct birdloop *loop) -{ - times_update(); - ev_run_list(&loop->event_list); -} - -void -ev2_schedule(event *e) -{ - if (birdloop_current->poll_active && EMPTY_LIST(birdloop_current->event_list)) - wakeup_kick(birdloop_current); - - if (e->n.next) - rem_node(&e->n); - - add_tail(&birdloop_current->event_list, &e->n); + if (loop == birdloop_wakeup_masked) + birdloop_wakeup_masked_count++; + else + wakeup_do_kick(loop); } @@ -213,13 +170,13 @@ sockets_add(struct birdloop *loop, sock *s) s->index = -1; loop->poll_changed = 1; - if (loop->poll_active) - wakeup_kick(loop); + birdloop_ping(loop); } void sk_start(sock *s) { + ASSERT_DIE(birdloop_current != &main_birdloop); sockets_add(birdloop_current, s); } @@ -230,28 +187,21 @@ sockets_remove(struct birdloop *loop, sock *s) loop->sock_num--; if (s->index >= 0) + { loop->poll_sk.data[s->index] = NULL; - - s->index = -1; - loop->poll_changed = 1; - - /* Wakeup moved to sk_stop() */ + s->index = -1; + loop->poll_changed = 1; + loop->close_scheduled = 1; + birdloop_ping(loop); + } + else + close(s->fd); } void sk_stop(sock *s) { sockets_remove(birdloop_current, s); - - if (birdloop_current->poll_active) - { - birdloop_current->close_scheduled = 1; - wakeup_kick(birdloop_current); - } - else - close(s->fd); - - s->fd = -1; } static inline uint sk_want_events(sock *s) @@ -351,12 +301,15 @@ sockets_fire(struct birdloop *loop) if (pfd->revents & POLLIN) while (e && *psk && (*psk)->rx_hook) - e = sk_read(*psk, 0); + e = sk_read(*psk, pfd->revents); e = 1; if (pfd->revents & POLLOUT) + { + loop->poll_changed = 1; while (e && *psk) e = sk_write(*psk); + } } } @@ -365,104 +318,168 @@ sockets_fire(struct birdloop *loop) * Birdloop */ -static void *birdloop_main(void *arg); +struct birdloop main_birdloop; + +static void birdloop_enter_locked(struct birdloop *loop); + +void +birdloop_init(void) +{ + wakeup_init(&main_birdloop); + + main_birdloop.time.domain = the_bird_domain.the_bird; + main_birdloop.time.loop = &main_birdloop; + + times_update(); + timers_init(&main_birdloop.time, &root_pool); + + birdloop_enter_locked(&main_birdloop); +} + +static void birdloop_main(void *arg); struct birdloop * -birdloop_new(void) +birdloop_new(pool *pp, uint order, const char *name) { - pool *p = rp_new(NULL, "Birdloop root"); + struct domain_generic *dg = domain_new(name, order); + + pool *p = rp_new(pp, name); struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop)); loop->pool = p; - pthread_mutex_init(&loop->mutex, NULL); - wakeup_init(loop); + loop->time.domain = dg; + loop->time.loop = loop; + + birdloop_enter(loop); - events_init(loop); + wakeup_init(loop); + ev_init_list(&loop->event_list, loop, name); timers_init(&loop->time, p); sockets_init(loop); + loop->time.coro = coro_run(p, birdloop_main, loop); + + birdloop_leave(loop); + return loop; } +static void +birdloop_do_stop(struct birdloop *loop, void (*stopped)(void *data), void *data) +{ + loop->stopped = stopped; + loop->stop_data = data; + wakeup_do_kick(loop); +} + void -birdloop_start(struct birdloop *loop) +birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data) { - int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop); - if (rv) - die("pthread_create(): %M", rv); + DG_LOCK(loop->time.domain); + birdloop_do_stop(loop, stopped, data); + DG_UNLOCK(loop->time.domain); } void -birdloop_stop(struct birdloop *loop) +birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data) { - pthread_mutex_lock(&loop->mutex); - loop->stop_called = 1; - wakeup_do_kick(loop); - pthread_mutex_unlock(&loop->mutex); + ASSERT_DIE(loop == birdloop_current); + ASSERT_DIE(DG_IS_LOCKED(loop->time.domain)); - int rv = pthread_join(loop->thread, NULL); - if (rv) - die("pthread_join(): %M", rv); + birdloop_do_stop(loop, stopped, data); } void birdloop_free(struct birdloop *loop) { + ASSERT_DIE(loop->links == 0); + domain_free(loop->time.domain); rfree(loop->pool); } +static void +birdloop_enter_locked(struct birdloop *loop) +{ + ASSERT_DIE(DG_IS_LOCKED(loop->time.domain)); + ASSERT_DIE(!birdloop_inside(loop)); + + /* Store the old context */ + loop->prev_loop = birdloop_current; + + /* Put the new context */ + birdloop_current = loop; +} void birdloop_enter(struct birdloop *loop) { - /* TODO: these functions could save and restore old context */ - pthread_mutex_lock(&loop->mutex); - birdloop_set_current(loop); + DG_LOCK(loop->time.domain); + return birdloop_enter_locked(loop); +} + +static void +birdloop_leave_locked(struct birdloop *loop) +{ + /* Check the current context */ + ASSERT_DIE(birdloop_current == loop); + + /* Restore the old context */ + birdloop_current = loop->prev_loop; } void birdloop_leave(struct birdloop *loop) { - /* TODO: these functions could save and restore old context */ - birdloop_set_current(NULL); - pthread_mutex_unlock(&loop->mutex); + birdloop_leave_locked(loop); + DG_UNLOCK(loop->time.domain); } void birdloop_mask_wakeups(struct birdloop *loop) { - pthread_mutex_lock(&loop->mutex); - loop->wakeup_masked = 1; - pthread_mutex_unlock(&loop->mutex); + ASSERT_DIE(birdloop_wakeup_masked == NULL); + birdloop_wakeup_masked = loop; } void birdloop_unmask_wakeups(struct birdloop *loop) { - pthread_mutex_lock(&loop->mutex); - if (loop->wakeup_masked == 2) + ASSERT_DIE(birdloop_wakeup_masked == loop); + birdloop_wakeup_masked = NULL; + if (birdloop_wakeup_masked_count) wakeup_do_kick(loop); - loop->wakeup_masked = 0; - pthread_mutex_unlock(&loop->mutex); + + birdloop_wakeup_masked_count = 0; } -static void * +void +birdloop_link(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_inside(loop)); + loop->links++; +} + +void +birdloop_unlink(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_inside(loop)); + loop->links--; +} + +static void birdloop_main(void *arg) { struct birdloop *loop = arg; timer *t; int rv, timeout; - birdloop_set_current(loop); + btime loop_begin = current_time(); - pthread_mutex_lock(&loop->mutex); + birdloop_enter(loop); while (1) { - events_fire(loop); - timers_fire(&loop->time); - - times_update(); - if (events_waiting(loop)) + timers_fire(&loop->time, 0); + if (ev_run_list(&loop->event_list)) timeout = 0; else if (t = timers_first(&loop->time)) timeout = (tm_remains(t) TO_MS) + 1; @@ -472,8 +489,11 @@ birdloop_main(void *arg) if (loop->poll_changed) sockets_prepare(loop); - loop->poll_active = 1; - pthread_mutex_unlock(&loop->mutex); + btime duration = current_time() - loop_begin; + if (duration > config->watchdog_warning) + log(L_WARN "I/O loop cycle took %d ms", (int) (duration TO_MS)); + + birdloop_leave(loop); try: rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout); @@ -484,25 +504,32 @@ birdloop_main(void *arg) die("poll: %m"); } - pthread_mutex_lock(&loop->mutex); - loop->poll_active = 0; + birdloop_enter(loop); if (loop->close_scheduled) sockets_close_fds(loop); - if (loop->stop_called) + if (loop->stopped) break; + loop_begin = current_time(); + if (rv) sockets_fire(loop); - timers_fire(&loop->time); + atomic_exchange_explicit(&loop->ping_sent, 0, memory_order_acq_rel); } - loop->stop_called = 0; - pthread_mutex_unlock(&loop->mutex); + /* Flush remaining events */ + ASSERT_DIE(!ev_run_list(&loop->event_list)); + + /* No timers allowed */ + ASSERT_DIE(timers_count(&loop->time) == 0); + ASSERT_DIE(EMPTY_LIST(loop->sock_list)); + ASSERT_DIE(loop->sock_num == 0); - return NULL; + birdloop_leave(loop); + loop->stopped(loop->stop_data); } diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h index d858b04e..4024b6c5 100644 --- a/sysdep/unix/io-loop.h +++ b/sysdep/unix/io-loop.h @@ -4,31 +4,32 @@ * Can be freely distributed and used under the terms of the GNU GPL. */ -#ifndef _BIRD_IO_LOOP_H_ -#define _BIRD_IO_LOOP_H_ +#ifndef _BIRD_SYSDEP_UNIX_IO_LOOP_H_ +#define _BIRD_SYSDEP_UNIX_IO_LOOP_H_ -#include "nest/bird.h" -#include "lib/lists.h" -#include "lib/resource.h" -#include "lib/event.h" -#include "lib/timer.h" -#include "lib/socket.h" +struct birdloop +{ + pool *pool; + struct timeloop time; + event_list event_list; + list sock_list; + uint sock_num; -void ev2_schedule(event *e); + BUFFER(sock *) poll_sk; + BUFFER(struct pollfd) poll_fd; + u8 poll_changed; + u8 close_scheduled; -void sk_start(sock *s); -void sk_stop(sock *s); + _Atomic u32 ping_sent; + int wakeup_fds[2]; -struct birdloop *birdloop_new(void); -void birdloop_start(struct birdloop *loop); -void birdloop_stop(struct birdloop *loop); -void birdloop_free(struct birdloop *loop); + uint links; -void birdloop_enter(struct birdloop *loop); -void birdloop_leave(struct birdloop *loop); -void birdloop_mask_wakeups(struct birdloop *loop); -void birdloop_unmask_wakeups(struct birdloop *loop); + void (*stopped)(void *data); + void *stop_data; + struct birdloop *prev_loop; +}; -#endif /* _BIRD_IO_LOOP_H_ */ +#endif diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index 90bb5d64..c91f2856 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -43,6 +43,7 @@ #include "conf/conf.h" #include "sysdep/unix/unix.h" +#include "sysdep/unix/io-loop.h" #include CONFIG_INCLUDE_SYSIO_H /* Maximum number of calls of tx handler for one socket in one @@ -800,18 +801,16 @@ sk_free(resource *r) sk_ssh_free(s); #endif - if (s->fd < 0) + if ((s->fd < 0) || (s->flags & SKF_THREAD)) return; - /* FIXME: we should call sk_stop() for SKF_THREAD sockets */ - if (!(s->flags & SKF_THREAD)) - { - if (s == current_sock) - current_sock = sk_next(s); - if (s == stored_sock) - stored_sock = sk_next(s); + if (s == current_sock) + current_sock = sk_next(s); + if (s == stored_sock) + stored_sock = sk_next(s); + + if (enlisted(&s->n)) rem_node(&s->n); - } if (s->type != SK_SSH && s->type != SK_SSH_ACTIVE) close(s->fd); @@ -1104,7 +1103,11 @@ sk_passive_connected(sock *s, int type) return 1; } - sk_insert(t); + if (s->flags & SKF_PASSIVE_THREAD) + t->flags |= SKF_THREAD; + else + sk_insert(t); + sk_alloc_bufs(t); s->rx_hook(t, 0); return 1; @@ -1508,6 +1511,36 @@ sk_open_unix(sock *s, char *name) return 0; } +static void +sk_reloop_hook(void *_vs) +{ + sock *s = _vs; + if (birdloop_inside(&main_birdloop)) + { + s->flags &= ~SKF_THREAD; + sk_insert(s); + } + else + { + s->flags |= SKF_THREAD; + sk_start(s); + } +} + +void +sk_reloop(sock *s, struct birdloop *loop) +{ + if (enlisted(&s->n)) + rem_node(&s->n); + + s->reloop = (event) { + .hook = sk_reloop_hook, + .data = s, + }; + + ev_send_loop(loop, &s->reloop); +} + #define CMSG_RX_SPACE MAX(CMSG4_SPACE_PKTINFO+CMSG4_SPACE_TTL, \ CMSG6_SPACE_PKTINFO+CMSG6_SPACE_TTL) @@ -2143,8 +2176,9 @@ void io_init(void) { init_list(&sock_list); - init_list(&global_event_list); - init_list(&global_work_list); + ev_init_list(&global_event_list, &main_birdloop, "Global event list"); + ev_init_list(&global_work_list, &main_birdloop, "Global work list"); + ev_init_list(&main_birdloop.event_list, &main_birdloop, "Global fast event list"); krt_io_init(); // XXX init_times(); // XXX update_times(); @@ -2158,14 +2192,7 @@ static int short_loops = 0; #define SHORT_LOOP_MAX 10 #define WORK_EVENTS_MAX 10 -static int poll_reload_pipe[2]; - -void -io_loop_reload(void) -{ - char b; - write(poll_reload_pipe[1], &b, 1); -} +void pipe_drain(int fd); void io_loop(void) @@ -2178,21 +2205,19 @@ io_loop(void) int fdmax = 256; struct pollfd *pfd = xmalloc(fdmax * sizeof(struct pollfd)); - if (pipe(poll_reload_pipe) < 0) - die("pipe(poll_reload_pipe) failed: %m"); - watchdog_start1(); for(;;) { times_update(); events = ev_run_list(&global_event_list); events = ev_run_list_limited(&global_work_list, WORK_EVENTS_MAX) || events; - timers_fire(&main_timeloop); + events = ev_run_list(&main_birdloop.event_list) || events; + timers_fire(&main_birdloop.time, 1); io_close_event(); // FIXME poll_tout = (events ? 0 : 3000); /* Time in milliseconds */ - if (t = timers_first(&main_timeloop)) + if (t = timers_first(&main_birdloop.time)) { times_update(); timeout = (tm_remains(t) TO_MS) + 1; @@ -2200,7 +2225,7 @@ io_loop(void) } /* A hack to reload main io_loop() when something has changed asynchronously. */ - pfd[0].fd = poll_reload_pipe[0]; + pfd[0].fd = main_birdloop.wakeup_fds[0]; pfd[0].events = POLLIN; nfds = 1; @@ -2263,9 +2288,9 @@ io_loop(void) /* And finally enter poll() to find active sockets */ watchdog_stop(); - the_bird_unlock(); + birdloop_leave(&main_birdloop); pout = poll(pfd, nfds, poll_tout); - the_bird_lock(); + birdloop_enter(&main_birdloop); watchdog_start(); if (pout < 0) @@ -2279,8 +2304,8 @@ io_loop(void) if (pfd[0].revents & POLLIN) { /* IO loop reload requested */ - char b; - read(poll_reload_pipe[0], &b, 1); + pipe_drain(main_birdloop.wakeup_fds[0]); + atomic_exchange_explicit(&main_birdloop.ping_sent, 0, memory_order_acq_rel); continue; } diff --git a/sysdep/unix/main.c b/sysdep/unix/main.c index d35424ff..5da27cb6 100644 --- a/sysdep/unix/main.c +++ b/sysdep/unix/main.c @@ -908,10 +908,12 @@ main(int argc, char **argv) parse_args(argc, argv); log_switch(1, NULL, NULL); + the_bird_lock(); + random_init(); net_init(); resource_init(); - timer_init(); + birdloop_init(); olock_init(); io_init(); rt_init(); @@ -961,7 +963,6 @@ main(int argc, char **argv) dup2(0, 2); } - the_bird_lock(); main_thread_init(); diff --git a/sysdep/unix/unix.h b/sysdep/unix/unix.h index 313c97c3..ad85d1ea 100644 --- a/sysdep/unix/unix.h +++ b/sysdep/unix/unix.h @@ -106,7 +106,6 @@ extern volatile sig_atomic_t async_shutdown_flag; void io_init(void); void io_loop(void); -void io_loop_reload(void); void io_log_dump(void); int sk_open_unix(struct birdsock *s, char *name); struct rfile *rf_open(struct pool *, const char *name, const char *mode); diff --git a/test/bt-utils.c b/test/bt-utils.c index cbca3a6b..90815e77 100644 --- a/test/bt-utils.c +++ b/test/bt-utils.c @@ -53,16 +53,20 @@ cf_file_read(byte *dest, uint max_len, int fd) return l; } +void resource_sys_init(void); + void bt_bird_init(void) { + resource_sys_init(); if(bt_verbose) log_init_debug(""); log_switch(bt_verbose != 0, NULL, NULL); + the_bird_lock(); resource_init(); olock_init(); - timer_init(); + birdloop_init(); io_init(); rt_init(); if_init(); @@ -79,6 +83,7 @@ void bt_bird_cleanup(void) class_to_protocol[i] = NULL; config = new_config = NULL; + the_bird_unlock(); } static char * -- cgit v1.2.3 From f39e9aa20363eccc38289958f23c789d037c269b Mon Sep 17 00:00:00 2001 From: Ondrej Zajicek Date: Sat, 4 Jun 2022 17:54:08 +0200 Subject: IO: Improve resolution of latency debugging messages --- sysdep/unix/io.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'sysdep/unix') diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index 4fd77453..17dc05a3 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -2046,8 +2046,8 @@ io_update_time(void) event_open->duration = last_time - event_open->timestamp; if (event_open->duration > config->latency_limit) - log(L_WARN "Event 0x%p 0x%p took %d ms", - event_open->hook, event_open->data, (int) (event_open->duration TO_MS)); + log(L_WARN "Event 0x%p 0x%p took %u.%03u ms", + event_open->hook, event_open->data, (uint) (event_open->duration TO_MS), (uint) (event_open->duration % 1000)); event_open = NULL; } @@ -2151,8 +2151,8 @@ watchdog_stop(void) btime duration = last_time - loop_time; if (duration > config->watchdog_warning) - log(L_WARN "I/O loop cycle took %d ms for %d events", - (int) (duration TO_MS), event_log_num); + log(L_WARN "I/O loop cycle took %u.%03u ms for %d events", + (uint) (duration TO_MS), (uint) (duration % 1000), event_log_num); } -- cgit v1.2.3 From beb5f78ada79ac90f31f2c4923302c74d9ab38bf Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Mon, 27 Jun 2022 19:04:22 +0200 Subject: Preexport callback now takes the channel instead of protocol as argument Passing protocol to preexport was in fact a historical relic from the old times when channels weren't a thing. Refactoring that to match current extensibility needs. --- nest/protocol.h | 2 +- nest/route.h | 2 +- nest/rt-show.c | 2 +- nest/rt-table.c | 7 ++++--- proto/babel/babel.c | 4 ++-- proto/bgp/attrs.c | 4 ++-- proto/bgp/bgp.h | 2 +- proto/ospf/ospf.c | 8 ++++---- proto/pipe/pipe.c | 4 ++-- proto/radv/radv.c | 6 +++--- sysdep/unix/krt.c | 4 ++-- 11 files changed, 23 insertions(+), 22 deletions(-) (limited to 'sysdep/unix') diff --git a/nest/protocol.h b/nest/protocol.h index abcc505d..3b3812a5 100644 --- a/nest/protocol.h +++ b/nest/protocol.h @@ -217,7 +217,7 @@ struct proto { void (*neigh_notify)(struct neighbor *neigh); void (*make_tmp_attrs)(struct rte *rt, struct linpool *pool); void (*store_tmp_attrs)(struct rte *rt, struct linpool *pool); - int (*preexport)(struct proto *, struct rte **rt, struct linpool *pool); + int (*preexport)(struct channel *, struct rte **rt, struct linpool *pool); void (*reload_routes)(struct channel *); void (*feed_begin)(struct channel *, int initial); void (*feed_end)(struct channel *); diff --git a/nest/route.h b/nest/route.h index 395139a3..1dacd92f 100644 --- a/nest/route.h +++ b/nest/route.h @@ -357,7 +357,7 @@ rte *rte_find(net *net, struct rte_src *src); rte *rte_get_temp(struct rta *); void rte_update2(struct channel *c, const net_addr *n, rte *new, struct rte_src *src); /* rte_update() moved to protocol.h to avoid dependency conflicts */ -int rt_examine(rtable *t, net_addr *a, struct proto *p, const struct filter *filter); +int rt_examine(rtable *t, net_addr *a, struct channel *c, const struct filter *filter); rte *rt_export_merged(struct channel *c, net *net, rte **rt_free, linpool *pool, int silent); void rt_refresh_begin(rtable *t, struct channel *c); void rt_refresh_end(rtable *t, struct channel *c); diff --git a/nest/rt-show.c b/nest/rt-show.c index f8b7ba51..7639e5f7 100644 --- a/nest/rt-show.c +++ b/nest/rt-show.c @@ -154,7 +154,7 @@ rt_show_net(struct cli *c, net *n, struct rt_show_data *d) else if (d->export_mode) { struct proto *ep = ec->proto; - int ic = ep->preexport ? ep->preexport(ep, &e, c->show_pool) : 0; + int ic = ep->preexport ? ep->preexport(ec, &e, c->show_pool) : 0; if (ec->ra_mode == RA_OPTIMAL || ec->ra_mode == RA_MERGED) pass = 1; diff --git a/nest/rt-table.c b/nest/rt-table.c index 2cf55421..a2fa5e77 100644 --- a/nest/rt-table.c +++ b/nest/rt-table.c @@ -870,7 +870,7 @@ export_filter_(struct channel *c, rte *rt0, rte **rt_free, linpool *pool, int si rt = rt0; *rt_free = NULL; - v = p->preexport ? p->preexport(p, &rt, pool) : 0; + v = p->preexport ? p->preexport(c, &rt, pool) : 0; if (v < 0) { if (silent) @@ -1883,8 +1883,9 @@ rte_modify(rte *old) /* Check rtable for best route to given net whether it would be exported do p */ int -rt_examine(rtable *t, net_addr *a, struct proto *p, const struct filter *filter) +rt_examine(rtable *t, net_addr *a, struct channel *c, const struct filter *filter) { + struct proto *p = c->proto; net *n = net_find(t, a); rte *rt = n ? n->routes : NULL; @@ -1894,7 +1895,7 @@ rt_examine(rtable *t, net_addr *a, struct proto *p, const struct filter *filter) rte_update_lock(); /* Rest is stripped down export_filter() */ - int v = p->preexport ? p->preexport(p, &rt, rte_update_pool) : 0; + int v = p->preexport ? p->preexport(c, &rt, rte_update_pool) : 0; if (v == RIC_PROCESS) { rte_make_tmp_attrs(&rt, rte_update_pool, NULL); diff --git a/proto/babel/babel.c b/proto/babel/babel.c index 71452a6f..0d0304f9 100644 --- a/proto/babel/babel.c +++ b/proto/babel/babel.c @@ -2230,12 +2230,12 @@ babel_kick_timer(struct babel_proto *p) static int -babel_preexport(struct proto *P, struct rte **new, struct linpool *pool UNUSED) +babel_preexport(struct channel *C, struct rte **new, struct linpool *pool UNUSED) { struct rta *a = (*new)->attrs; /* Reject our own unreachable routes */ - if ((a->dest == RTD_UNREACHABLE) && (a->src->proto == P)) + if ((a->dest == RTD_UNREACHABLE) && (a->src->proto == C->proto)) return -1; return 0; diff --git a/proto/bgp/attrs.c b/proto/bgp/attrs.c index a9787173..9dd9fb1a 100644 --- a/proto/bgp/attrs.c +++ b/proto/bgp/attrs.c @@ -1669,11 +1669,11 @@ bgp_free_prefix(struct bgp_channel *c, struct bgp_prefix *px) */ int -bgp_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +bgp_preexport(struct channel *C, rte **new, struct linpool *pool UNUSED) { rte *e = *new; struct proto *SRC = e->attrs->src->proto; - struct bgp_proto *p = (struct bgp_proto *) P; + struct bgp_proto *p = (struct bgp_proto *) C->proto; struct bgp_proto *src = (SRC->proto == &proto_bgp) ? (struct bgp_proto *) SRC : NULL; /* Reject our routes */ diff --git a/proto/bgp/bgp.h b/proto/bgp/bgp.h index 4969c0b9..7cd1c27d 100644 --- a/proto/bgp/bgp.h +++ b/proto/bgp/bgp.h @@ -589,7 +589,7 @@ int bgp_rte_mergable(rte *pri, rte *sec); int bgp_rte_recalculate(rtable *table, net *net, rte *new, rte *old, rte *old_best); struct rte *bgp_rte_modify_stale(struct rte *r, struct linpool *pool); void bgp_rt_notify(struct proto *P, struct channel *C, net *n, rte *new, rte *old); -int bgp_preexport(struct proto *, struct rte **, struct linpool *); +int bgp_preexport(struct channel *, struct rte **, struct linpool *); int bgp_get_attr(const struct eattr *e, byte *buf, int buflen); void bgp_get_route_info(struct rte *, byte *buf); int bgp_total_aigp_metric_(rte *e, u64 *metric, const struct adata **ad); diff --git a/proto/ospf/ospf.c b/proto/ospf/ospf.c index ba8c2e2b..42ffdb06 100644 --- a/proto/ospf/ospf.c +++ b/proto/ospf/ospf.c @@ -107,7 +107,7 @@ #include #include "ospf.h" -static int ospf_preexport(struct proto *P, rte **new, struct linpool *pool); +static int ospf_preexport(struct channel *C, rte **new, struct linpool *pool); static void ospf_make_tmp_attrs(struct rte *rt, struct linpool *pool); static void ospf_store_tmp_attrs(struct rte *rt, struct linpool *pool); static void ospf_reload_routes(struct channel *C); @@ -484,14 +484,14 @@ ospf_disp(timer * timer) * import to the filters. */ static int -ospf_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +ospf_preexport(struct channel *C, rte **new, struct linpool *pool UNUSED) { - struct ospf_proto *p = (struct ospf_proto *) P; + struct ospf_proto *p = (struct ospf_proto *) C->proto; struct ospf_area *oa = ospf_main_area(p); rte *e = *new; /* Reject our own routes */ - if (e->attrs->src->proto == P) + if (e->attrs->src->proto == &p->p) return -1; /* Do not export routes to stub areas */ diff --git a/proto/pipe/pipe.c b/proto/pipe/pipe.c index f991d09a..481f5804 100644 --- a/proto/pipe/pipe.c +++ b/proto/pipe/pipe.c @@ -101,11 +101,11 @@ pipe_rt_notify(struct proto *P, struct channel *src_ch, net *n, rte *new, rte *o } static int -pipe_preexport(struct proto *P, rte **ee, struct linpool *p UNUSED) +pipe_preexport(struct channel *C, rte **ee, struct linpool *p UNUSED) { struct proto *pp = (*ee)->sender->proto; - if (pp == P) + if (pp == C->proto) return -1; /* Avoid local loops automatically */ return 0; diff --git a/proto/radv/radv.c b/proto/radv/radv.c index 66e8eb4b..fe3713ef 100644 --- a/proto/radv/radv.c +++ b/proto/radv/radv.c @@ -391,10 +391,10 @@ radv_net_match_trigger(struct radv_config *cf, net *n) } int -radv_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +radv_preexport(struct channel *C, rte **new, struct linpool *pool UNUSED) { // struct radv_proto *p = (struct radv_proto *) P; - struct radv_config *cf = (struct radv_config *) (P->cf); + struct radv_config *cf = (struct radv_config *) (C->proto->cf); if (radv_net_match_trigger(cf, (*new)->net)) return RIC_PROCESS; @@ -555,7 +555,7 @@ radv_check_active(struct radv_proto *p) return 1; struct channel *c = p->p.main_channel; - return rt_examine(c->table, &cf->trigger, &p->p, c->out_filter); + return rt_examine(c->table, &cf->trigger, c, c->out_filter); } static void diff --git a/sysdep/unix/krt.c b/sysdep/unix/krt.c index 7c2614b1..a02cf977 100644 --- a/sysdep/unix/krt.c +++ b/sysdep/unix/krt.c @@ -903,12 +903,12 @@ krt_store_tmp_attrs(struct rte *rt, struct linpool *pool) } static int -krt_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +krt_preexport(struct channel *C, rte **new, struct linpool *pool UNUSED) { // struct krt_proto *p = (struct krt_proto *) P; rte *e = *new; - if (e->attrs->src->proto == P) + if (e->attrs->src->proto == C->proto) return -1; if (!krt_capable(e)) -- cgit v1.2.3 From 946cedfcfe3bdfd850613c4d891f8abfd6b2cafd Mon Sep 17 00:00:00 2001 From: "Ondrej Zajicek (work)" Date: Sun, 6 Mar 2022 16:37:30 +0100 Subject: Filter: Implement soft scopes Soft scopes are anonymous scopes that most likely do not contain any symbol, so allocating regular scope is postponed when it is really needed. --- conf/cf-lex.l | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++--- conf/conf.h | 9 ++++++++- sysdep/unix/main.c | 2 +- 3 files changed, 64 insertions(+), 5 deletions(-) (limited to 'sysdep/unix') diff --git a/conf/cf-lex.l b/conf/cf-lex.l index c9d2f5a5..d6f46223 100644 --- a/conf/cf-lex.l +++ b/conf/cf-lex.l @@ -574,6 +574,8 @@ check_eof(void) return 0; } +static inline void cf_swap_soft_scope(void); + static struct symbol * cf_new_symbol(const byte *c) { @@ -583,6 +585,8 @@ cf_new_symbol(const byte *c) if (l > SYM_MAX_LEN) cf_error("Symbol too long"); + cf_swap_soft_scope(); + s = cfg_allocz(sizeof(struct symbol) + l + 1); *s = (struct symbol) { .scope = conf_this_scope, .class = SYM_VOID, }; strcpy(s->name, c); @@ -654,10 +658,10 @@ cf_localize_symbol(struct symbol *sym) /* If the symbol type is void, it has been recently allocated just in this scope. */ if (!sym->class) return sym; - + /* If the scope is the current, it is already defined in this scope. */ - if (sym->scope == conf_this_scope) - cf_error("Symbol already defined"); + if (cf_symbol_is_local(sym)) + cf_error("Symbol '%s' already defined", sym->name); /* Not allocated here yet, doing it now. */ return cf_new_symbol(sym->name); @@ -787,12 +791,60 @@ cf_push_scope(struct symbol *sym) void cf_pop_scope(void) { + ASSERT(!conf_this_scope->soft_scopes); + conf_this_scope->active = 0; conf_this_scope = conf_this_scope->next; ASSERT(conf_this_scope); } +/** + * cf_push_soft_scope - enter new soft scope + * + * If we want to enter a new anonymous scope that most likely will not contain + * any symbols, we can use cf_push_soft_scope() insteas of cf_push_scope(). + * Such scope will be converted to a regular scope on first use. + */ +void +cf_push_soft_scope(void) +{ + if (conf_this_scope->soft_scopes < 0xfe) + conf_this_scope->soft_scopes++; + else + cf_push_scope(NULL); +} + +/** + * cf_pop_soft_scope - leave a soft scope + * + * Leave a soft scope entered by cf_push_soft_scope(). + */ +void +cf_pop_soft_scope(void) +{ + if (conf_this_scope->soft_scopes) + conf_this_scope->soft_scopes--; + else + cf_pop_scope(); +} + +/** + * cf_swap_soft_scope - convert soft scope to regular scope + * + * Soft scopes cannot hold symbols, so they must be converted to regular scopes + * on first use. It is done automatically by cf_new_symbol(). + */ +static inline void +cf_swap_soft_scope(void) +{ + if (conf_this_scope->soft_scopes) + { + conf_this_scope->soft_scopes--; + cf_push_scope(NULL); + } +} + /** * cf_symbol_class_name - get name of a symbol class * @sym: symbol diff --git a/conf/conf.h b/conf/conf.h index 3bc37959..be46f172 100644 --- a/conf/conf.h +++ b/conf/conf.h @@ -133,7 +133,8 @@ struct sym_scope { struct sym_scope *next; /* Next on scope stack */ struct symbol *name; /* Name of this scope */ uint slots; /* Variable slots */ - int active; /* Currently entered */ + byte active; /* Currently entered */ + byte soft_scopes; /* Number of soft scopes above */ }; struct bytestring { @@ -190,6 +191,9 @@ struct symbol *cf_get_symbol(const byte *c); struct symbol *cf_default_name(char *template, int *counter); struct symbol *cf_localize_symbol(struct symbol *sym); +static inline int cf_symbol_is_local(struct symbol *sym) +{ return (sym->scope == conf_this_scope) && !conf_this_scope->soft_scopes; } + /** * cf_define_symbol - define meaning of a symbol * @sym: symbol to be defined @@ -213,6 +217,9 @@ struct symbol *cf_localize_symbol(struct symbol *sym); void cf_push_scope(struct symbol *); void cf_pop_scope(void); +void cf_push_soft_scope(void); +void cf_pop_soft_scope(void); + char *cf_symbol_class_name(struct symbol *sym); /* Parser */ diff --git a/sysdep/unix/main.c b/sysdep/unix/main.c index 392aff9d..f87d0c43 100644 --- a/sysdep/unix/main.c +++ b/sysdep/unix/main.c @@ -116,7 +116,7 @@ add_num_const(char *name, int val, const char *file, const uint line) struct f_val *v = cfg_alloc(sizeof(struct f_val)); *v = (struct f_val) { .type = T_INT, .val.i = val }; struct symbol *sym = cf_get_symbol(name); - if (sym->class && (sym->scope == conf_this_scope)) + if (sym->class && cf_symbol_is_local(sym)) cf_error("Error reading value for %s from %s:%d: already defined", name, file, line); cf_define_symbol(sym, SYM_CONSTANT | T_INT, val, v); -- cgit v1.2.3 From e858dce757f8766a29f2151ec3ffef3df85af66b Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Thu, 28 Jul 2022 19:49:03 +0200 Subject: Moved the thread starting code to IO loop code --- lib/coro.h | 29 -------- lib/timer.c | 3 +- lib/timer.h | 1 - nest/proto.c | 1 - nest/protocol.h | 4 +- nest/rt-table.c | 2 +- sysdep/unix/Makefile | 2 +- sysdep/unix/coroutine.c | 192 ------------------------------------------------ sysdep/unix/domain.c | 116 +++++++++++++++++++++++++++++ sysdep/unix/io-loop.c | 50 +++++++++++-- sysdep/unix/io-loop.h | 4 + sysdep/unix/log.c | 10 +-- 12 files changed, 172 insertions(+), 242 deletions(-) delete mode 100644 lib/coro.h delete mode 100644 sysdep/unix/coroutine.c create mode 100644 sysdep/unix/domain.c (limited to 'sysdep/unix') diff --git a/lib/coro.h b/lib/coro.h deleted file mode 100644 index 17ccff89..00000000 --- a/lib/coro.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * BIRD Coroutines - * - * (c) 2017 Martin Mares - * (c) 2020-2021 Maria Matejka - * - * Can be freely distributed and used under the terms of the GNU GPL. - */ - -#ifndef _BIRD_CORO_H_ -#define _BIRD_CORO_H_ - -#include "lib/resource.h" - -/* A completely opaque coroutine handle. */ -struct coroutine; - -/* Coroutines are independent threads bound to pools. - * You request a coroutine by calling coro_run(). - * It is forbidden to free a running coroutine from outside. - * The running coroutine must free itself by rfree() before returning. - */ -struct coroutine *coro_run(pool *, void (*entry)(void *), void *data); - -/* Get self. */ -extern _Thread_local struct coroutine *this_coro; - - -#endif diff --git a/lib/timer.c b/lib/timer.c index c1241ba7..ff6975a4 100644 --- a/lib/timer.c +++ b/lib/timer.c @@ -32,7 +32,6 @@ #include "nest/bird.h" -#include "lib/coro.h" #include "lib/heap.h" #include "lib/resource.h" #include "lib/timer.h" @@ -117,7 +116,7 @@ tm_set_in_tl(timer *t, btime when, struct timeloop *local_timeloop) t->loop = local_timeloop; - if ((t->index == 1) && (local_timeloop->coro != this_coro)) + if (t->index == 1) birdloop_ping(local_timeloop->loop); } diff --git a/lib/timer.h b/lib/timer.h index 04544ace..555fc96f 100644 --- a/lib/timer.h +++ b/lib/timer.h @@ -41,7 +41,6 @@ struct timeloop BUFFER_(timer *) timers; struct domain_generic *domain; struct birdloop *loop; - struct coroutine *coro; }; #define TLOCK_TIMER_ASSERT(loop) ASSERT_DIE((loop)->domain && DG_IS_LOCKED((loop)->domain)) diff --git a/nest/proto.c b/nest/proto.c index 17cd08f7..1d480ba2 100644 --- a/nest/proto.c +++ b/nest/proto.c @@ -15,7 +15,6 @@ #include "lib/event.h" #include "lib/timer.h" #include "lib/string.h" -#include "lib/coro.h" #include "conf/conf.h" #include "nest/rt.h" #include "nest/iface.h" diff --git a/nest/protocol.h b/nest/protocol.h index d6224015..3c823ae1 100644 --- a/nest/protocol.h +++ b/nest/protocol.h @@ -134,7 +134,7 @@ struct proto { u32 debug; /* Debugging flags */ u32 mrtdump; /* MRTDump flags */ uint active_channels; /* Number of active channels */ - uint active_coroutines; /* Number of active coroutines */ + uint active_loops; /* Number of active IO loops */ byte net_type; /* Protocol network type (NET_*), 0 for undefined */ byte disabled; /* Manually disabled */ byte vrf_set; /* Related VRF instance (above) is defined */ @@ -342,7 +342,7 @@ void proto_notify_state(struct proto *p, unsigned state); */ static inline int proto_is_inactive(struct proto *p) -{ return (p->active_channels == 0) && (p->active_coroutines == 0); } +{ return (p->active_channels == 0) && (p->active_loops == 0); } /* diff --git a/nest/rt-table.c b/nest/rt-table.c index ca092678..65fc142a 100644 --- a/nest/rt-table.c +++ b/nest/rt-table.c @@ -1984,7 +1984,7 @@ rt_export_stopped(void *data) /* Reporting the hook as finished. */ CALL(tab->done, hook); - /* Free the hook together with its coroutine. */ + /* Free the hook. */ rfree(hook->pool); } diff --git a/sysdep/unix/Makefile b/sysdep/unix/Makefile index 9831dc96..3f1a8b3a 100644 --- a/sysdep/unix/Makefile +++ b/sysdep/unix/Makefile @@ -1,4 +1,4 @@ -src := alloc.c io.c io-loop.c krt.c log.c main.c random.c coroutine.c +src := alloc.c io.c io-loop.c krt.c log.c main.c random.c domain.c obj := $(src-o-files) $(all-daemon) $(cf-local) diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c deleted file mode 100644 index 4758c056..00000000 --- a/sysdep/unix/coroutine.c +++ /dev/null @@ -1,192 +0,0 @@ -/* - * BIRD Coroutines - * - * (c) 2017 Martin Mares - * (c) 2020 Maria Matejka - * - * Can be freely distributed and used under the terms of the GNU GPL. - */ - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#undef LOCAL_DEBUG - -#undef DEBUG_LOCKING - -#include "lib/birdlib.h" -#include "lib/locking.h" -#include "lib/coro.h" -#include "lib/resource.h" -#include "lib/timer.h" - -#include "conf/conf.h" - -#define CORO_STACK_SIZE 65536 - -/* - * Implementation of coroutines based on POSIX threads - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Locking subsystem - */ - -_Thread_local struct lock_order locking_stack = {}; -_Thread_local struct domain_generic **last_locked = NULL; - -#define ASSERT_NO_LOCK ASSERT_DIE(last_locked == NULL) - -struct domain_generic { - pthread_mutex_t mutex; - uint order; - struct domain_generic **prev; - struct lock_order *locked_by; - const char *name; -}; - -#define DOMAIN_INIT(_name, _order) { .mutex = PTHREAD_MUTEX_INITIALIZER, .name = _name, .order = _order } - -static struct domain_generic the_bird_domain_gen = DOMAIN_INIT("The BIRD", OFFSETOF(struct lock_order, the_bird)); - -DOMAIN(the_bird) the_bird_domain = { .the_bird = &the_bird_domain_gen }; - -struct domain_generic * -domain_new(const char *name, uint order) -{ - ASSERT_DIE(order < sizeof(struct lock_order)); - struct domain_generic *dg = xmalloc(sizeof(struct domain_generic)); - *dg = (struct domain_generic) DOMAIN_INIT(name, order); - return dg; -} - -void -domain_free(struct domain_generic *dg) -{ - pthread_mutex_destroy(&dg->mutex); - xfree(dg); -} - -uint dg_order(struct domain_generic *dg) -{ - return dg->order; -} - -void do_lock(struct domain_generic *dg, struct domain_generic **lsp) -{ - if ((char *) lsp - (char *) &locking_stack != dg->order) - bug("Trying to lock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); - - if (lsp <= last_locked) - bug("Trying to lock in a bad order"); - if (*lsp) - bug("Inconsistent locking stack state on lock"); - - btime lock_begin = current_time(); - pthread_mutex_lock(&dg->mutex); - btime duration = current_time() - lock_begin; - if (config && (duration > config->watchdog_warning)) - log(L_WARN "Locking of %s took %d ms", dg->name, (int) (duration TO_MS)); - - if (dg->prev || dg->locked_by) - bug("Previous unlock not finished correctly"); - dg->prev = last_locked; - *lsp = dg; - last_locked = lsp; - dg->locked_by = &locking_stack; -} - -void do_unlock(struct domain_generic *dg, struct domain_generic **lsp) -{ - if ((char *) lsp - (char *) &locking_stack != dg->order) - bug("Trying to unlock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); - - if (dg->locked_by != &locking_stack) - bug("Inconsistent domain state on unlock"); - if ((last_locked != lsp) || (*lsp != dg)) - bug("Inconsistent locking stack state on unlock"); - dg->locked_by = NULL; - last_locked = dg->prev; - *lsp = NULL; - dg->prev = NULL; - pthread_mutex_unlock(&dg->mutex); -} - -/* Coroutines */ -struct coroutine { - resource r; - pthread_t id; - pthread_attr_t attr; - void (*entry)(void *); - void *data; -}; - -static _Thread_local _Bool coro_cleaned_up = 0; - -static void coro_free(resource *r) -{ - struct coroutine *c = (void *) r; - ASSERT_DIE(pthread_equal(pthread_self(), c->id)); - pthread_attr_destroy(&c->attr); - coro_cleaned_up = 1; -} - -static struct resclass coro_class = { - .name = "Coroutine", - .size = sizeof(struct coroutine), - .free = coro_free, -}; - -_Thread_local struct coroutine *this_coro = NULL; - -static void *coro_entry(void *p) -{ - struct coroutine *c = p; - - ASSERT_DIE(c->entry); - - this_coro = c; - - c->entry(c->data); - ASSERT_DIE(coro_cleaned_up); - - return NULL; -} - -struct coroutine *coro_run(pool *p, void (*entry)(void *), void *data) -{ - ASSERT_DIE(entry); - ASSERT_DIE(p); - - struct coroutine *c = ralloc(p, &coro_class); - - c->entry = entry; - c->data = data; - - int e = 0; - - if (e = pthread_attr_init(&c->attr)) - die("pthread_attr_init() failed: %M", e); - - if (e = pthread_attr_setstacksize(&c->attr, CORO_STACK_SIZE)) - die("pthread_attr_setstacksize(%u) failed: %M", CORO_STACK_SIZE, e); - - if (e = pthread_attr_setdetachstate(&c->attr, PTHREAD_CREATE_DETACHED)) - die("pthread_attr_setdetachstate(PTHREAD_CREATE_DETACHED) failed: %M", e); - - if (e = pthread_create(&c->id, &c->attr, coro_entry, c)) - die("pthread_create() failed: %M", e); - - return c; -} diff --git a/sysdep/unix/domain.c b/sysdep/unix/domain.c new file mode 100644 index 00000000..0a5858a6 --- /dev/null +++ b/sysdep/unix/domain.c @@ -0,0 +1,116 @@ +/* + * BIRD Locking + * + * (c) 2020 Maria Matejka + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#undef LOCAL_DEBUG + +#undef DEBUG_LOCKING + +#include "lib/birdlib.h" +#include "lib/locking.h" +#include "lib/resource.h" +#include "lib/timer.h" + +#include "conf/conf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Locking subsystem + */ + +_Thread_local struct lock_order locking_stack = {}; +_Thread_local struct domain_generic **last_locked = NULL; + +#define ASSERT_NO_LOCK ASSERT_DIE(last_locked == NULL) + +struct domain_generic { + pthread_mutex_t mutex; + uint order; + struct domain_generic **prev; + struct lock_order *locked_by; + const char *name; +}; + +#define DOMAIN_INIT(_name, _order) { .mutex = PTHREAD_MUTEX_INITIALIZER, .name = _name, .order = _order } + +static struct domain_generic the_bird_domain_gen = DOMAIN_INIT("The BIRD", OFFSETOF(struct lock_order, the_bird)); + +DOMAIN(the_bird) the_bird_domain = { .the_bird = &the_bird_domain_gen }; + +struct domain_generic * +domain_new(const char *name, uint order) +{ + ASSERT_DIE(order < sizeof(struct lock_order)); + struct domain_generic *dg = xmalloc(sizeof(struct domain_generic)); + *dg = (struct domain_generic) DOMAIN_INIT(name, order); + return dg; +} + +void +domain_free(struct domain_generic *dg) +{ + pthread_mutex_destroy(&dg->mutex); + xfree(dg); +} + +uint dg_order(struct domain_generic *dg) +{ + return dg->order; +} + +void do_lock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to lock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + + if (lsp <= last_locked) + bug("Trying to lock in a bad order"); + if (*lsp) + bug("Inconsistent locking stack state on lock"); + + btime lock_begin = current_time(); + pthread_mutex_lock(&dg->mutex); + btime duration = current_time() - lock_begin; + if (config && (duration > config->watchdog_warning)) + log(L_WARN "Locking of %s took %d ms", dg->name, (int) (duration TO_MS)); + + if (dg->prev || dg->locked_by) + bug("Previous unlock not finished correctly"); + dg->prev = last_locked; + *lsp = dg; + last_locked = lsp; + dg->locked_by = &locking_stack; +} + +void do_unlock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to unlock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + + if (dg->locked_by != &locking_stack) + bug("Inconsistent domain state on unlock"); + if ((last_locked != lsp) || (*lsp != dg)) + bug("Inconsistent locking stack state on unlock"); + dg->locked_by = NULL; + last_locked = dg->prev; + *lsp = NULL; + dg->prev = NULL; + pthread_mutex_unlock(&dg->mutex); +} diff --git a/sysdep/unix/io-loop.c b/sysdep/unix/io-loop.c index dfb2ce49..3419d9d8 100644 --- a/sysdep/unix/io-loop.c +++ b/sysdep/unix/io-loop.c @@ -17,7 +17,6 @@ #include "nest/bird.h" #include "lib/buffer.h" -#include "lib/coro.h" #include "lib/lists.h" #include "lib/resource.h" #include "lib/event.h" @@ -28,6 +27,8 @@ #include "sysdep/unix/io-loop.h" #include "conf/conf.h" +#define THREAD_STACK_SIZE 65536 /* To be lowered in near future */ + /* * Current thread context */ @@ -132,11 +133,10 @@ wakeup_do_kick(struct birdloop *loop) pipe_kick(loop->wakeup_fds[1]); } -void -birdloop_ping(struct birdloop *loop) +static inline void +birdloop_do_ping(struct birdloop *loop) { - u32 ping_sent = atomic_fetch_add_explicit(&loop->ping_sent, 1, memory_order_acq_rel); - if (ping_sent) + if (atomic_fetch_add_explicit(&loop->ping_sent, 1, memory_order_acq_rel)) return; if (loop == birdloop_wakeup_masked) @@ -145,6 +145,15 @@ birdloop_ping(struct birdloop *loop) wakeup_do_kick(loop); } +void +birdloop_ping(struct birdloop *loop) +{ + if (birdloop_inside(loop) && !loop->ping_pending) + loop->ping_pending++; + else + birdloop_do_ping(loop); +} + /* * Sockets @@ -336,7 +345,7 @@ birdloop_init(void) birdloop_enter_locked(&main_birdloop); } -static void birdloop_main(void *arg); +static void *birdloop_main(void *arg); struct birdloop * birdloop_new(pool *pp, uint order, const char *name) @@ -357,7 +366,19 @@ birdloop_new(pool *pp, uint order, const char *name) timers_init(&loop->time, p); sockets_init(loop); - loop->time.coro = coro_run(p, birdloop_main, loop); + int e = 0; + + if (e = pthread_attr_init(&loop->thread_attr)) + die("pthread_attr_init() failed: %M", e); + + if (e = pthread_attr_setstacksize(&loop->thread_attr, THREAD_STACK_SIZE)) + die("pthread_attr_setstacksize(%u) failed: %M", THREAD_STACK_SIZE, e); + + if (e = pthread_attr_setdetachstate(&loop->thread_attr, PTHREAD_CREATE_DETACHED)) + die("pthread_attr_setdetachstate(PTHREAD_CREATE_DETACHED) failed: %M", e); + + if (e = pthread_create(&loop->thread_id, &loop->thread_attr, birdloop_main, loop)) + die("pthread_create() failed: %M", e); birdloop_leave(loop); @@ -393,6 +414,10 @@ void birdloop_free(struct birdloop *loop) { ASSERT_DIE(loop->links == 0); + ASSERT_DIE(pthread_equal(pthread_self(), loop->thread_id)); + + pthread_attr_destroy(&loop->thread_attr); + domain_free(loop->time.domain); rfree(loop->pool); } @@ -423,6 +448,13 @@ birdloop_leave_locked(struct birdloop *loop) /* Check the current context */ ASSERT_DIE(birdloop_current == loop); + /* Send pending pings */ + if (loop->ping_pending) + { + loop->ping_pending = 0; + birdloop_do_ping(loop); + } + /* Restore the old context */ birdloop_current = loop->prev_loop; } @@ -466,7 +498,7 @@ birdloop_unlink(struct birdloop *loop) loop->links--; } -static void +static void * birdloop_main(void *arg) { struct birdloop *loop = arg; @@ -532,6 +564,8 @@ birdloop_main(void *arg) birdloop_leave(loop); loop->stopped(loop->stop_data); + + return NULL; } diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h index 4024b6c5..ca4322fc 100644 --- a/sysdep/unix/io-loop.h +++ b/sysdep/unix/io-loop.h @@ -21,9 +21,13 @@ struct birdloop u8 poll_changed; u8 close_scheduled; + uint ping_pending; _Atomic u32 ping_sent; int wakeup_fds[2]; + pthread_t thread_id; + pthread_attr_t thread_attr; + uint links; void (*stopped)(void *data); diff --git a/sysdep/unix/log.c b/sysdep/unix/log.c index f48588b6..185231e8 100644 --- a/sysdep/unix/log.c +++ b/sysdep/unix/log.c @@ -36,10 +36,10 @@ static FILE *dbgf; static list *current_log_list; static char *current_syslog_name; /* NULL -> syslog closed */ -static _Atomic uint max_coro_id = ATOMIC_VAR_INIT(1); -static _Thread_local uint this_coro_id; +static _Atomic uint max_thread_id = ATOMIC_VAR_INIT(1); +static _Thread_local uint this_thread_id; -#define THIS_CORO_ID (this_coro_id ?: (this_coro_id = atomic_fetch_add_explicit(&max_coro_id, 1, memory_order_acq_rel))) +#define THIS_THREAD_ID (this_thread_id ?: (this_thread_id = atomic_fetch_add_explicit(&max_thread_id, 1, memory_order_acq_rel))) #include @@ -183,7 +183,7 @@ log_commit(int class, buffer *buf) l->pos += msg_len; } - fprintf(l->fh, "%s [%04x] <%s> ", tbuf, THIS_CORO_ID, class_names[class]); + fprintf(l->fh, "%s [%04x] <%s> ", tbuf, THIS_THREAD_ID, class_names[class]); } fputs(buf->start, l->fh); fputc('\n', l->fh); @@ -329,7 +329,7 @@ debug(const char *msg, ...) sec = dbg_time.tv_sec - dbg_time_start.tv_sec - 1; } - int n = bsnprintf(pos, max, "%u.%09u: [%04x] ", sec, nsec, THIS_CORO_ID); + int n = bsnprintf(pos, max, "%u.%09u: [%04x] ", sec, nsec, THIS_THREAD_ID); pos += n; max -= n; -- cgit v1.2.3 From 058ed711397df75350d905fc135758a6470c0143 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Fri, 12 Nov 2021 22:58:40 +0100 Subject: Introducing basic RCU primitives for lock-less shared data structures --- lib/Makefile | 2 +- lib/io-loop.h | 4 +++ lib/locking.h | 1 + lib/rcu.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/rcu.h | 55 +++++++++++++++++++++++++++++++++++ lib/resource.c | 2 ++ sysdep/unix/io-loop.c | 9 +++++- sysdep/unix/io-loop.h | 4 +++ 8 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 lib/rcu.c create mode 100644 lib/rcu.h (limited to 'sysdep/unix') diff --git a/lib/Makefile b/lib/Makefile index 15f757d9..f4ade9a6 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -1,4 +1,4 @@ -src := a-path.c a-set.c bitmap.c bitops.c blake2s.c blake2b.c checksum.c event.c flowspec.c idm.c ip.c lists.c mac.c md5.c mempool.c net.c patmatch.c printf.c resource.c sha1.c sha256.c sha512.c slab.c slists.c strtoul.c tbf.c timer.c xmalloc.c +src := a-path.c a-set.c bitmap.c bitops.c blake2s.c blake2b.c checksum.c event.c flowspec.c idm.c ip.c lists.c mac.c md5.c mempool.c net.c patmatch.c printf.c rcu.c resource.c sha1.c sha256.c sha512.c slab.c slists.c strtoul.c tbf.c timer.c xmalloc.c obj := $(src-o-files) $(all-daemon) diff --git a/lib/io-loop.h b/lib/io-loop.h index dec7d040..2450a609 100644 --- a/lib/io-loop.h +++ b/lib/io-loop.h @@ -51,4 +51,8 @@ void birdloop_unlink(struct birdloop *loop); void birdloop_ping(struct birdloop *loop); void birdloop_init(void); + +/* Yield for a little while. Use only in special cases. */ +void birdloop_yield(void); + #endif /* _BIRD_IO_LOOP_H_ */ diff --git a/lib/locking.h b/lib/locking.h index 8ea1c968..a9a8aa9b 100644 --- a/lib/locking.h +++ b/lib/locking.h @@ -16,6 +16,7 @@ struct lock_order { struct domain_generic *the_bird; struct domain_generic *proto; struct domain_generic *rtable; + struct domain_generic *resource; }; extern _Thread_local struct lock_order locking_stack; diff --git a/lib/rcu.c b/lib/rcu.c new file mode 100644 index 00000000..83fdd022 --- /dev/null +++ b/lib/rcu.c @@ -0,0 +1,79 @@ +/* + * BIRD Library -- Read-Copy-Update Basic Operations + * + * (c) 2021 Maria Matejka + * (c) 2021 CZ.NIC z.s.p.o. + * + * Can be freely distributed and used under the terms of the GNU GPL. + * Note: all the relevant patents shall be expired. + * + * Using the Supplementary Material for User-Level Implementations of Read-Copy-Update + * by Matthieu Desnoyers, Paul E. McKenney, Alan S. Stern, Michel R. Dagenais and Jonathan Walpole + * obtained from https://www.efficios.com/pub/rcu/urcu-supp-accepted.pdf + */ + +#include "lib/rcu.h" +#include "lib/io-loop.h" +#include "lib/locking.h" + +_Atomic uint rcu_gp_ctl = RCU_NEST_CNT; +_Thread_local struct rcu_birdloop *this_rcu_birdloop = NULL; + +static list rcu_birdloop_list; + +static struct rcu_birdloop main_rcu_birdloop; + +DEFINE_DOMAIN(resource); +static DOMAIN(resource) rcu_domain; + +static int +rcu_gp_ongoing(_Atomic uint *ctl) +{ + uint val = atomic_load(ctl); + return (val & RCU_NEST_CNT) && ((val ^ rcu_gp_ctl) & RCU_GP_PHASE); +} + +static void +update_counter_and_wait(void) +{ + atomic_fetch_xor(&rcu_gp_ctl, RCU_GP_PHASE); + struct rcu_birdloop *rc; + WALK_LIST(rc, rcu_birdloop_list) + while (rcu_gp_ongoing(&rc->ctl)) + birdloop_yield(); +} + +void +synchronize_rcu(void) +{ + LOCK_DOMAIN(resource, rcu_domain); + update_counter_and_wait(); + update_counter_and_wait(); + UNLOCK_DOMAIN(resource, rcu_domain); +} + +void +rcu_birdloop_start(struct rcu_birdloop *rc) +{ + LOCK_DOMAIN(resource, rcu_domain); + add_tail(&rcu_birdloop_list, &rc->n); + this_rcu_birdloop = rc; + UNLOCK_DOMAIN(resource, rcu_domain); +} + +void +rcu_birdloop_stop(struct rcu_birdloop *rc) +{ + LOCK_DOMAIN(resource, rcu_domain); + this_rcu_birdloop = NULL; + rem_node(&rc->n); + UNLOCK_DOMAIN(resource, rcu_domain); +} + +void +rcu_init(void) +{ + rcu_domain = DOMAIN_NEW(resource, "Read-Copy-Update"); + init_list(&rcu_birdloop_list); + rcu_birdloop_start(&main_rcu_birdloop); +} diff --git a/lib/rcu.h b/lib/rcu.h new file mode 100644 index 00000000..c537a1ef --- /dev/null +++ b/lib/rcu.h @@ -0,0 +1,55 @@ +/* + * BIRD Library -- Read-Copy-Update Basic Operations + * + * (c) 2021 Maria Matejka + * (c) 2021 CZ.NIC z.s.p.o. + * + * Can be freely distributed and used under the terms of the GNU GPL. + * Note: all the relevant patents shall be expired. + */ + +#ifndef _BIRD_RCU_H_ +#define _BIRD_RCU_H_ + +#include "lib/birdlib.h" +#include "lib/lists.h" +#include + +#define RCU_GP_PHASE 0x100000 +#define RCU_NEST_MASK 0x0fffff +#define RCU_NEST_CNT 0x000001 + +extern _Atomic uint rcu_gp_ctl; + +struct rcu_birdloop { + node n; + _Atomic uint ctl; +}; + +extern _Thread_local struct rcu_birdloop *this_rcu_birdloop; + +static inline void rcu_read_lock(void) +{ + uint cmp = atomic_load_explicit(&this_rcu_birdloop->ctl, memory_order_acquire); + + if (cmp & RCU_NEST_MASK) + atomic_store_explicit(&this_rcu_birdloop->ctl, cmp + RCU_NEST_CNT, memory_order_relaxed); + else + atomic_store(&this_rcu_birdloop->ctl, atomic_load_explicit(&rcu_gp_ctl, memory_order_acquire)); +} + +static inline void rcu_read_unlock(void) +{ + atomic_fetch_sub(&this_rcu_birdloop->ctl, RCU_NEST_CNT); +} + +void synchronize_rcu(void); + +/* Registering and unregistering a birdloop. To be called from birdloop implementation */ +void rcu_birdloop_start(struct rcu_birdloop *); +void rcu_birdloop_stop(struct rcu_birdloop *); + +/* Run this from resource init */ +void rcu_init(void); + +#endif diff --git a/lib/resource.c b/lib/resource.c index a33fd214..898fb533 100644 --- a/lib/resource.c +++ b/lib/resource.c @@ -14,6 +14,7 @@ #include "nest/bird.h" #include "lib/resource.h" #include "lib/string.h" +#include "lib/rcu.h" /** * DOC: Resource pools @@ -279,6 +280,7 @@ void resource_init(void) { resource_sys_init(); + rcu_init(); root_pool.r.class = &pool_class; root_pool.name = "Root"; diff --git a/sysdep/unix/io-loop.c b/sysdep/unix/io-loop.c index 3419d9d8..3e3fc31a 100644 --- a/sysdep/unix/io-loop.c +++ b/sysdep/unix/io-loop.c @@ -416,6 +416,7 @@ birdloop_free(struct birdloop *loop) ASSERT_DIE(loop->links == 0); ASSERT_DIE(pthread_equal(pthread_self(), loop->thread_id)); + rcu_birdloop_stop(&loop->rcu); pthread_attr_destroy(&loop->thread_attr); domain_free(loop->time.domain); @@ -505,6 +506,8 @@ birdloop_main(void *arg) timer *t; int rv, timeout; + rcu_birdloop_start(&loop->rcu); + btime loop_begin = current_time(); tmp_init(loop->pool); @@ -568,4 +571,8 @@ birdloop_main(void *arg) return NULL; } - +void +birdloop_yield(void) +{ + usleep(100); +} diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h index ca4322fc..31c40459 100644 --- a/sysdep/unix/io-loop.h +++ b/sysdep/unix/io-loop.h @@ -7,6 +7,8 @@ #ifndef _BIRD_SYSDEP_UNIX_IO_LOOP_H_ #define _BIRD_SYSDEP_UNIX_IO_LOOP_H_ +#include "lib/rcu.h" + struct birdloop { pool *pool; @@ -28,6 +30,8 @@ struct birdloop pthread_t thread_id; pthread_attr_t thread_attr; + struct rcu_birdloop rcu; + uint links; void (*stopped)(void *data); -- cgit v1.2.3