diff options
122 files changed, 11317 insertions, 4867 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e7eaa66b..2c1f7175 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,21 +134,11 @@ docker_fedora-34-amd64: IMG_NAME: "fedora-34-amd64" <<: *docker_build -docker_centos-7-amd64: - variables: - IMG_NAME: "centos-7-amd64" - <<: *docker_build - docker_centos-8-amd64: variables: IMG_NAME: "centos-8-amd64" <<: *docker_build -docker_ubuntu-14_04-amd64: - variables: - IMG_NAME: "ubuntu-14.04-amd64" - <<: *docker_build - docker_ubuntu-16_04-amd64: variables: IMG_NAME: "ubuntu-16.04-amd64" @@ -312,18 +302,10 @@ build-fedora-34-amd64: <<: *build-linux image: registry.nic.cz/labs/bird:fedora-33-amd64 -build-centos-7-amd64: - <<: *build-linux - image: registry.nic.cz/labs/bird:centos-7-amd64 - build-centos-8-amd64: <<: *build-linux image: registry.nic.cz/labs/bird:centos-8-amd64 -build-ubuntu-14_04-amd64: - <<: *build-linux - image: registry.nic.cz/labs/bird:ubuntu-14.04-amd64 - build-ubuntu-16_04-amd64: <<: *build-linux image: registry.nic.cz/labs/bird:ubuntu-16.04-amd64 @@ -468,13 +450,6 @@ pkg-fedora-34-amd64: needs: [build-fedora-34-amd64] image: registry.nic.cz/labs/bird:fedora-34-amd64 -pkg-centos-7-amd64: - <<: *pkg-rpm-wa - variables: - LC_ALL: en_US.UTF-8 - needs: [build-centos-7-amd64] - image: registry.nic.cz/labs/bird:centos-7-amd64 - pkg-centos-8-amd64: <<: *pkg-rpm-wa needs: [build-centos-8-amd64] @@ -27,9 +27,9 @@ Requirements For compiling BIRD you need these programs and libraries: - - GNU C Compiler (or LLVM Clang) + - GNU C Compiler (or LLVM Clang) capable of compiling C11 code - GNU Make - - GNU Bison + - GNU Bison (at least 3.0) - GNU M4 - Flex @@ -1,5 +1,6 @@ dnl ** Additional Autoconf tests for BIRD configure script dnl ** (c) 1999 Martin Mares <mj@ucw.cz> +dnl ** (c) 2021 Maria Matejka <mq@jmq.cz> AC_DEFUN([BIRD_CHECK_THREAD_LOCAL], [ @@ -9,14 +10,23 @@ AC_DEFUN([BIRD_CHECK_THREAD_LOCAL], AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [ - _Thread_local static int x = 42; + static _Thread_local int x = 42; ], [] ) ], [bird_cv_thread_local=yes], + [AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM( + [ + static __thread int x = 42; + ], + [] + ) + ], + [bird_cv_thread_local=__thread], [bird_cv_thread_local=no] - ) + )]) ) ]) diff --git a/bird-gdb.py b/bird-gdb.py index 3cf65a9c..e11f9757 100644 --- a/bird-gdb.py +++ b/bird-gdb.py @@ -1,12 +1,16 @@ +import itertools +import functools + class BIRDPrinter: def __init__(self, val): self.val = val @classmethod def lookup(cls, val): - if val.type.code != cls.typeCode: + t = val.type.strip_typedefs() + if t.code != cls.typeCode: return None - if val.type.tag != cls.typeTag: + if t.tag != cls.typeTag: return None return cls(val) @@ -123,7 +127,7 @@ class BIRDFLinePrinter(BIRDPrinter): "n": n, "code": str(self.val['items'][n]['fi_code']), } if n % 8 == 0 else str(self.val['items'][n]['fi_code']) for n in range(cnt)])) - + class BIRDFExecStackPrinter(BIRDPrinter): "Print BIRD's struct f_exec_stack" @@ -141,6 +145,315 @@ class BIRDFExecStackPrinter(BIRDPrinter): "n": n } for n in range(cnt-1, -1, -1) ]) + +class BIRD: + def skip_back(t, i, v): + if isinstance(t, str): + t = gdb.lookup_type(t) + elif isinstance(t, gdb.Value): + t = gdb.lookup_type(t.string()) + elif not isinstance(t, gdb.Type): + raise Exception(f"First argument of skip_back(t, i, v) must be a type, got {type(t)}") + + t = t.strip_typedefs() + nullptr = gdb.Value(0).cast(t.pointer()) + + if isinstance(i, gdb.Value): + i = i.string() + elif not isinstance(i, str): + raise Exception(f"Second argument of skip_back(t, i, v) must be a item name, got {type(i)}") + + if not isinstance(v, gdb.Value): + raise Exception(f"Third argument of skip_back(t, i, v) must be a value, got {type(v)}") + if v.type.code != gdb.TYPE_CODE_PTR and v.type.code != gdb.TYPE_CODE_REF: + raise Exception(f"Third argument of skip_back(t, i, v) must be a pointer, is {v.type} ({v.type.code})") + if v.type.target().strip_typedefs() != nullptr[i].type: + raise Exception(f"Third argument of skip_back(t, i, v) points to type {v.type.target().strip_typedefs()}, should be {nullptr[i].type}") + + uintptr_t = gdb.lookup_type("uintptr_t") + taddr = v.dereference().address.cast(uintptr_t) - nullptr[i].address.cast(uintptr_t) + return gdb.Value(taddr).cast(t.pointer()) + + class skip_back_gdb(gdb.Function): + "Given address of a structure item, returns address of the structure, as the SKIP_BACK macro does" + def __init__(self): + gdb.Function.__init__(self, "SKIP_BACK") + + def invoke(self, t, i, v): + return BIRD.skip_back(t, i, v) + + +BIRD.skip_back_gdb() + + +class BIRDList: + def __init__(self, val): + ltype = val.type.strip_typedefs() + if ltype.code != gdb.TYPE_CODE_UNION or ltype.tag != "list": + raise Exception(f"Not a list, is type {ltype}") + + self.head = val["head"] + self.tail_node = val["tail_node"] + + if str(self.head.address) == '0x0': + raise Exception("List head is NULL") + + if str(self.tail_node["prev"].address) == '0x0': + raise Exception("List tail is NULL") + + def __iter__(self): + cur = self.head + while cur.dereference() != self.tail_node: + yield cur + cur = cur.dereference()["next"] + + def __len__(self): + return sum([1 for _ in self]) + + def __getitem__(self, key): + for item in itertools.islice(self, key): + return item + + raise KeyError("Not enough elements in list") + +class BIRDListLength(gdb.Function): + """Returns length of the list, as in + print $list_length(routing_tables)""" + def __init__(self): + super(BIRDListLength, self).__init__("list_length") + + def invoke(self, l): + return len(BIRDList(l)) + +BIRDListLength() + +class BIRDListItem(gdb.Function): + """Returns n-th item of the list.""" + def __init__(self): + super(BIRDListItem, self).__init__("list_item") + + def invoke(self, l, n, t=None, item="n"): + if t is None: + return BIRDList(l)[n] + else: + return BIRD.skip_back(t, item, BIRDList(l)[n]) + +BIRDListItem() + +class BIRDResourceSize(): + def __init__(self, netto, overhead, free): + self.netto = netto + self.overhead = overhead + self.free = free + + def __str__(self): + ns = str(self.netto) + os = str(self.overhead) + fs = str(self.free) + + return "{: >12s} | {: >12s} | {: >12s}".format(ns, os, fs) + + def __add__(self, val): + return BIRDResourceSize(self.netto + val.netto, self.overhead + val.overhead, self.free + val.free) + +class BIRDResource(): + def __init__(self, val): + self.val = val + + def __str__(self): + return f"Item {self.val.address} of class \"{self.val['class']['name'].string()}\"" + + def memsize(self): + if str(self.val["class"]["memsize"]) == '0x0': + size = self.val["class"]["size"] + ressize = gdb.lookup_type("struct resource").sizeof + return BIRDResourceSize(size - ressize, ressize, 0) + else: + raise Exception(f"Resource class {self.val['class']['name']} with defined memsize() not known by Python") + + def parse(self): + pass + +class BIRDMBResource(BIRDResource): + def __init__(self, val): + self.mbtype = gdb.lookup_type("struct mblock") + self.val = val.cast(self.mbtype) + + def memsize(self): + return BIRDResourceSize(self.val["size"], 8 + self.mbtype.sizeof, 0) + + def __str__(self): + return f"Standalone memory block {self.val.address} of size {self.val['size']}, data at {self.val['data'].address}" + +class BIRDLinPoolResource(BIRDResource): + def __init__(self, val): + self.lptype = gdb.lookup_type("struct linpool") + self.val = val.cast(self.lptype) + self.info = None + self.std = self.ChunkList(self.val["first"]) + self.large = self.ChunkList(self.val["first_large"]) + + class ChunkList: + def __init__(self, val): + self.val = val + + def __iter__(self): + chunk = self.val + while str(chunk) != 0x0: + yield chunk + chunk = chunk.dereference()["next"] + + def __len__(self): + return sum([1 for _ in self]) + + def parse(self): + self.info = { + "std_chunks": len(self.std), + "large_chunks": len(self.large), + } + + def memsize(self): + if self.info is None: + self.parse() + + overhead = (8 - 8*self.val["use_pages"]) + gdb.lookup_type("struct lp_chunk").sizeof + return BIRDResourceSize( + self.val["total"] + self.val["total_large"], + (self.info["std_chunks"] + self.info["large_chunks"]) * overhead, + 0) + + def __str__(self): + if self.info is None: + self.parse() + + return f"Linpool {self.val.address} with {self.info['std_chunks']} standard chunks of size {self.val['chunk_size']} and {self.info['large_chunks']} large chunks" + +class BIRDSlabResource(BIRDResource): + def __init__(self, val): + self.slabtype = gdb.lookup_type("struct slab") + self.val = val.cast(self.slabtype) + self.info = None + + def count_heads(self, which): + self.hcnt = 0 + self.used = 0 + for item in BIRDList(self.val[which + "_heads"]): + self.hcnt += 1 + self.used += item.dereference().cast(self.slheadtype)["num_full"] + + self.info[which + "_heads"] = self.hcnt + self.info[which + "_used"] = self.used + return (self.hcnt, self.used) + + def parse(self): + self.page_size = gdb.lookup_symbol("page_size")[0].value() + self.slheadtype = gdb.lookup_type("struct sl_head") + self.info = {} + self.count_heads("empty") + self.count_heads("partial") + self.count_heads("full") + + def memsize(self): + if self.info is None: + self.parse() + + total_used = self.info["empty_used"] + self.info["partial_used"] + self.info["full_used"] + total_heads = self.info["empty_heads"] + self.info["partial_heads"] + self.info["full_heads"] + + eff_size = total_used * self.val["obj_size"] + free_size = self.info["empty_heads"] * self.page_size + total_size = total_heads * self.page_size + self.slabtype.sizeof + + return BIRDResourceSize( eff_size, total_size - free_size - eff_size, free_size) + + def __str__(self): + if self.info is None: + self.parse() + + return f"Slab {self.val.address} " + ", ".join([ + f"{self.info[x + '_heads']} {x} heads" for x in [ "empty", "partial", "full" ]]) + \ + f", {self.val['objs_per_slab']} objects of size {self.val['obj_size']} per head" + + +class BIRDIOLoopResource(BIRDResource): + def __init__(self, val): + self.iolooptype = gdb.lookup_type("struct birdloop") + self.val = val.cast(self.iolooptype) + self.pages = self.val["pages"] + self.page_size = gdb.lookup_symbol("page_size")[0].value() + + def memsize(self): + return BIRDResourceSize(0, self.iolooptype.sizeof, self.pages['cnt'] * self.page_size) + + def __str__(self): + return f"IO Loop {self.val.address} containing {self.pages['cnt']} free pages (min {self.pages['min']} max {self.pages['max']}), cleanup event {self.pages['cleanup'].dereference().address}: " + \ + ", ".join([ str(p) for p in BIRDList(self.pages["list"])]) + + +class BIRDPoolResource(BIRDResource): + def __init__(self, val): + self.pooltype = gdb.lookup_type("struct pool") + self.resptrtype = gdb.lookup_type("struct resource").pointer() + self.val = val.cast(self.pooltype) + self.inside = BIRDList(self.val["inside"]) + + def __iter__(self): + for val in self.inside: + yield BIRDNewResource(val.cast(self.resptrtype).dereference()) + + def __len__(self): + return len(self.inside) + + def memsize(self): + sum = BIRDResourceSize(0, self.pooltype.sizeof, 0) +# for i in self.items: +# sum += i.memsize() + + return sum + + def __str__(self): +# for i in self.items: +# print(i) + + return f"Resource pool {self.val.address} \"{self.val['name'].string()}\" containing {len(self)} items" + +BIRDResourceMap = { + "mbl_memsize": BIRDMBResource, + "pool_memsize": BIRDPoolResource, + "lp_memsize": BIRDLinPoolResource, + "slab_memsize": BIRDSlabResource, + "birdloop_memsize": BIRDIOLoopResource, + } + +def BIRDNewResource(res): + cms = res["class"].dereference()["memsize"] + for cx in BIRDResourceMap: + if cms == gdb.lookup_symbol(cx)[0].value(): + return BIRDResourceMap[cx](res) + + return BIRDResource(res) + + +class BIRDResourcePrinter(BIRDPrinter): + "Print BIRD's resource" + typeCode = gdb.TYPE_CODE_STRUCT + typeTag = "resource" + + def __init__(self, val): + super(BIRDResourcePrinter, self).__init__(val) + self.resource = BIRDNewResource(val) + self.resourcetype = gdb.lookup_type("struct resource") + + if type(self.resource) == BIRDPoolResource: + self.children = self.pool_children + + def pool_children(self): + return iter([ ("\n", i.val.cast(self.resourcetype)) for i in self.resource ]) + + def to_string(self): + return f"[ {str(self.resource.memsize())} ] {str(self.resource)}" + + def register_printers(objfile): objfile.pretty_printers.append(BIRDFInstPrinter.lookup) objfile.pretty_printers.append(BIRDFValPrinter.lookup) @@ -148,6 +461,7 @@ def register_printers(objfile): objfile.pretty_printers.append(BIRDFLineItemPrinter.lookup) objfile.pretty_printers.append(BIRDFLinePrinter.lookup) objfile.pretty_printers.append(BIRDFExecStackPrinter.lookup) + objfile.pretty_printers.append(BIRDResourcePrinter.lookup) register_printers(gdb.current_objfile()) diff --git a/conf/conf.c b/conf/conf.c index a2b01667..e8f1559a 100644 --- a/conf/conf.c +++ b/conf/conf.c @@ -89,7 +89,7 @@ int undo_available; /* Undo was not requested from last reconfiguration */ struct config * config_alloc(const char *name) { - pool *p = rp_new(&root_pool, "Config"); + pool *p = rp_new(&root_pool, &main_birdloop, "Config"); linpool *l = lp_new_default(p); struct config *c = lp_allocz(l, sizeof(struct config)); @@ -196,7 +196,7 @@ void config_free(struct config *c) { if (c) - rfree(c->pool); + rp_free(c->pool, &root_pool); } void diff --git a/conf/conf.h b/conf/conf.h index 3bc37959..4f6aa6eb 100644 --- a/conf/conf.h +++ b/conf/conf.h @@ -35,6 +35,7 @@ struct config { u32 proto_default_debug; /* Default protocol debug mask */ u32 proto_default_mrtdump; /* Default protocol mrtdump mask */ u32 channel_default_debug; /* Default channel debug mask */ + u16 filter_vstk, filter_estk; /* Filter stack depth */ struct timeformat tf_route; /* Time format for 'show route' */ struct timeformat tf_proto; /* Time format for 'show protocol' */ struct timeformat tf_log; /* Time format for the logfile */ @@ -44,6 +45,7 @@ struct config { int cli_debug; /* Tracing of CLI connections and commands */ int latency_debug; /* I/O loop tracks duration of each event */ + int table_debug; /* Track route propagation through tables */ u32 latency_limit; /* Events with longer duration are logged (us) */ u32 watchdog_warning; /* I/O loop watchdog limit for warning (us) */ u32 watchdog_timeout; /* Watchdog timeout (in seconds, 0 = disabled) */ diff --git a/configure.ac b/configure.ac index 64181d29..5c0cf002 100644 --- a/configure.ac +++ b/configure.ac @@ -36,12 +36,6 @@ AC_ARG_ENABLE([memcheck], [enable_memcheck=yes] ) -AC_ARG_ENABLE([pthreads], - [AS_HELP_STRING([--enable-pthreads], [enable POSIX threads support @<:@try@:>@])], - [], - [enable_pthreads=try] -) - AC_ARG_ENABLE([libssh], [AS_HELP_STRING([--enable-libssh], [enable LibSSH support in RPKI @<:@try@:>@])], [], @@ -125,25 +119,19 @@ if test -z "$GCC" ; then fi BIRD_CHECK_THREAD_LOCAL -if test "$bird_cv_thread_local" = yes ; then - AC_DEFINE([HAVE_THREAD_LOCAL], [1], [Define to 1 if _Thread_local is available]) +if test "$bird_cv_thread_local" = no ; then + AC_MSG_ERROR([This program requires thread local storage.]) +elif test "$bird_cv_thread_local" != yes ; then + AC_DEFINE_UNQUOTED([_Thread_local], [$bird_cv_thread_local], [Legacy _Thread_local]) fi -if test "$enable_pthreads" != no ; then - BIRD_CHECK_PTHREADS +BIRD_CHECK_PTHREADS - if test "$bird_cv_lib_pthreads" = yes ; then - AC_DEFINE([USE_PTHREADS], [1], [Define to 1 if pthreads are enabled]) - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - proto_bfd=bfd - elif test "$enable_pthreads" = yes ; then - AC_MSG_ERROR([POSIX threads not available.]) - fi - - if test "$enable_pthreads" = try ; then - enable_pthreads="$bird_cv_lib_pthreads" - fi +if test "$bird_cv_lib_pthreads" = yes ; then + CFLAGS="$CFLAGS -pthread" + LDFLAGS="$LDFLAGS -pthread" +else + AC_MSG_ERROR([POSIX threads not available.]) fi # This is assumed to be necessary for proper BIRD build @@ -304,8 +292,7 @@ if test "$enable_mpls_kernel" != no ; then fi fi -all_protocols="$proto_bfd babel bgp mrt ospf perf pipe radv rip rpki static" - +all_protocols="bfd babel bgp mrt ospf perf pipe radv rip rpki static" all_protocols=`echo $all_protocols | sed 's/ /,/g'` if test "$with_protocols" = all ; then @@ -351,9 +338,15 @@ case $sysdesc in esac AC_CHECK_HEADERS_ONCE([alloca.h syslog.h]) -AC_CHECK_HEADER([sys/mman.h], [AC_DEFINE([HAVE_MMAP], [1], [Define to 1 if mmap() is available.])]) +AC_CHECK_HEADER([sys/mman.h], [AC_DEFINE([HAVE_MMAP], [1], [Define to 1 if mmap() is available.])], have_mman=no) +AC_CHECK_FUNC([aligned_alloc], [AC_DEFINE([HAVE_ALIGNED_ALLOC], [1], [Define to 1 if aligned_alloc() is available.])], have_aligned_alloc=no) AC_CHECK_MEMBERS([struct sockaddr.sa_len], [], [], [#include <sys/socket.h>]) +if test "$have_aligned_alloc" = "no" && test "$have_mman" = "no" ; then + AC_MSG_ERROR([No means of aligned alloc found. Need mmap() or aligned_alloc().]) +fi + + AC_C_BIGENDIAN( [AC_DEFINE([CPU_BIG_ENDIAN], [1], [Define to 1 if cpu is big endian])], [AC_DEFINE([CPU_LITTLE_ENDIAN], [1], [Define to 1 if cpu is little endian])], @@ -402,7 +395,7 @@ if test "$enable_debug" = yes ; then fi fi - if test "enable_debug_expensive" = yes ; then + if test "$enable_debug_expensive" = yes ; then AC_DEFINE([ENABLE_EXPENSIVE_CHECKS], [1], [Define to 1 if you want to run expensive consistency checks.]) fi fi @@ -467,7 +460,6 @@ AC_MSG_RESULT([ Object directory: $objdir]) AC_MSG_RESULT([ Iproute2 directory: $iproutedir]) AC_MSG_RESULT([ System configuration: $sysdesc]) AC_MSG_RESULT([ Debugging: $enable_debug]) -AC_MSG_RESULT([ POSIX threads: $enable_pthreads]) AC_MSG_RESULT([ Routing protocols: $protocols]) AC_MSG_RESULT([ LibSSH support in RPKI: $enable_libssh]) AC_MSG_RESULT([ Kernel MPLS support: $enable_mpls_kernel]) diff --git a/doc/bird.sgml b/doc/bird.sgml index a63493da..99aa140a 100644 --- a/doc/bird.sgml +++ b/doc/bird.sgml @@ -145,6 +145,13 @@ BIRD executable by configuring out routing protocols you don't use, and <p>You can pass several command-line options to bird: <descrip> + <tag><label id="argv-block">-B <m/exp/</tag> + allocate memory using 2^<cf/exp/ byte sized blocks; + if you're expecting high memory load, raise this to + reduce number of allocated memory pages. For a million routes + in one table, the recommended setting is 18. + Default is your system page size, typically 12 for 4096 bytes. + <tag><label id="argv-config">-c <m/config name/</tag> use given configuration file instead of <it/prefix/<file>/etc/bird.conf</file>. @@ -1700,7 +1707,7 @@ Common route attributes are: <tag><label id="rta-source"><m/enum/ source</tag> what protocol has told me about this route. Possible values: - <cf/RTS_DUMMY/, <cf/RTS_STATIC/, <cf/RTS_INHERIT/, <cf/RTS_DEVICE/, + <cf/RTS_STATIC/, <cf/RTS_INHERIT/, <cf/RTS_DEVICE/, <cf/RTS_RIP/, <cf/RTS_OSPF/, <cf/RTS_OSPF_IA/, <cf/RTS_OSPF_EXT1/, <cf/RTS_OSPF_EXT2/, <cf/RTS_BGP/, <cf/RTS_PIPE/, <cf/RTS_BABEL/. @@ -4126,6 +4133,14 @@ include standard channel config options; see the example below. <tag><label id="pipe-peer-table">peer table <m/table/</tag> Defines secondary routing table to connect to. The primary one is selected by the <cf/table/ keyword. + + <tag><label id="pipe-max-generation">max generation <m/expr/</tag> + Sets maximal generation of route that may pass through this pipe. + The generation value is increased by one by each pipe on its path. + Not meeting this requirement causes an error message complaining about + an overpiped route. If you have long chains of pipes, you probably want + to raise this value; anyway the default of 16 should be enough for even + most strange uses. Maximum is 254. </descrip> <sect1>Attributes diff --git a/doc/threads/.gitignore b/doc/threads/.gitignore new file mode 100644 index 00000000..23f832b7 --- /dev/null +++ b/doc/threads/.gitignore @@ -0,0 +1,2 @@ +*.html +*.pdf diff --git a/doc/threads/00_filter_structure.png b/doc/threads/00_filter_structure.png Binary files differnew file mode 100644 index 00000000..a61603cb --- /dev/null +++ b/doc/threads/00_filter_structure.png diff --git a/doc/threads/00_the_name_of_the_game.md b/doc/threads/00_the_name_of_the_game.md new file mode 100644 index 00000000..ddc4f638 --- /dev/null +++ b/doc/threads/00_the_name_of_the_game.md @@ -0,0 +1,114 @@ +# BIRD Journey to Threads. Chapter 0: The Reason Why. + +BIRD is a fast, robust and memory-efficient routing daemon designed and +implemented at the end of 20th century. Its concept of multiple routing +tables with pipes between them, as well as a procedural filtering language, +has been unique for a long time and is still one of main reasons why people use +BIRD for big loads of routing data. + +## IPv4 / IPv6 duality: Solved + +The original design of BIRD has also some drawbacks. One of these was an idea +of two separate daemons – one BIRD for IPv4 and another BIRD for IPv6, built from the same +codebase, cleverly using `#ifdef IPV6` constructions to implement the +common parts of BIRD algorithms and data structures only once. +If IPv6 adoption went forward as people thought in that time, +it would work; after finishing the worldwide transition to IPv6, people could +just stop building BIRD for IPv4 and drop the `#ifdef`-ed code. + +The history went other way, however. BIRD developers therefore decided to *integrate* +these two versions into one daemon capable of any address family, allowing for +not only IPv6 but for virtually anything. This rework brought quite a lot of +backward-incompatible changes, therefore we decided to release it as a version 2.0. +This work was mostly finished in 2018 and as for March 2021, we have already +switched the 1.6.x branch to a bugfix-only mode. + +## BIRD is single-threaded now + +The second drawback is a single-threaded design. Looking back to 1998, this was +a good idea. A common PC had one single core and BIRD was targeting exactly +this segment. As the years went by, the manufacturers launched multicore x86 chips +(AMD Opteron in 2004, Intel Pentium D in 2005). This ultimately led to a world +where as of March 2021, there is virtually no new PC sold with a single-core CPU. + +Together with these changes, the speed of one single core has not been growing as fast +as the Internet is growing. BIRD is still capable to handle the full BGP table +(868k IPv4 routes in March 2021) with one core, anyway when BIRD starts, it may take +long minutes to converge. + +## Intermezzo: Filters + +In 2018, we took some data we had from large internet exchanges and simulated +a cold start of BIRD as a route server. We used `linux-perf` to find most time-critical +parts of BIRD and it pointed very clearly to the filtering code. It also showed that the +IPv4 version of BIRD v1.6.x is substantially faster than the *integrated* version, while +the IPv6 version was quite as fast as the *integrated* one. + +Here we should show a little bit more about how the filters really work. Let's use +an example of a simple filter: + +``` +filter foo { + if net ~ [10.0.0.0/8+] then reject; + preference = 2 * preference - 41; + accept; +} +``` + +This filter gets translated to an infix internal structure. + + + +When executing, the filter interpreter just walks the filter internal structure recursively in the +right order, executes the instructions, collects their results and finishes by +either rejection or acceptation of the route + +## Filter rework + +Further analysis of the filter code revealed an absurdly-looking result. The +most executed parts of the interpreter function were the `push` CPU +instructions on its very beginning and the `pop` CPU instructions on its very +end. This came from the fact that the interpreter function was quite long, yet +most of the filter instructions used an extremely short path, doing all the +stack manipulation at the beginning, branching by the filter instruction type, +then it executed just several CPU instructions, popped everything from the +stack back and returned. + +After some thoughts how to minimize stack manipulation when everything you need +is to take two numbers and multiply them, we decided to preprocess the filter +internal structure to another structure which is much easier to execute. The +interpreter is now using a data stack and behaves generally as a +postfix-ordered language. We also thought about Lua which showed up to be quite +a lot of work implementing all the glue achieving about the same performance. + +After these changes, we managed to reduce the filter execution time by 10–40%, +depending on how complex the filter is. +Anyway, even this reduction is quite too little when there is one CPU core +running for several minutes while others are sleeping. + +## We need more threads + +As a side effect of the rework, the new filter interpreter is also completely +thread-safe. It seemed to be the way to go – running the filters in parallel +while keeping everything else single-threaded. The main problem of this +solution is a too fine granularity of parallel jobs. We would spend lots of +time on synchronization overhead. + +The only filter parallel execution was also too one-sided, useful only for +configurations with complex filters. In other cases, the major problem is best +route recalculation, OSPF recalculation or also kernel synchronization. +It also turned out to be dirty a lot from the code cleanliness' point of view. + +Therefore we chose to make BIRD multithreaded completely. We designed a way how +to gradually enable parallel computation and best usage of all available CPU +cores. Our goals are three: + +* We want to keep current functionality. Parallel computation should never drop + a useful feature. +* We want to do little steps. No big reworks, even though even the smallest + possible step will need quite a lot of refactoring before. +* We want to be backwards compatible as much as possible. + +*It's still a long road to the version 2.1. This series of texts should document +what is needed to be changed, why we do it and how. In the next chapter, we're +going to describe the structures for routes and their attributes. Stay tuned!* diff --git a/doc/threads/01_the_route_and_its_attributes.md b/doc/threads/01_the_route_and_its_attributes.md new file mode 100644 index 00000000..079eb365 --- /dev/null +++ b/doc/threads/01_the_route_and_its_attributes.md @@ -0,0 +1,159 @@ +# BIRD Journey to Threads. Chapter 1: The Route and its Attributes + +BIRD is a fast, robust and memory-efficient routing daemon designed and +implemented at the end of 20th century. We're doing a significant amount of +BIRD's internal structure changes to make it possible to run in multiple +threads in parallel. This chapter covers necessary changes of data structures +which store every single routing data. + +*If you want to see the changes in code, look (basically) into the +`route-storage-updates` branch. Not all of them are already implemented, anyway +most of them are pretty finished as of end of March, 2021.* + +## How routes are stored + +BIRD routing table is just a hierarchical noSQL database. On top level, the +routes are keyed by their destination, called *net*. Due to historic reasons, +the *net* is not only *IPv4 prefix*, *IPv6 prefix*, *IPv4 VPN prefix* etc., +but also *MPLS label*, *ROA information* or *BGP Flowspec record*. As there may +be several routes for each *net*, an obligatory part of the key is *src* aka. +*route source*. The route source is a tuple of the originating protocol +instance and a 32-bit unsigned integer. If a protocol wants to withdraw a route, +it is enough and necessary to have the *net* and *src* to identify what route +is to be withdrawn. + +The route itself consists of (basically) a list of key-value records, with +value types ranging from a 16-bit unsigned integer for preference to a complex +BGP path structure. The keys are pre-defined by protocols (e.g. BGP path or +OSPF metrics), or by BIRD core itself (preference, route gateway). +Finally, the user can declare their own attribute keys using the keyword +`attribute` in config. + +## Attribute list implementation + +Currently, there are three layers of route attributes. We call them *route* +(*rte*), *attributes* (*rta*) and *extended attributes* (*ea*, *eattr*). + +The first layer, *rte*, contains the *net* pointer, several fixed-size route +attributes (mostly preference and protocol-specific metrics), flags, lastmod +time and a pointer to *rta*. + +The second layer, *rta*, contains the *src* (a pointer to a singleton instance), +a route gateway, several other fixed-size route attributes and a pointer to +*ea* list. + +The third layer, *ea* list, is a variable-length list of key-value attributes, +containing all the remaining route attributes. + +Distribution of the route attributes between the attribute layers is somehow +arbitrary. Mostly, in the first and second layer, there are attributes that +were thought to be accessed frequently (e.g. in best route selection) and +filled in in most routes, while the third layer is for infrequently used +and/or infrequently accessed route attributes. + +## Attribute list deduplication + +When protocols originate routes, there are commonly more routes with the +same attribute list. BIRD could ignore this fact, anyway if you have several +tables connected with pipes, it is more memory-efficient to store the same +attribute lists only once. + +Therefore, the two lower layers (*rta* and *ea*) are hashed and stored in a +BIRD-global database. Routes (*rte*) contain a pointer to *rta* in this +database, maintaining a use-count of each *rta*. Attributes (*rta*) contain +a pointer to normalized (sorted by numerical key ID) *ea*. + +## Attribute list rework + +The first thing to change is the distribution of route attributes between +attribute list layers. We decided to make the first layer (*rte*) only the key +and other per-record internal technical information. Therefore we move *src* to +*rte* and preference to *rta* (beside other things). *This is already done.* + +We also found out that the nexthop (gateway), originally one single IP address +and an interface, has evolved to a complex attribute with several sub-attributes; +not only considering multipath routing but also MPLS stacks and other per-route +attributes. This has led to a too complex data structure holding the nexthop set. + +We decided finally to squash *rta* and *ea* to one type of data structure, +allowing for completely dynamic route attribute lists. This is also supported +by adding other *net* types (BGP FlowSpec or ROA) where lots of the fields make +no sense at all, yet we still want to use the same data structures and implementation +as we don't like duplicating code. *Multithreading doesn't depend on this change, +anyway this change is going to happen soon anyway.* + +## Route storage + +The process of route import from protocol into a table can be divided into several phases: + +1. (In protocol code.) Create the route itself (typically from + protocol-internal data) and choose the right channel to use. +2. (In protocol code.) Create the *rta* and *ea* and obtain an appropriate + hashed pointer. Allocate the *rte* structure and fill it in. +3. (Optionally.) Store the route to the *import table*. +4. Run filters. If reject, free everything. +5. Check whether this is a real change (it may be idempotent). If not, free everything and do nothing more. +6. Run the best route selection algorithm. +7. Execute exports if needed. + +We found out that the *rte* structure allocation is done too early. BIRD uses +global optimized allocators for fixed-size blocks (which *rte* is) to reduce +its memory footprint, therefore the allocation of *rte* structure would be a +synchronization point in multithreaded environment. + +The common code is also much more complicated when we have to track whether the +current *rte* has to be freed or not. This is more a problem in export than in +import as the export filter can also change the route (and therefore allocate +another *rte*). The changed route must be therefore freed after use. All the +route changing code must also track whether this route is writable or +read-only. + +We therefore introduce a variant of *rte* called *rte_storage*. Both of these +hold the same, the layer-1 route information (destination, author, cached +attribute pointer, flags etc.), anyway *rte* is always local and *rte_storage* +is intended to be put in global data structures. + +This change allows us to remove lots of the code which only tracks whether any +*rte* is to be freed as *rte*'s are almost always allocated on-stack, naturally +limiting their lifetime. If not on-stack, it's the responsibility of the owner +to free the *rte* after import is done. + +This change also removes the need for *rte* allocation in protocol code and +also *rta* can be safely allocated on-stack. As a result, protocols can simply +allocate all the data on stack, call the update routine and the common code in +BIRD's *nest* does all the storage for them. + +Allocating *rta* on-stack is however not required. BGP and OSPF use this to +import several routes with the same attribute list. In BGP, this is due to the +format of BGP update messages containing first the attributes and then the +destinations (BGP NLRI's). In OSPF, in addition to *rta* deduplication, it is +also presumed that no import filter (or at most some trivial changes) is applied +as OSPF would typically not work well when filtered. + +*This change is already done.* + +## Route cleanup and table maintenance + +In some cases, the route update is not originated by a protocol/channel code. +When the channel shuts down, all routes originated by that channel are simply +cleaned up. Also routes with recursive routes may get changed without import, +simply by changing the IGP route. + +This is currently done by a `rt_event` (see `nest/rt-table.c` for source code) +which is to be converted to a parallel thread, running when nobody imports any +route. *This change is freshly done in branch `guernsey`.* + +## Parallel protocol execution + +The long-term goal of these reworks is to allow for completely independent +execution of all the protocols. Typically, there is no direct interaction +between protocols; everything is done thought BIRD's *nest*. Protocols should +therefore run in parallel in future and wait/lock only when something is needed +to do externally. + +We also aim for a clean and documented protocol API. + +*It's still a long road to the version 2.1. This series of texts should document +what is needed to be changed, why we do it and how. In the next chapter, we're +going to describe how the route is exported from table to protocols and how this +process is changing. Stay tuned!* diff --git a/doc/threads/02_asynchronous_export.md b/doc/threads/02_asynchronous_export.md new file mode 100644 index 00000000..62e4ea9f --- /dev/null +++ b/doc/threads/02_asynchronous_export.md @@ -0,0 +1,463 @@ +# BIRD Journey to Threads. Chapter 2: Asynchronous route export + +Route export is a core algorithm of BIRD. This chapter covers how we are making +this procedure multithreaded. Desired outcomes are mostly lower latency of +route import, flap dampening and also faster route processing in large +configurations with lots of export from one table. + +BIRD is a fast, robust and memory-efficient routing daemon designed and +implemented at the end of 20th century. We're doing a significant amount of +BIRD's internal structure changes to make it possible to run in multiple +threads in parallel. + +## How routes are propagated through BIRD + +In the [previous chapter](https://en.blog.nic.cz/2021/03/23/bird-journey-to-threads-chapter-1-the-route-and-its-attributes/), you could learn how the route import works. We should +now extend that process by the route export. + +1. (In protocol code.) Create the route itself and propagate it through the + right channel by calling `rte_update`. +2. The channel runs its import filter. +3. New best route is selected. +4. For each channel: + 1. The channel runs its preexport hook and export filter. + 2. (Optionally.) The channel merges the nexthops to create an ECMP route. + 3. The channel calls the protocol's `rt_notify` hook. +5. After all exports are finished, the `rte_update` call finally returns and + the source protocol may do anything else. + +Let's imagine that all the protocols are running in parallel. There are two +protocols with a route prepared to import. One of those wins the table lock, +does the import and then the export touches the other protocol which must +either: + +* store the route export until it finishes its own imports, or +* have independent import and export parts. + +Both of these conditions are infeasible for common use. Implementing them would +make protocols much more complicated with lots of new code to test and release +at once and also quite a lot of corner cases. Risk of deadlocks is also worth +mentioning. + +## Asynchronous route export + +We decided to make it easier for protocols and decouple the import and export +this way: + +1. The import is done. +2. Best route is selected. +3. Resulting changes are stored. + +Then, after the importing protocol returns, the exports are processed for each +exporting channel in parallel: Some protocols +may process the export directly after it is stored, other protocols wait +until they finish another job. + +This eliminates the risk of deadlocks and all protocols' `rt_notify` hooks can +rely on their independence. There is only one question. How to store the changes? + +## Route export modes + +To find a good data structure for route export storage, we shall first know the +readers. The exporters may request different modes of route export. + +### Export everything + +This is the most simple route export mode. The exporter wants to know about all +the routes as they're changing. We therefore simply store the old route until +the change is fully exported and then we free the old stored route. + +To manage this, we can simply queue the changes one after another and postpone +old route cleanup after all channels have exported the change. The queue member +would look like this: + +``` +struct { + struct rte_storage *new; + struct rte_storage *old; +}; +``` + +### Export best + +This is another simple route export mode. We check whether the best route has +changed; if not, no export happens. Otherwise, the export is propagated as the +old best route changing to the new best route. + +To manage this, we could use the queue from the previous point by adding new +best and old best pointers. It is guaranteed that both the old best and new +best pointers are always valid in time of export as all the changes in them +must be stored in future changes which have not been exported yet by this +channel and therefore not freed yet. + +``` +struct { + struct rte_storage *new; + struct rte_storage *new_best; + struct rte_storage *old; + struct rte_storage *old_best; +}; +``` + +Anyway, we're getting to the complicated export modes where this simple +structure is simply not enough. + +### Export merged + +Here we're getting to some kind of problems. The exporting channel requests not +only the best route but also all routes that are good enough to be considered +ECMP-eligible (we call these routes *mergable*). The export is then just one +route with just the nexthops merged. Export filters are executed before +merging and if the best route is rejected, nothing is exported at all. + +To achieve this, we have to re-evaluate export filters any time the best route +or any mergable route changes. Until now, the export could just do what it wanted +as there was only one thread working. To change this, we need to access the +whole route list and process it. + +### Export first accepted + +In this mode, the channel runs export filters on a sorted list of routes, best first. +If the best route gets rejected, it asks for the next one until it finds an +acceptable route or exhausts the list. This export mode requires a sorted table. +BIRD users may know this export mode as `secondary` in BGP. + +For now, BIRD stores two bits per route for each channel. The *export bit* is set +if the route has been really exported to that channel. The *reject bit* is set +if the route was rejected by the export filter. + +When processing a route change for accepted, the algorithm first checks the +export bit for the old route. If this bit is set, the old route is that one +exported so we have to find the right one to export. Therefore the sorted route +list is walked best to worst to find a new route to export, using the reject +bit to evaluate only routes which weren't rejected in previous runs of this +algorithm. + +If the old route bit is not set, the algorithm walks the sorted route list best +to worst, checking the position of new route with respect to the exported route. +If the new route is worse, nothing happens, otherwise the new route is sent to +filters and finally exported if passes. + +### Export by feed + +To resolve problems arising from previous two export modes (merged and first accepted), +we introduce a way to process a whole route list without locking the table +while export filters are running. To achieve this, we follow this algorithm: + +1. The exporting channel sees a pending export. +2. *The table is locked.* +3. All routes (pointers) for the given destination are dumped to a local array. +4. Also first and last pending exports for the given destination are stored. +5. *The table is unlocked.* +6. The channel processes the local array of route pointers. +7. All pending exports between the first and last stored (incl.) are marked as processed to allow for cleanup. + +After unlocking the table, the pointed-to routes are implicitly guarded by the +sole fact that no pending export has not yet been processed by all channels +and the cleanup routine frees only resources after being processed. + +The pending export range must be stored together with the feed. While +processing export filters for the feed, another export may come in. We +must process the export once again as the feed is now outdated, therefore we +must mark only these exports that were pending for this destination when the +feed was being stored. We also can't mark them before actually processing them +as they would get freed inbetween. + +## Pending export data structure + +As the two complicated export modes use the export-by-feed algorithm, the +pending export data structure may be quite minimalistic. + +``` +struct rt_pending_export { + struct rt_pending_export * _Atomic next; /* Next export for the same destination */ + struct rte_storage *new; /* New route */ + struct rte_storage *new_best; /* New best route in unsorted table */ + struct rte_storage *old; /* Old route */ + struct rte_storage *old_best; /* Old best route in unsorted table */ + _Atomic u64 seq; /* Sequential ID (table-local) of the pending export */ +}; +``` + +To allow for squashing outdated pending exports (e.g. for flap dampening +purposes), there is a `next` pointer to the next export for the same +destination. This is also needed for the export-by-feed algorithm to traverse +the list of pending exports. + +We should also add several items into `struct channel`. + +``` + struct coroutine *export_coro; /* Exporter and feeder coroutine */ + struct bsem *export_sem; /* Exporter and feeder semaphore */ + struct rt_pending_export * _Atomic last_export; /* Last export processed */ + struct bmap export_seen_map; /* Keeps track which exports were already processed */ + u64 flush_seq; /* Table export seq when the channel announced flushing */ +``` + +To run the exports in parallel, `export_coro` is run and `export_sem` is +used for signalling new exports to it. The exporter coroutine also marks all +seen sequential IDs in its `export_seen_map` to make it possible to skip over +them if seen again. The exporter coroutine is started when export is requested +and stopped when export is stopped. + +There is also a table cleaner routine +(see [previous chapter](https://en.blog.nic.cz/2021/03/23/bird-journey-to-threads-chapter-1-the-route-and-its-attributes/)) +which must cleanup also the pending exports after all the channels are finished with them. +To signal that, there is `last_export` working as a release point: the channel +guarantees that it doesn't touch the pointed-to pending export (or any older), nor any data +from it. + +The last tricky point here is channel flushing. When any channel stops, all its +routes are automatically freed and withdrawals are exported if appropriate. +Until now, the routes could be flushed synchronously, anyway now flush has +several phases, stored in `flush_active` channel variable: + +1. Flush started. +2. Withdrawals for all the channel's routes are issued. + Here the channel stores the `seq` of last current pending export to `flush_seq`) +3. When the table's cleanup routine cleans up the withdrawal with `flush_seq`, + the channel may safely stop and free its structures as all `sender` pointers in routes are now gone. + +Finally, some additional information has to be stored in tables: + +``` + _Atomic byte export_used; /* Export journal cleanup scheduled */ \ + struct rt_pending_export * _Atomic first_export; /* First export to announce */ \ + byte export_scheduled; /* Export is scheduled */ + list pending_exports; /* List of packed struct rt_pending_export */ + struct fib export_fib; /* Auxiliary fib for storing pending exports */ + u64 next_export_seq; /* The next export will have this ID */ +``` + +The exports are: +1. Assigned the `next_export_seq` sequential ID, incrementing this item by one. +2. Put into `pending_exports` and `export_fib` for both sequential and by-destination access. +3. Signalled by setting `export_scheduled` and `first_export`. + +After processing several exports, `export_used` is set and route table maintenance +coroutine is woken up to possibly do cleanup. + +The `struct rt_pending_export` seems to be best allocated by requesting a whole +memory page, containing a common list node, a simple header and packed all the +structures in the rest of the page. This may save a significant amount of memory. +In case of congestion, there will be lots of exports and every spare kilobyte +counts. If BIRD is almost idle, the optimization does nothing on the overall performance. + +## Export algorithm + +As we have explained at the beginning, the current export algorithm is +synchronous and table-driven. The table walks the channel list and propagates the update. +The new export algorithm is channel-driven. The table just indicates that it +has something new in export queue and the channel decides what to do with that and when. + +### Pushing an export + +When a table has something to export, it enqueues an instance of +`struct rt_pending_export` together with updating the `last` pointer (and +possibly also `first`) for this destination's pending exports. + +Then it pings its maintenance coroutine (`rt_event`) to notify the exporting +channels about a new route. Before the maintenance coroutine acquires the table +lock, the importing protocol may e.g. prepare the next route inbetween. +The maintenance coroutine, when it wakes up, walks the list of channels and +wakes their export coroutines. + +These two levels of asynchronicity are here for an efficiency reason. + +1. In case of low table load, the export is announced just after the import happens. +2. In case of table congestion, the export notification locks the table as well + as all route importers, effectively reducing the number of channel list traversals. + +### Processing an export + +After these two pings, the channel finally knows that there is an export pending. + +1. The channel waits for a semaphore. This semaphore is posted by the table + maintenance coroutine. +2. The channel checks whether there is a `last_export` stored. + 1. If yes, it proceeds with the next one. + 2. Otherwise it takes `first_export` from the table. This special + pointer is atomic and can be accessed without locking and also without clashing + with the export cleanup routine. +3. The channel checks its `export_seen_map` whether this export has been + already processed. If so, it goes back to 1. to get the next export. No + action is needed with this one. +4. As now the export is clearly new, the export chain (single-linked list) is + scanned for the current first and last export. This is done by following the + `next` pointer in the exports. +5. If all-routes mode is used, the exports are processed one-by-one. In future + versions, we may employ some simple flap-dampening by checking the pending + export list for the same route src. *No table locking happens.* +6. If best-only mode is employed, just the first and last exports are + considered to find the old and new best routes. The inbetween exports do nothing. *No table locking happens.* +7. If export-by-feed is used, the current state of routes in table are fetched and processed + as described above in the "Export by feed" section. +8. All processed exports are marked as seen. +9. The channel stores the first processed export to `last_export` and returns + to beginning.to wait for next exports. The latter exports are then skipped by + step 3 when the export coroutine gets to them. + +## The full life-cycle of routes + +Until now, we're always assuming that the channels *just exist*. In real life, +any channel may go up or down and we must handle it, flushing the routes +appropriately and freeing all the memory just in time to avoid both +use-after-free and memory leaks. BIRD is written in C which has no garbage +collector or other modern features alike so memory management is a thing. + +### Protocols and channels as viewed from a route + +BIRD consists effectively of protocols and tables. **Protocols** are active parts, +kind-of subprocesses manipulating routes and other data. **Tables** are passive, +serving as a database of routes. To connect a protocol to a table, a +**channel** is created. + +Every route has its `sender` storing the channel which has put the route into +the current table. Therefore we know which routes to flush when a channel goes down. + +Every route also has its `src`, a route source allocated by the protocol which +originated it first. This is kept when a route is passed through a *pipe*. The +route source is always bound to protocol; it is possible that a protocol +announces routes via several channels using the same src. + +Both `src` and `sender` must point to active protocols and channels as inactive +protocols and channels may be deleted any time. + +### Protocol and channel lifecycle + +In the beginning, all channels and protocols are down. Until they fully start, +no route from them is allowed to any table. When the protocol and channel is up, +they may originate and receive routes freely. However, the transitions are worth mentioning. + +### Channel startup and feed + +When protocols and channels start, they need to get the current state of the +appropriate table. Therefore, after a protocol and channel start, also the +export-feed coroutine is initiated. + +Tables can contain millions of routes. It may lead to long import latency if a channel +was feeding itself in one step. The table structure is (at least for now) too +complicated to be implemented as lockless, thus even read access needs locking. +To mitigate this, the feeds are split to allow for regular route propagation +with a reasonable latency. + +When the exports were synchronous, we simply didn't care and just announced the +exports to the channels from the time they started feeding. When making exports +asynchronous, it is crucial to avoid (hopefully) all the possible race conditions +which could arise from simultaneous feed and export. As the feeder routines had +to be rewritten, it is a good opportunity to make this precise. + +Therefore, when a channel goes up, it also starts exports: + +1. Start the feed-export coroutine. +2. *Lock the table.* +3. Store the last export in queue. +4. Read a limited number of routes to local memory together with their pending exports. +5. If there are some routes to process: + 1. *Unlock the table.* + 2. Process the loaded routes. + 3. Set the appropriate pending exports as seen. + 4. *Lock the table* + 5. Go to 4. to continue feeding. +6. If there was a last export stored, load the next one to be processed. Otherwise take the table's `first_export`. +7. *Unlock the table.* +8. Run the exporter loop. + +*Note: There are some nuances not mentioned here how to do things in right +order to avoid missing some events while changing state. For specifics, look +into the code in `nest/rt-table.c` in branch `alderney`.* + +When the feeder loop finishes, it continues smoothly to process all the exports +that have been queued while the feed was running. Step 5.3 ensures that already +seen exports are skipped, steps 3 and 6 ensure that no export is missed. + +### Channel flush + +Protocols and channels need to stop for a handful of reasons, All of these +cases follow the same routine. + +1. (Maybe.) The protocol requests to go down or restart. +2. The channel requests to go down or restart. +3. The channel requests to stop export. +4. In the feed-export coroutine: + 1. At a designated cancellation point, check cancellation. + 2. Clean up local data. + 3. *Lock main BIRD context* + 4. If shutdown requested, switch the channel to *flushing* state and request table maintenance. + 5. *Stop the coroutine and unlock main BIRD context.* +5. In the table maintenance coroutine: + 1. Walk across all channels and check them for *flushing* state, setting `flush_active` to 1. + 2. Walk across the table (split to allow for low latency updates) and + generate a withdrawal for each route sent by the flushing channels. + 3. When all the table is traversed, the flushing channels' `flush_active` is set to 2 and + `flush_seq` is set to the current last export seq. + 3. Wait until all the withdrawals are processed by checking the `flush_seq`. + 4. Mark the flushing channels as *down* and eventually proceed to the protocol shutdown or restart. + +There is also a separate routine that handles bulk cleanup of `src`'s which +contain a pointer to the originating protocol. This routine may get reworked in +future; for now it is good enough. + +### Route export cleanup + +Last but not least is the export cleanup routine. Until now, the withdrawn +routes were exported synchronously and freed directly after the import was +done. This is not possible anymore. The export is stored and the import returns +to let the importing protocol continue its work. We therefore need a routine to +cleanup the withdrawn routes and also the processed exports. + +First of all, this routine refuses to cleanup when any export is feeding or +shutting down. In future, cleanup while feeding should be possible, anyway for +now we aren't sure about possible race conditions. + +Anyway, when all the exports are in a steady state, the routine works as follows: + +1. Walk the active exports and find a minimum (oldest export) between their `last_export` values. +2. If there is nothing to clear between the actual oldest export and channels' oldest export, do nothing. +3. Find the table's new `first_export` and set it. Now there is nobody pointing to the old exports. +4. Free the withdrawn routes. +5. Free the old exports, removing them also from the first-last list of exports for the same destination. + +## Results of these changes + +This step is a first major step to move forward. Using just this version may be +still as slow as the single-threaded version, at least if your export filters are trivial. +Anyway, the main purpose of this step is not an immediate speedup. It is more +of a base for the next steps: + +* Unlocking of pipes should enable parallel execution of all the filters on + pipes, limited solely by the principle *one thread for every direction of + pipe*. +* Conversion of CLI's `show route` to the new feed-export coroutines should + enable faster table queries. Moreover, this approach will allow for + better splitting of model and view in CLI with a good opportunity to + implement more output formats, e.g. JSON. +* Unlocking of kernel route synchronization should fix latency issues induced + by long-lasting kernel queries. +* Partial unlocking of BGP packet processing should allow for parallel + execution in almost all phases of BGP route propagation. +* Partial unlocking of OSPF route recalculation should raise the useful + maximums of topology size. + +The development is now being done mostly in the branch `alderney`. If you asked +why such strange branch names like `jersey`, `guernsey` and `alderney`, here is +a kind-of reason. Yes, these branches could be named `mq-async-export`, +`mq-async-export-new`, `mq-async-export-new-new`, `mq-another-async-export` and +so on. That's so ugly, isn't it? Let's be creative. *Jersey* is an island where a +same-named knit was first produced – and knits are made of *threads*. Then, you +just look into a map and find nearby islands. + +Also why so many branches? The development process is quite messy. BIRD's code +heavily depends on single-threaded approach. This is (in this case) +exceptionally good for performance, as long as you have one thread only. On the +other hand, lots of these assumptions are not documented so in many cases one +desired change yields a chain of other unforeseen changes which must precede. +This brings lots of backtracking, branch rebasing and other Git magic. There is +always a can of worms somewhere in the code. + +*It's still a long road to the version 2.1. This series of texts should document +what is needed to be changed, why we do it and how. The +[previous chapter](https://en.blog.nic.cz/2021/03/23/bird-journey-to-threads-chapter-1-the-route-and-its-attributes/) +showed the necessary changes in route storage. In the next chapter, we're going +to describe how the coroutines are implemented and what kind of locking system +are we employing to prevent deadlocks. Stay tuned!* diff --git a/doc/threads/03_coroutines.md b/doc/threads/03_coroutines.md new file mode 100644 index 00000000..8d8e6c96 --- /dev/null +++ b/doc/threads/03_coroutines.md @@ -0,0 +1,235 @@ +# BIRD Journey to Threads. Chapter 3: Parallel execution and message passing. + +Parallel execution in BIRD uses an underlying mechanism of dedicated IO loops +and hierarchical locks. The original event scheduling module has been converted +to do message passing in multithreaded environment. These mechanisms are +crucial for understanding what happens inside BIRD and how its internal API changes. + +BIRD is a fast, robust and memory-efficient routing daemon designed and +implemented at the end of 20th century. We're doing a significant amount of +BIRD's internal structure changes to make it run in multiple threads in parallel. + +## Locking and deadlock prevention + +Most of BIRD data structures and algorithms are thread-unsafe and not even +reentrant. Checking and possibly updating all of these would take an +unreasonable amount of time, thus the multithreaded version uses standard mutexes +to lock all the parts which have not been checked and updated yet. + +The authors of original BIRD concepts wisely chose a highly modular structure +which allows to create a hierarchy for locks. The main chokepoint was between +protocols and tables and it has been removed by implementing asynchronous exports +as described in the [previous chapter](https://en.blog.nic.cz/2021/06/14/bird-journey-to-threads-chapter-2-asynchronous-route-export/). + +Locks in BIRD (called domains, as they always lock some defined part of BIRD) +are partially ordered. Every *domain* has its *type* and all threads are +strictly required to lock the domains in the order of their respective types. +The full order is defined in `lib/locking.h`. It's forbidden to lock more than +one domain of a type (these domains are uncomparable) and recursive locking is +forbidden as well. + +The locking hiearchy is (roughly; as of February 2022) like this: + +1. The BIRD Lock (for everything not yet checked and/or updated) +2. Protocols (as of February 2022, it is BFD, RPKI, Pipe and BGP) +3. Routing tables +4. Global route attribute cache +5. Message passing +6. Internals and memory management + +There are heavy checks to ensure proper locking and to help debugging any +problem when any code violates the hierarchy rules. This impedes performance +depending on how much that domain is contended and in some cases I have already +implemented lockless (or partially lockless) data structures to overcome this. + +You may ask, why are these heavy checks then employed in production builds? +Risks arising from dropping some locking checks include: + +* deadlocks; these are deadly in BIRD anyway so it should just fail with a meaningful message, or +* data corruption; it either kills BIRD anyway, or it results into a slow and vicious death, + leaving undebuggable corefiles behind. + +To be honest, I believe in principles like *"every nontrivial software has at least one bug"* +and I also don't trust my future self or anybody else to always write bugless code when +it comes to proper locking. I also believe that if a lock becomes a bottle-neck, +then we should think about what is locked inside and how to optimize that, +possibly implementing a lockless or waitless data structure instead of dropping +thorough consistency checks, especially in a multithreaded environment. + +### Choosing the right locking order + +When considering the locking order of protocols and route tables, the answer +was quite easy. We had to make either import or export asynchronous (or both). +Major reasons for asynchronous export have been stated in the previous chapter, +therefore it makes little sense to allow entering protocol context from table code. + +As I write further in this text, even accessing table context from protocol +code leads to contention on table locks, yet for now, it is good enough and the +lock order features routing tables after protocols to make the multithreading +goal easier to achieve. + +The major lock level is still The BIRD Lock, containing not only the +not-yet-converted protocols (like Babel, OSPF or RIP) but also processing CLI +commands and reconfiguration. This involves an awful lot of direct access into +other contexts which would be unnecessarily complicated to implement by message +passing. Therefore, this lock is simply *"the director"*, sitting on the top +with its own category. + +The lower lock levels under routing tables are mostly for shared global data +structures accessed from everywhere. We'll address some of these later. + +## IO Loop + +There has been a protocol, BFD, running in its own thread since 2013. This +separation has a good reason; it needs low latency and the main BIRD loop just +walks round-robin around all the available sockets and one round-trip may take +a long time (even more than a minute with large configurations). BFD had its +own IO loop implementation and simple message passing routines. This code could +be easily updated for general use so I did it. + +To understand the internal principles, we should say that in the `master` +branch, there is a big loop centered around a `poll()` call, dispatching and +executing everything as needed. In the `sark` branch, there are multiple loops +of this kind. BIRD has several means how to get something dispatched from a +loop. + +1. Requesting to read from a **socket** makes the main loop call your hook when there is some data received. + The same happens when a socket refuses to write data. Then the data is buffered and you are called when + the buffer is free to continue writing. There is also a third callback, an error hook, for obvious reasons. + +2. Requesting to be called back after a given amount of time. This is called **timer**. + As is common with all timers, they aren't precise and the callback may be + delayed significantly. This was also the reason to have BFD loop separate + since the very beginning, yet now the abundance of threads may lead to + problems with BFD latency in large-scale configurations. We haven't tested + this yet. + +3. Requesting to be called back from a clean context when possible. This is + useful to run anything not reentrant which might mess with the caller's + data, e.g. when a protocol decides to shutdown due to some inconsistency + in received data. This is called **event**. + +4. Requesting to do some work when possible. These are also events, there is only + a difference where this event is enqueued; in the main loop, there is a + special *work queue* with an execution limit, allowing sockets and timers to be + handled with a reasonable latency while still doing all the work needed. + Other loops don't have designated work queues (we may add them later). + +All these, sockets, timers and events, are tightly bound to some domain. +Sockets typically belong to a protocol, timers and events to a protocol or table. +With the modular structure of BIRD, the easy and convenient approach to multithreading +is to get more IO loops, each bound to a specific domain, running their events, timers and +socket hooks in their threads. + +## Message passing and loop entering + +To request some work in another module, the standard way is to pass a message. +For this purpose, events have been modified to be sent to a given loop without +locking that loop's domain. In fact, every event queue has its own lock with a +low priority, allowing to pass messages from almost any part of BIRD, and also +an assigned loop which executes the events enqueued. When a message is passed +to a queue executed by another loop, that target loop must be woken up so we +must know what loop to wake up to avoid unnecessary delays. Then the target +loop opens its mailbox and processes the task in its context. + +The other way is a direct access of another domain. This approach blocks the +appropriate loop from doing anything and we call it *entering a birdloop* to +remember that the task must be fast and *leave the birdloop* as soon as possible. +Route import is done via direct access from protocols to tables; in large +setups with fast filters, this is a major point of contention (after filters +have been parallelized) and will be addressed in future optimization efforts. +Reconfiguration and interface updates also use direct access; more on that later. +In general, this approach should be avoided unless there are good reasons to use it. + +Even though direct access is bad, sending lots of messages may be even worse. +Imagine one thousand post(wo)men, coming one by one every minute, ringing your +doorbell and delivering one letter each to you. Horrible! Asynchronous message +passing works exactly this way. After queuing the message, the source sends a +byte to a pipe to wakeup the target loop to process the task. We could also +periodically poll for messages instead of waking up the targets, yet it would +add quite a lot of latency which we also don't like. + +Messages in BIRD don't typically suffer from the problem of amount and the +overhead is negligible compared to the overall CPU consumption. With one notable +exception: route import/export. + +### Route export message passing + +If we had to send a ping for every route we import to every exporting channel, +we'd spend more time pinging than doing anything else. Been there, seen +those unbelievable 80%-like figures in Perf output. Never more. + +Route update is quite a complicated process. BIRD must handle large-scale +configurations with lots of importers and exporters. Therefore, a +triple-indirect delayed route announcement is employed: + +1. First, when a channel imports a route by entering a loop, it sends an event + to its own loop (no ping needed in such case). This operation is idempotent, + thus for several routes in a row, only one event is enqueued. This reduces + several route import announcements (even hundreds in case of massive BGP + withdrawals) to one single event. +2. When the channel is done importing (or at least takes a coffee break and + checks its mailbox), the scheduled event in its own loop is run, sending + another event to the table's loop, saying basically *"Hey, table, I've just + imported something."*. This event is also idempotent and further reduces + route import announcements from multiple sources to one single event. +3. The table's announcement event is then executed from its loop, enqueuing export + events for all connected channels, finally initiating route exports. As we + already know, imports are done by direct access, therefore if protocols keep + importing, export announcements are slowed down. +4. The actual data on what has been updated is stored in a table journal. This + peculiar technique is used only for informing the exporting channels that + *"there is something to do"*. + +This may seem overly complicated, yet it should work and it seems to work. In +case of low load, all these notifications just come through smoothly. In case +of high load, it's common that multiple updates come for the same destination. +Delaying the exports allows for the updates to settle down and export just the +final result, reducing CPU load and export traffic. + +## Cork + +Route propagation is involved in yet another problem which has to be addressed. +In the old versions with synchronous route propagation, all the buffering +happened after exporting routes to BGP. When a packet arrived, all the work was +done in BGP receive hook – parsing, importing into a table, running all the +filters and possibly sending to the peers. No more routes until the previous +was done. This self-regulating mechanism doesn't work any more. + +Route table import now returns immediately after inserting the route into a +table, creating a buffer there. These buffers have to be processed by other protocols' +export events. In large-scale configurations, one route import has to be +processed by hundreds, even thousands of exports. Unlimited imports are a major +cause of buffer bloating. This is even worse in configurations with pipes, +as these multiply the exports by propagating them all the way down to other +tables, eventually eating about twice the amount of memory than the single-threaded version. + +There is therefore a cork to make this stop. Every table is checking how many +exports it has pending, and when adding a new export to the queue, it may request +a cork, saying simply "please stop the flow for a while". When the export buffer +size is reduced low enough, the table uncorks. + +On the other side, there are events and sockets with a cork assigned. When +trying to enqueue an event and the cork is applied, the event is instead put +into the cork's queue and released only when the cork is released. In case of +sockets, when read is indicated or when `poll` arguments are recalculated, +the corked socket is simply not checked for received packets, effectively +keeping them in the TCP queue and slowing down the flow until cork is released. + +The cork implementation is quite crude and rough and fragile. It may get some +rework while stabilizing the multi-threaded version of BIRD or we may even +completely drop it for some better mechanism. One of these candidates is this +kind of API: + +* (table to protocol) please do not import +* (table to protocol) you may resume imports +* (protocol to table) not processing any exports +* (protocol to table) resuming export processing + +Anyway, cork works as intended in most cases at least for now. + +*It's a long road to the version 2.1. This series of texts should document what +is changing, why we do it and how. The +[previous chapter](https://en.blog.nic.cz/2021/06/14/bird-journey-to-threads-chapter-2-asynchronous-route-export/) +shows how the route export had to change to allow parallel execution. In the next chapter, some memory management +details are to be explained together with the reasons why memory management matters. Stay tuned!* diff --git a/doc/threads/04_memory_management.md b/doc/threads/04_memory_management.md new file mode 100644 index 00000000..24ef89d3 --- /dev/null +++ b/doc/threads/04_memory_management.md @@ -0,0 +1,223 @@ +# BIRD Journey to Threads. Chapter 4: Memory and other resource management. + +BIRD is mostly a large specialized database engine, storing mega/gigabytes of +Internet routing data in memory. To keep accounts of every byte of allocated data, +BIRD has its own resource management system which must be adapted to the +multithreaded environment. The resource system has not changed much, yet it +deserves a short chapter. + +BIRD is a fast, robust and memory-efficient routing daemon designed and +implemented at the end of 20th century. We're doing a significant amount of +BIRD's internal structure changes to make it run in multiple threads in parallel. + +## Resources + +Inside BIRD, (almost) every piece of allocated memory is a resource. To achieve this, +every such memory block includes a generic `struct resource` header. The node +is enlisted inside a linked list of a *resource pool* (see below), the class +pointer defines basic operations done on resources. + +``` +typedef struct resource { + node n; /* Inside resource pool */ + struct resclass *class; /* Resource class */ +} resource; + +struct resclass { + char *name; /* Resource class name */ + unsigned size; /* Standard size of single resource */ + void (*free)(resource *); /* Freeing function */ + void (*dump)(resource *); /* Dump to debug output */ + resource *(*lookup)(resource *, unsigned long); /* Look up address (only for debugging) */ + struct resmem (*memsize)(resource *); /* Return size of memory used by the resource, may be NULL */ +}; + +void *ralloc(pool *, struct resclass *); +``` + +Resource cycle begins with an allocation of a resource. To do that, you should call `ralloc()`, +passing the parent pool and the appropriate resource class as arguments. BIRD +allocates a memory block of size given by the given class member `size`. +Beginning of the block is reserved for `struct resource` itself and initialized +by the given arguments. Therefore, you may sometimes see an idiom where a structure +has a first member `struct resource r;`, indicating that this item should be +allocated as a resource. + +The counterpart is resource freeing. This may be implicit (by resource pool +freeing) or explicit (by `rfree()`). In both cases, the `free()` function of +the appropriate class is called to cleanup the resource before final freeing. + +To account for `dump` and `memsize` calls, there are CLI commands `dump +resources` and `show memory`, using these to dump resources or show memory +usage as perceived by BIRD. + +The last, `lookup`, is quite an obsolete way to identify a specific pointer +from a debug interface. You may call `rlookup(pointer)` and BIRD should dump +that resource to the debug output. This mechanism is probably incomplete as no +developer uses it actively for debugging. + +Resources can be also moved between pools by `rmove` when needed. + +## Resource pools + +The first internal resource class is a recursive resource – a resource pool. In +the singlethreaded version, this is just a simple structure: + +``` +struct pool { + resource r; + list inside; + struct birdloop *loop; /* In multithreaded version only */ + const char *name; +}; +``` + +Resource pools are used for grouping resources together. There are pools everywhere +and it is a common idiom inside BIRD to just `rfree` the appropriate pool when +e.g. a protocol or table is going down. Everything left there is cleaned up. + +There are anyway several classes which must be freed with care. In the +singlethreaded version, the *slab* allocator (see below) must be empty before +it may be freed and this is kept to the multithreaded version while other +restrictions have been added. + +There is also a global pool, `root_pool`, containing every single resource BIRD +knows about, either directly or via another resource pool. + +### Thread safety in resource pools + +In the multithreaded version, every resource pool is bound to a specific IO +loop and therefore includes an IO loop pointer. This is important for allocations +as the resource list inside the pool is thread-unsafe. All pool operations +therefore require the IO loop to be entered to do anything with them, if possible. +(In case of `rfree`, the pool data structure is not accessed at all so no +assert is possible. We're currently relying on the caller to ensure proper locking. +In future, this may change.) + +Each IO loop also has its base resource pool for its allocations. All pools +inside the IO loop pool must belong to the same loop or to a loop with a +subordinate lock (see the previous chapter for lock ordering). If there is a +need for multiple IO loops to access one shared data structure, it must be +locked by another lock and allocated in such a way that is independent on these +accessor loops. + +The pool structure should follow the locking order. Any pool should belong to +either the same loop as its parent or its loop lock should be after its parent +loop lock in the locking order. This is not enforced explicitly, yet it is +virtually impossible to write some working code violating this recommendation. + +### Resource pools in the wilderness + +Root pool contains (among others): + +* route attributes and sources +* routing tables +* protocols +* interfaces +* configuration data + +Each table has its IO loop and uses the loop base pool for allocations. +The same holds for protocols. Each protocol has its pool; it is either its IO +loop base pool or an ordinary pool bound to main loop. + +## Memory allocators + +BIRD stores data in memory blocks allocated by several allocators. There are 3 +of them: simple memory blocks, linear pools and slabs. + +### Simple memory block + +When just a chunk of memory is needed, `mb_alloc()` or `mb_allocz()` is used +to get it. The first with `malloc()` semantics, the other is also zeroed. +There is also `mb_realloc()` available, `mb_free()` to explicitly free such a +memory and `mb_move()` to move that memory to another pool. + +Simple memory blocks consume a fixed amount of overhead memory (32 bytes on +systems with 64-bit pointers) so they are suitable mostly for big chunks, +taking advantage of the default *stdlib* allocator which is used by this +allocation strategy. There are anyway some parts of BIRD (in all versions) +where this allocator is used for little blocks. This will be fixed some day. + +### Linear pools + +Sometimes, memory is allocated temporarily. When the data may just sit on +stack, we put it there. Anyway, many tasks need more structured execution where +stack allocation is incovenient or even impossible (e.g. when callbacks from +parsers are involved). For such a case, a *linpool* is the best choice. + +This data structure allocates memory blocks of requested size with negligible +overhead in functions `lp_alloc()` (uninitialized) or `lp_allocz()` (zeroed). +There is anyway no `realloc` and no `free` call; to have a larger chunk, you +need to allocate another block. All this memory is freed at once by `lp_flush()` +when it is no longer needed. + +You may see linpools in parsers (BGP, Linux netlink, config) or in filters. + +In the multithreaded version, linpools have received an update, allocating +memory pages directly by `mmap()` instead of calling `malloc()`. More on memory +pages below. + +### Slabs + +To allocate lots of same-sized objects, a [slab allocator](https://en.wikipedia.org/wiki/Slab_allocation) +is an ideal choice. In versions until 2.0.8, our slab allocator used blocks +allocated by `malloc()`, every object included a *slab head* pointer and free objects +were linked into a single-linked list. This led to memory inefficiency and to +contra-intuitive behavior where a use-after-free bug could do lots of damage +before finally crashing. + +Versions from 2.0.9, and also all the multithreaded versions, are coming with +slabs using directly allocated memory pages and usage bitmaps instead of +single-linking the free objects. This approach however relies on the fact that +pointers returned by `mmap()` are always divisible by page size. Freeing of a +slab object involves zeroing (mostly) 13 least significant bits of its pointer +to get the page pointer where the slab head resides. + +This update helps with memory consumption by about 5% compared to previous +versions; exact numbers depend on the usage pattern. + +## Raw memory pages + +Until 2.0.8 (incl.), BIRD allocated all memory by `malloc()`. This method is +suitable for lots of use cases, yet when gigabytes of memory should be +allocated by little pieces, BIRD uses its internal allocators to keep track +about everything. This brings some ineffectivity as stdlib allocator has its +own overhead and doesn't allocate aligned memory unless asked for. + +Slabs and linear pools are backed by blocks of memory of kilobyte sizes. As a +typical memory page size is 4 kB, it is a logical step to drop stdlib +allocation from these allocators and to use `mmap()` directly. This however has +some drawbacks, most notably the need of a syscall for every memory mapping and +unmapping. For allocations, this is not much a case and the syscall time is typically +negligible compared to computation time. When freeing memory, this is much +worse as BIRD sometimes frees gigabytes of data in a blink of eye. + +To minimize the needed number of syscalls, there is a per-thread page cache, +keeping pages for future use: + +* When a new page is requested, first the page cache is tried. +* When a page is freed, the per-thread page cache keeps it without telling the kernel. +* When the number of pages in any per-thread page cache leaves a pre-defined range, + a cleanup routine is scheduled to free excessive pages or request more in advance. + +This method gives the multithreaded BIRD not only faster memory management than +ever before but also almost immediate shutdown times as the cleanup routine is +not scheduled on shutdown at all. + +## Other resources + +Some objects are not only a piece of memory; notable items are sockets, owning +the underlying mechanism of I/O, and *object locks*, owning *the right to use a +specific I/O*. This ensures that collisions on e.g. TCP port numbers and +addresses are resolved in a predictable way. + +All these resources should be used with the same locking principles as the +memory blocks. There aren't many checks inside BIRD code to ensure that yet, +nevertheless violating this recommendation may lead to multiple-access issues. + +*It's still a long road to the version 2.1. This series of texts should document +what is needed to be changed, why we do it and how. The +[previous chapter](TODO) +showed the locking system and how the parallel execution is done. +The next chapter will cover a bit more detailed explanation about route sources +and route attributes and how lockless data structures are employed there. Stay tuned!* diff --git a/doc/threads/Makefile b/doc/threads/Makefile new file mode 100644 index 00000000..fafb8d47 --- /dev/null +++ b/doc/threads/Makefile @@ -0,0 +1,29 @@ +SUFFICES := .pdf -wordpress.html +CHAPTERS := 00_the_name_of_the_game 01_the_route_and_its_attributes 02_asynchronous_export 03_coroutines + +all: $(foreach ch,$(CHAPTERS),$(addprefix $(ch),$(SUFFICES))) + +00_the_name_of_the_game.pdf: 00_filter_structure.png + +%.pdf: %.md + pandoc -f markdown -t latex -o $@ $< + +%.html: %.md + pandoc -f markdown -t html5 -o $@ $< + +%-wordpress.html: %.html Makefile + sed -r 's#</p>#\n#g; s#<p>##g; s#<(/?)code>#<\1tt>#g; s#<pre><tt>#<code>#g; s#</tt></pre>#</code>#g; s#</?figure>##g; s#<figcaption>#<p style="text-align: center">#; s#</figcaption>#</p>#; ' $< > $@ + +stats-%.csv: stats.csv stats-filter.pl + perl stats-filter.pl $< $* > $@ + +STATS_VARIANTS := multi imex mulimex single +stats-all: $(patsubst %,stats-%.csv,$(STATS_VARIANTS)) + +stats-2d-%.pdf: stats.csv stats-filter-2d.pl + perl stats-filter-2d.pl $< $* $@ + +stats-2d-%.png: stats-2d-%.pdf + gs -dBATCH -dNOPAUSE -sDEVICE=pngalpha -sOutputFile=$@ -r300 $< + +stats-all-2d: $(foreach suf,pdf png,$(patsubst %,stats-2d-%.$(suf),$(STATS_VARIANTS))) diff --git a/doc/threads/stats-draw.gnuplot b/doc/threads/stats-draw.gnuplot new file mode 100644 index 00000000..734f14a6 --- /dev/null +++ b/doc/threads/stats-draw.gnuplot @@ -0,0 +1,41 @@ +set datafile columnheaders +set datafile separator ";" +#set isosample 15 +set dgrid3d 8,8 +set logscale +set view 80,15,1,1 +set autoscale xy +#set pm3d + +set term pdfcairo size 20cm,15cm + +set xlabel "TOTAL ROUTES" offset 0,-1.5 +set xrange [10000:320000] +set xtics offset 0,-0.5 +set xtics (10000,15000,30000,50000,100000,150000,300000) + +set ylabel "PEERS" +#set yrange [10:320] +#set ytics (10,15,30,50,100,150,300) +set yrange [10:320] +set ytics (10,15,30,50,100,150,300) + +set zrange [1:2000] +set xyplane at 1 + +set border 895 + +#set grid ztics lt 20 + +set output ARG1 . "-" . ARG4 . ".pdf" + +splot \ + ARG1 . ".csv" \ + using "TOTAL_ROUTES":"PEERS":ARG2."/".ARG4 \ + with lines \ + title ARG2."/".ARG4, \ + "" \ + using "TOTAL_ROUTES":"PEERS":ARG3."/".ARG4 \ + with lines \ + title ARG3."/".ARG4 + diff --git a/doc/threads/stats-filter-2d.pl b/doc/threads/stats-filter-2d.pl new file mode 100644 index 00000000..59eda26b --- /dev/null +++ b/doc/threads/stats-filter-2d.pl @@ -0,0 +1,137 @@ +#!/usr/bin/perl + +use common::sense; +use Data::Dump; +use List::Util; + +my @GROUP_BY = qw/VERSION PEERS TOTAL_ROUTES/; +my @VALUES = qw/TIMEDIF/; + +my ($FILE, $TYPE, $OUTPUT) = @ARGV; + +### Load data ### +my %data; +open F, "<", $FILE or die $!; +my @header = split /;/, <F>; +chomp @header; + +my $line = undef; +while ($line = <F>) +{ + chomp $line; + $line =~ s/;;(.*);;/;;\1;/; + $line =~ s/v2\.0\.8-1[89][^;]+/bgp/; + $line =~ s/v2\.0\.8-[^;]+/sark/; + $line =~ s/master;/v2.0.8;/; + my %row; + @row{@header} = split /;/, $line; + push @{$data{join ";", @row{@GROUP_BY}}}, { %row } if $row{TYPE} eq $TYPE; +} + +### Do statistics ### +sub avg { + return List::Util::sum(@_) / @_; +} + +sub stdev { + my $avg = shift; + return 0 if @_ <= 1; + return sqrt(List::Util::sum(map { ($avg - $_)**2 } @_) / (@_-1)); +} + +my %output; +my %vers; +my %peers; + +STATS: +foreach my $k (keys %data) +{ + my %cols = map { my $vk = $_; $vk => [ map { $_->{$vk} } @{$data{$k}} ]; } @VALUES; + + my %avg = map { $_ => avg(@{$cols{$_}})} @VALUES; + my %stdev = map { $_ => stdev($avg{$_}, @{$cols{$_}})} @VALUES; + + foreach my $v (@VALUES) { + next if $stdev{$v} / $avg{$v} < 0.035; + + for (my $i=0; $i<@{$cols{$v}}; $i++) + { + my $dif = $cols{$v}[$i] - $avg{$v}; + next if $dif < $stdev{$v} * 2 and $dif > $stdev{$v} * (-2); +=cut + printf "Removing an outlier for %s/%s: avg=%f, stdev=%f, variance=%.1f%%, val=%f, valratio=%.1f%%\n", + $k, $v, $avg{$v}, $stdev{$v}, (100 * $stdev{$v} / $avg{$v}), $cols{$v}[$i], (100 * $dif / $stdev{$v}); +=cut + splice @{$data{$k}}, $i, 1, (); + redo STATS; + } + } + + $vers{$data{$k}[0]{VERSION}}++; + $peers{$data{$k}[0]{PEERS}}++; + $output{$data{$k}[0]{VERSION}}{$data{$k}[0]{PEERS}}{$data{$k}[0]{TOTAL_ROUTES}} = { %avg }; +} + +(3 == scalar %vers) and $vers{sark} and $vers{bgp} and $vers{"v2.0.8"} or die "vers size is " . (scalar %vers) . ", items ", join ", ", keys %vers; + +### Export the data ### + +open PLOT, "|-", "gnuplot" or die $!; + +say PLOT <<EOF; +set logscale + +set term pdfcairo size 20cm,15cm + +set xlabel "Total number of routes" offset 0,-1.5 +set xrange [10000:1500000] +set xtics offset 0,-0.5 +set xtics (10000,15000,30000,50000,100000,150000,300000,500000,1000000) + +set ylabel "Time to converge (s)" +set yrange [0.5:10800] + +set grid + +set key left top + +set output "$OUTPUT" +EOF + +my @colors = ( + [ 1, 0.3, 0.3 ], + [ 1, 0.7, 0 ], + [ 0.3, 1, 0 ], + [ 0, 1, 0.3 ], + [ 0, 0.7, 1 ], + [ 0.3, 0.3, 1 ], +); + +my $steps = (scalar %peers) - 1; + +my @plot_data; +foreach my $v (sort keys %vers) { + my $color = shift @colors; + my $endcolor = shift @colors; + my $stepcolor = [ map +( ($endcolor->[$_] - $color->[$_]) / $steps ), (0, 1, 2) ]; + + foreach my $p (sort { int $a <=> int $b } keys %peers) { + my $vnodot = $v; $vnodot =~ s/\.//g; + say PLOT "\$data_${vnodot}_${p} << EOD"; + foreach my $tr (sort { int $a <=> int $b } keys %{$output{$v}{$p}}) { + say PLOT "$tr $output{$v}{$p}{$tr}{TIMEDIF}"; + } + say PLOT "EOD"; + + my $colorstr = sprintf "linecolor rgbcolor \"#%02x%02x%02x\"", map +( int($color->[$_] * 255 + 0.5)), (0, 1, 2); + push @plot_data, "\$data_${vnodot}_${p} using 1:2 with lines $colorstr linewidth 2 title \"$v, $p peers\""; + $color = [ map +( $color->[$_] + $stepcolor->[$_] ), (0, 1, 2) ]; + } +} + +push @plot_data, "2 with lines lt 1 dashtype 2 title \"Measurement instability\""; + +say PLOT "plot ", join ", ", @plot_data; +close PLOT; + + diff --git a/doc/threads/stats-filter.pl b/doc/threads/stats-filter.pl new file mode 100644 index 00000000..51690651 --- /dev/null +++ b/doc/threads/stats-filter.pl @@ -0,0 +1,84 @@ +#!/usr/bin/perl + +use common::sense; +use Data::Dump; +use List::Util; + +my @GROUP_BY = qw/VERSION PEERS TOTAL_ROUTES/; +my @VALUES = qw/RSS SZ VSZ TIMEDIF/; + +my ($FILE, $TYPE) = @ARGV; + +### Load data ### +my %data; +open F, "<", $FILE or die $!; +my @header = split /;/, <F>; +chomp @header; + +my $line = undef; +while ($line = <F>) +{ + chomp $line; + my %row; + @row{@header} = split /;/, $line; + push @{$data{join ";", @row{@GROUP_BY}}}, { %row } if $row{TYPE} eq $TYPE; +} + +### Do statistics ### +sub avg { + return List::Util::sum(@_) / @_; +} + +sub stdev { + my $avg = shift; + return 0 if @_ <= 1; + return sqrt(List::Util::sum(map { ($avg - $_)**2 } @_) / (@_-1)); +} + +my %output; +my %vers; + +STATS: +foreach my $k (keys %data) +{ + my %cols = map { my $vk = $_; $vk => [ map { $_->{$vk} } @{$data{$k}} ]; } @VALUES; + + my %avg = map { $_ => avg(@{$cols{$_}})} @VALUES; + my %stdev = map { $_ => stdev($avg{$_}, @{$cols{$_}})} @VALUES; + + foreach my $v (@VALUES) { + next if $stdev{$v} / $avg{$v} < 0.035; + + for (my $i=0; $i<@{$cols{$v}}; $i++) + { + my $dif = $cols{$v}[$i] - $avg{$v}; + next if $dif < $stdev{$v} * 2 and $dif > $stdev{$v} * (-2); +=cut + printf "Removing an outlier for %s/%s: avg=%f, stdev=%f, variance=%.1f%%, val=%f, valratio=%.1f%%\n", + $k, $v, $avg{$v}, $stdev{$v}, (100 * $stdev{$v} / $avg{$v}), $cols{$v}[$i], (100 * $dif / $stdev{$v}); +=cut + splice @{$data{$k}}, $i, 1, (); + redo STATS; + } + } + + $vers{$data{$k}[0]{VERSION}}++; + $output{"$data{$k}[0]{PEERS};$data{$k}[0]{TOTAL_ROUTES}"}{$data{$k}[0]{VERSION}} = { %avg }; +} + +### Export the data ### + +say "PEERS;TOTAL_ROUTES;" . join ";", ( map { my $vk = $_; map { "$_/$vk" } keys %vers; } @VALUES ); + +sub keysort { + my ($pa, $ta) = split /;/, $_[0]; + my ($pb, $tb) = split /;/, $_[1]; + + return (int $ta) <=> (int $tb) if $pa eq $pb; + return (int $pa) <=> (int $pb); +} + +foreach my $k (sort { keysort($a, $b); } keys %output) +{ + say "$k;" . join ";", ( map { my $vk = $_; map { $output{$k}{$_}{$vk}; } keys %vers; } @VALUES ); +} diff --git a/doc/threads/stats-longfilters.csv b/doc/threads/stats-longfilters.csv new file mode 100644 index 00000000..3b794e53 --- /dev/null +++ b/doc/threads/stats-longfilters.csv @@ -0,0 +1,1964 @@ +VERSION;TYPE;PEERS;ROUTES;BLOCKSIZE;TOTAL_ROUTES;TOTAL_PREFICES;RSS;SZ;VSZ;i;TIMEDIF;DATETIME +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;21320;187203;748812;20;4.301861676 +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;21140;187190;748760;22;4.735876270 +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;21044;187207;748828;20;4.304013846 +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;20772;187196;748784;19;4.084681546 +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;20772;187188;748752;21;4.524855352 +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;20800;187197;748788;19;4.089552928 +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;21072;187192;748768;20;4.301069856 +v2.0.8-161-ga788372e;multi;10;10000;;12925;6245;21148;187192;748768;18;3.870419206 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19392;6769;27076;24;5.186131393 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19456;6749;26996;24;5.171237831 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19528;6754;27016;25;5.404748204 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19292;6768;27072;24;5.170694801 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19396;6760;27040;27;5.838374799 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19508;6777;27108;25;5.402606133 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19284;6752;27008;24;5.182178320 +v2.0.8-161-ga788372e;imex;10;10000;;12925;6245;19548;6763;27052;24;5.174747059 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;6968;3344;13376;26;5.608568489 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;7224;3377;13508;26;5.615165423;2021-12-06@14:05:32 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;7316;3375;13500;26;5.618026061;2021-12-06@14:06:00 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;7260;3352;13408;23;4.961829208;2021-12-06@14:06:27 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;7252;3369;13476;25;5.388673471;2021-12-06@14:06:55 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;7288;3350;13400;26;5.621786939;2021-12-06@14:07:23 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;7188;3369;13476;26;5.609636187;2021-12-06@14:07:51 +v2.0.8-161-ga788372e;single;10;10000;;12925;6245;;7316;3364;13456;24;5.172085664;2021-12-06@14:08:19 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;31332;189917;759668;17;3.659687724;2021-12-06@14:08:45 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;30948;189945;759780;22;4.730535289;2021-12-06@14:09:11 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;30540;189905;759620;20;4.306343387;2021-12-06@14:09:38 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;30444;189942;759768;20;4.302684747;2021-12-06@14:10:06 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;30808;189919;759676;18;3.875768252;2021-12-06@14:10:32 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;30272;189907;759628;19;4.086659602;2021-12-06@14:10:58 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;30672;189903;759612;20;4.302416044;2021-12-06@14:11:25 +v2.0.8-161-ga788372e;mulimex;10;10000;;12925;6245;30368;189886;759544;21;4.518095173;2021-12-06@14:11:52 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;33640;321631;1286524;19;4.085633944;2021-12-06@14:12:21 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;33060;321630;1286520;21;4.516950872;2021-12-06@14:12:53 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;32888;321622;1286488;20;4.307035392;2021-12-06@14:13:24 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;32536;321610;1286440;18;3.875851167;2021-12-06@14:13:54 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;33352;321615;1286460;22;4.739828577;2021-12-06@14:14:24 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;33432;321630;1286520;18;3.879518698;2021-12-06@14:14:56 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;33368;321637;1286548;21;4.519740200;2021-12-06@14:15:25 +v2.0.8-161-ga788372e;multi;18;10000;;15314;6305;33268;321610;1286440;20;4.305856029;2021-12-06@14:15:56 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27564;9030;36120;40;8.684061410;2021-12-06@14:16:30 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27752;9060;36240;38;8.233427802;2021-12-06@14:17:06 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27436;9051;36204;41;8.896879840;2021-12-06@14:17:40 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27672;9067;36268;40;8.668315617;2021-12-06@14:18:14 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27552;9055;36220;40;8.673992917;2021-12-06@14:18:49 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27768;9070;36280;40;8.675525863;2021-12-06@14:19:23 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27604;9063;36252;41;8.896695554;2021-12-06@14:19:58 +v2.0.8-161-ga788372e;imex;18;10000;;15314;6305;27620;9068;36272;37;8.026595858;2021-12-06@14:20:35 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8164;3609;14436;38;8.246006718;2021-12-06@14:21:08 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8240;3616;14464;39;8.451448843;2021-12-06@14:21:42 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8312;3630;14520;41;8.885674030;2021-12-06@14:22:17 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8256;3608;14432;39;8.452951862;2021-12-06@14:22:51 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8296;3612;14448;39;8.447394502;2021-12-06@14:23:26 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8224;3611;14444;39;8.461078185;2021-12-06@14:24:01 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8172;3599;14396;40;8.671040923;2021-12-06@14:24:35 +v2.0.8-161-ga788372e;single;18;10000;;15314;6305;;8272;3606;14424;41;8.892147820;2021-12-06@14:25:09 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;49120;326319;1305276;19;4.089241311;2021-12-06@14:25:40 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;49224;326251;1305004;22;4.737475091;2021-12-06@14:26:10 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;49192;326268;1305072;17;3.657795376;2021-12-06@14:26:40 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;49432;326269;1305076;21;4.520732609;2021-12-06@14:27:10 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;48564;326226;1304904;20;4.310730874;2021-12-06@14:27:40 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;49160;326273;1305092;19;4.090173008;2021-12-06@14:28:11 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;49368;326279;1305116;20;4.307600202;2021-12-06@14:28:41 +v2.0.8-161-ga788372e;mulimex;18;10000;;15314;6305;49696;326329;1305316;22;4.746332733;2021-12-06@14:29:11 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;53904;556871;2227484;22;4.744210933;2021-12-06@14:29:47 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;53844;556877;2227508;19;4.092227972;2021-12-06@14:30:25 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;54532;556860;2227440;22;4.737008659;2021-12-06@14:31:04 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;55396;556884;2227536;19;4.102128776;2021-12-06@14:31:43 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;55708;556931;2227724;21;4.530517178;2021-12-06@14:32:23 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;54508;556848;2227392;18;3.875664935;2021-12-06@14:33:02 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;54032;556799;2227196;18;3.878299866;2021-12-06@14:33:40 +v2.0.8-161-ga788372e;multi;32;10000;;16892;6334;54284;556884;2227536;19;4.092237866;2021-12-06@14:34:18 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40140;12621;50484;65;14.149852026;2021-12-06@14:35:06 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40324;12634;50536;64;13.926114889;2021-12-06@14:35:54 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40184;12618;50472;65;14.145470802;2021-12-06@14:36:44 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40572;12700;50800;64;13.919928624;2021-12-06@14:37:31 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40176;12619;50476;66;14.370538072;2021-12-06@14:38:19 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40308;12612;50448;63;13.718296061;2021-12-06@14:39:07 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40296;12626;50504;65;14.138538412;2021-12-06@14:39:55 +v2.0.8-161-ga788372e;imex;32;10000;;16892;6334;40156;12614;50456;63;13.717587900;2021-12-06@14:40:43 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;9324;3886;15544;69;15.006621142;2021-12-06@14:41:34 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;9044;3874;15496;67;14.576172686;2021-12-06@14:42:22 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;9168;3877;15508;67;14.580721314;2021-12-06@14:43:09 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;9360;3874;15496;70;15.215318150;2021-12-06@14:43:57 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;9092;3861;15444;67;14.565809392;2021-12-06@14:44:45 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;8948;3852;15408;67;14.565712451;2021-12-06@14:45:32 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;9308;3908;15632;68;14.795375802;2021-12-06@14:46:22 +v2.0.8-161-ga788372e;single;32;10000;;16892;6334;;9080;3881;15524;65;14.137421494;2021-12-06@14:47:10 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;82076;564613;2258452;21;4.522385738;2021-12-06@14:47:48 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;83560;564925;2259700;20;4.309231785;2021-12-06@14:48:28 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;82556;564756;2259024;22;4.740118038;2021-12-06@14:49:08 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;81832;564848;2259392;20;4.307002878;2021-12-06@14:49:48 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;81964;564749;2258996;20;4.308891257;2021-12-06@14:50:28 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;82512;564788;2259152;19;4.095858857;2021-12-06@14:51:07 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;81600;564683;2258732;18;3.878144182;2021-12-06@14:51:46 +v2.0.8-161-ga788372e;mulimex;32;10000;;16892;6334;81688;564605;2258420;18;3.880192724;2021-12-06@14:52:23 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;90636;960532;3842128;25;5.413937245;2021-12-06@14:53:11 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;90560;960331;3841324;26;5.625838555;2021-12-06@14:54:04 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;90656;960231;3840924;26;5.622985139;2021-12-06@14:54:54 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;88600;959985;3839940;25;5.421197146;2021-12-06@14:55:44 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;92224;960576;3842304;25;5.412104972;2021-12-06@14:56:36 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;90032;960088;3840352;25;5.429419527;2021-12-06@14:57:28 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;88716;960064;3840256;25;5.409571988;2021-12-06@14:58:19 +v2.0.8-161-ga788372e;multi;56;10000;;17919;6345;89904;959874;3839496;25;5.402138865;2021-12-06@14:59:09 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;60804;18448;73792;113;24.614735142;2021-12-06@15:00:42 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;61532;18608;74432;109;23.732816525;2021-12-06@15:01:53 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;60684;18462;73848;109;23.745012029;2021-12-06@15:03:09 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;60824;18483;73932;111;24.182505333;2021-12-06@15:04:22 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;60740;18485;73940;109;23.743722383;2021-12-06@15:05:37 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;60708;18482;73928;109;23.737597586;2021-12-06@15:06:54 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;60812;18493;73972;111;24.171849671;2021-12-06@15:08:12 +v2.0.8-161-ga788372e;imex;56;10000;;17919;6345;60624;18453;73812;109;23.750090485;2021-12-06@15:09:30 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10500;4193;16772;115;25.090458989;2021-12-06@15:10:45 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10576;4198;16792;115;25.053933566;2021-12-06@15:12:01 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10440;4175;16700;114;24.832875853;2021-12-06@15:13:17 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10672;4178;16712;114;24.838940159;2021-12-06@15:14:34 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10424;4191;16764;114;24.845332591;2021-12-06@15:15:51 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10536;4185;16740;114;24.842495171;2021-12-06@15:17:09 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10660;4192;16768;114;24.835230118;2021-12-06@15:18:21 +v2.0.8-161-ga788372e;single;56;10000;;17919;6345;10452;4176;16704;114;24.824440064;2021-12-06@15:19:36 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;133784;973094;3892376;26;5.639381304;2021-12-06@15:20:34 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;134832;973308;3893232;27;5.856331757;2021-12-06@15:21:24 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;136404;973067;3892268;26;5.637067908;2021-12-06@15:22:18 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;133332;973120;3892480;27;5.855872096;2021-12-06@15:23:10 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;134404;973226;3892904;27;5.855301093;2021-12-06@15:24:02 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;137088;973290;3893160;27;5.860414051;2021-12-06@15:24:53 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;136044;973191;3892764;29;6.296841111;2021-12-06@15:25:48 +v2.0.8-161-ga788372e;mulimex;56;10000;;17919;6345;134192;973279;3893116;27;5.865426200;2021-12-06@15:26:41 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;157228;1699840;6799360;47;10.243395607;2021-12-06@15:27:58 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;153884;1699446;6797784;47;10.260627518;2021-12-06@15:29:21 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;154424;1699521;6798084;47;10.232038900;2021-12-06@15:30:40 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;156488;1699984;6799936;46;10.006832801;2021-12-06@15:31:59 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;155764;1700064;6800256;47;10.238049395;2021-12-06@15:33:18 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;158156;1700036;6800144;48;10.447295937;2021-12-06@15:34:39 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;153812;1699510;6798040;47;10.240523586;2021-12-06@15:36:01 +v2.0.8-161-ga788372e;multi;100;10000;;18563;6381;155628;1699815;6799260;46;10.007345700;2021-12-06@15:37:21 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;97528;28982;115928;192;41.865100099;2021-12-06@15:39:15 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;97660;28982;115928;196;42.765122043;2021-12-06@15:41:08 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;98328;29160;116640;196;42.776826745;2021-12-06@15:43:00 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;98092;29098;116392;193;42.113227186;2021-12-06@15:44:55 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;98024;29079;116316;191;41.670385310;2021-12-06@15:46:51 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;97596;28975;115900;194;42.347403585;2021-12-06@15:48:41 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;97940;29089;116356;193;42.124749758;2021-12-06@15:50:32 +v2.0.8-161-ga788372e;imex;100;10000;;18563;6381;98340;29184;116736;194;42.340088089;2021-12-06@15:52:25 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12396;4685;18740;201;43.886697347;2021-12-06@15:54:18 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12416;4650;18600;204;44.557125113;2021-12-06@15:56:12 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12360;4669;18676;203;44.345104651;2021-12-06@15:58:07 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12288;4668;18672;202;44.111571144;2021-12-06@16:00:02 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12372;4679;18716;217;47.417960550;2021-12-06@16:02:00 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12424;4684;18736;204;44.541854720;2021-12-06@16:03:52 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12428;4681;18724;205;44.783031958;2021-12-06@16:05:47 +v2.0.8-161-ga788372e;single;100;10000;;18563;6381;12372;4665;18660;204;44.552089966;2021-12-06@16:07:42 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;238176;1723216;6892864;49;10.710845221;2021-12-06@16:09:01 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;235456;1722775;6891100;50;10.916215590;2021-12-06@16:10:30 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;238756;1723260;6893040;50;10.921517102;2021-12-06@16:11:50 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;235632;1722288;6889152;51;11.104239267;2021-12-06@16:13:14 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;235808;1723028;6892112;50;10.924535204;2021-12-06@16:14:36 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;234916;1722808;6891232;50;10.932362482;2021-12-06@16:15:58 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;234384;1722756;6891024;49;10.673398292;2021-12-06@16:17:19 +v2.0.8-161-ga788372e;mulimex;100;10000;;18563;6381;232416;1722182;6888728;49;10.756571900;2021-12-06@16:18:41 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;274016;2159350;8637400;86;18.810143685;2021-12-06@16:20:41 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;275624;2159202;8636808;85;18.604219427;2021-12-06@16:22:51 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;275532;2159255;8637020;85;18.592356953;2021-12-06@16:25:02 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;270780;2158573;8634292;86;18.833227881;2021-12-06@16:27:11 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;267068;2157881;8631524;84;18.480694418;2021-12-06@16:29:22 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;269316;2159098;8636392;86;18.836943286;2021-12-06@16:31:34 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;270220;2159439;8637756;85;18.572625658;2021-12-06@16:33:45 +v2.0.8-161-ga788372e;multi;178;10000;;19033;6531;274572;2159315;8637260;86;18.796062554;2021-12-06@16:35:55 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;168512;49056;196224;372;81.435064869;2021-12-06@16:39:09 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;167024;48697;194788;368;80.558510744;2021-12-06@16:42:38 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;166748;48570;194280;368;80.585125026;2021-12-06@16:45:55 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;165476;48336;193344;374;81.875080009;2021-12-06@16:49:14 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;167888;48908;195632;383;83.810889861;2021-12-06@16:52:25 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;165844;48383;193532;372;81.602119381;2021-12-06@16:55:38 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;165848;48333;193332;362;79.242521109;2021-12-06@16:58:57 +v2.0.8-161-ga788372e;imex;178;10000;;19033;6531;166668;48572;194288;374;81.881825688;2021-12-06@17:02:05 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15676;5487;21948;381;83.730208659;2021-12-06@17:05:13 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15500;5482;21928;378;82.901160368;2021-12-06@17:08:21 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15760;5464;21856;398;87.240639421;2021-12-06@17:11:32 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15612;5481;21924;379;83.040451555;2021-12-06@17:14:42 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15696;5476;21904;385;84.677077832;2021-12-06@17:17:49 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15576;5473;21892;387;84.804172913;2021-12-06@17:21:00 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15804;5479;21916;388;85.163903295;2021-12-06@17:24:09 +v2.0.8-161-ga788372e;single;178;10000;;19033;6531;15612;5484;21936;381;83.447983331;2021-12-06@17:27:20 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;420112;2201113;8804452;90;19.734871982;2021-12-06@17:29:24 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;416132;2199642;8798568;91;19.886417571;2021-12-06@17:31:28 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;419452;2200580;8802320;91;19.890319401;2021-12-06@17:33:33 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;414268;2199304;8797216;91;19.893538266;2021-12-06@17:46:46 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;414816;2199111;8796444;91;19.908762754;2021-12-06@17:48:53 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;413064;2199621;8798484;92;20.170746382;2021-12-06@17:51:06 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;416784;2199609;8798436;91;19.914442486;2021-12-06@17:53:12 +v2.0.8-161-ga788372e;mulimex;178;10000;;19033;6531;414452;2200847;8803388;91;19.897780806;2021-12-06@17:55:19 +v2.0.8-161-ga788372e;multi;316;10000;;19553;6638;473272;2217524;8870096;160;35.109724230;2021-12-06@17:58:43 +v2.0.8-161-ga788372e;multi;316;10000;;19553;6638;489336;2216918;8867672;160;35.036585681;2021-12-06@18:02:37 +v2.0.8-161-ga788372e;multi;316;10000;;19553;6638;499536;2219768;8879072;166;36.453827085;2021-12-06@18:06:33 +v2.0.8-163-ge813cd67;multi;316;10000;;19553;6638;486876;2217561;8870244;165;36.219933598;2021-12-06@18:10:27 +v2.0.8-163-ge813cd67;multi;316;10000;;19553;6638;492756;2219329;8877316;168;36.803414011;2021-12-06@18:14:27 +v2.0.8-163-ge813cd67;multi;316;10000;;19553;6638;484292;2217366;8869464;167;36.641744843;2021-12-06@18:18:42 +v2.0.8-163-ge813cd67;multi;316;10000;;19553;6638;499180;2219884;8879536;166;36.367060291;2021-12-06@18:22:38 +v2.0.8-163-ge813cd67;multi;316;10000;;19553;6638;491808;2220103;8880412;166;36.263668177;2021-12-06@18:26:35 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;284668;82254;329016;739;162.528245515;2021-12-06@18:33:29 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;294544;84701;338804;729;160.674302810;2021-12-06@18:39:20 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;285424;82451;329804;785;172.657519651;2021-12-06@18:45:21 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;294592;84674;338696;734;161.864779166;2021-12-06@18:51:13 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;283944;82114;328456;731;161.452744341;2021-12-06@18:57:03 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;284620;82202;328808;726;159.871484038;2021-12-06@19:02:50 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;288772;83257;333028;747;164.672569104;2021-12-06@19:08:43 +v2.0.8-163-ge813cd67;imex;316;10000;;19553;6638;289132;83339;333356;723;159.327889660;2021-12-06@19:14:28 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;21032;6826;27304;799;176.389544066;2021-12-06@19:20:35 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;21096;6802;27208;771;170.469001660;2021-12-06@19:26:40 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;21228;6838;27352;771;170.445679124;2021-12-06@19:32:48 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;20984;6789;27156;782;173.457854715;2021-12-06@19:38:50 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;21096;6831;27324;781;172.483058648;2021-12-06@19:44:54 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;20852;6799;27196;775;170.973709728;2021-12-06@19:50:55 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;21016;6810;27240;767;169.147439080;2021-12-06@19:56:53 +v2.0.8-163-ge813cd67;single;316;10000;;19553;6638;21044;6811;27244;782;172.726994658;2021-12-06@20:02:55 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;747296;2293442;9173768;177;38.934544160;2021-12-06@20:06:41 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;748264;2294413;9177652;178;39.279433773;2021-12-06@20:07:22 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;740736;2292451;9169804;181;39.838252438;2021-12-06@20:08:03 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;742224;2292343;9169372;182;40.215076390;2021-12-06@20:08:45 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;740520;2291831;9167324;181;39.731890394;2021-12-06@20:09:26 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;734676;2290626;9162504;179;39.293273175;2021-12-06@20:10:07 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;753600;2294869;9179476;181;39.738133835;2021-12-06@20:10:49 +v2.0.8-163-ge813cd67;mulimex;316;10000;;19553;6638;741224;2292737;9170948;183;40.225612561;2021-12-06@20:11:30 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43220;192073;768292;16;3.501894537;2021-12-06@20:13:05 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43196;192024;768096;18;3.930211244;2021-12-06@20:13:11 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43300;192041;768164;19;4.153959610;2021-12-06@20:13:16 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43332;192050;768200;19;4.154886280;2021-12-06@20:13:22 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43324;192054;768216;19;4.156487913;2021-12-06@20:13:28 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43536;192058;768232;19;4.155744364;2021-12-06@20:13:33 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43232;192053;768212;19;4.158111188;2021-12-06@20:13:39 +v2.0.8-163-ge813cd67;multi;10;17783;;29952;12293;43076;192027;768108;19;4.151844965;2021-12-06@20:13:45 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36312;11009;44036;45;9.845653838;2021-12-06@20:14:05 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36424;11000;44000;46;10.105034944;2021-12-06@20:14:17 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36384;10999;43996;45;9.875427425;2021-12-06@20:14:28 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36440;10994;43976;47;10.319987275;2021-12-06@20:14:40 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36260;11007;44028;46;10.097267729;2021-12-06@20:14:52 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36300;11006;44024;47;10.315758143;2021-12-06@20:15:04 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36388;10993;43972;47;10.325056653;2021-12-06@20:15:15 +v2.0.8-163-ge813cd67;imex;10;17783;;29952;12293;36600;11025;44100;46;10.097867454;2021-12-06@20:15:27 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11824;4515;18060;46;10.075913204;2021-12-06@20:15:46 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11460;4504;18016;50;10.976056302;2021-12-06@20:15:58 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11708;4512;18048;48;10.537367488;2021-12-06@20:16:10 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11492;4504;18016;46;10.095178399;2021-12-06@20:16:22 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11744;4502;18008;46;10.090156325;2021-12-06@20:16:33 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11840;4532;18128;47;10.326571395;2021-12-06@20:16:45 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11748;4522;18088;47;10.317123405;2021-12-06@20:16:57 +v2.0.8-163-ge813cd67;single;10;17783;;29952;12293;11860;4533;18132;47;10.317285413;2021-12-06@20:17:09 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;62000;197033;788132;19;4.096507190;2021-12-06@20:17:22 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;61496;197049;788196;19;4.159589447;2021-12-06@20:17:28 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;61528;197013;788052;19;4.157080121;2021-12-06@20:17:33 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;61336;197054;788216;20;4.375605851;2021-12-06@20:17:39 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;61960;197035;788140;19;4.158493927;2021-12-06@20:17:45 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;62100;197061;788244;19;4.153872220;2021-12-06@20:17:50 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;62156;197028;788112;19;4.158705283;2021-12-06@20:17:56 +v2.0.8-163-ge813cd67;mulimex;10;17783;;29952;12293;61480;197031;788124;19;4.148763160;2021-12-06@20:18:02 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;64672;328806;1315224;20;4.318850415;2021-12-06@20:18:17 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;64820;329049;1316196;22;4.815345088;2021-12-06@20:18:23 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;65324;328908;1315632;22;4.818446147;2021-12-06@20:18:30 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;65360;328856;1315424;22;4.817503278;2021-12-06@20:18:36 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;64740;328777;1315108;23;5.041512642;2021-12-06@20:18:43 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;64620;328807;1315228;22;4.812746187;2021-12-06@20:18:49 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;65080;328962;1315848;23;5.036722026;2021-12-06@20:18:55 +v2.0.8-163-ge813cd67;multi;18;17783;;31744;12312;65008;328785;1315140;22;4.814744526;2021-12-06@20:19:02 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49696;14547;58188;76;16.710933951;2021-12-06@20:19:31 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49616;14539;58156;76;16.719335322;2021-12-06@20:19:49 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49788;14561;58244;76;16.720804601;2021-12-06@20:20:08 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49420;14537;58148;74;16.291328611;2021-12-06@20:20:25 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49760;14542;58168;75;16.506797991;2021-12-06@20:20:43 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49460;14543;58172;75;16.510428069;2021-12-06@20:21:01 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49488;14565;58260;78;17.159815365;2021-12-06@20:21:20 +v2.0.8-163-ge813cd67;imex;18;17783;;31744;12312;49628;14539;58156;77;16.937557715;2021-12-06@20:21:38 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12664;4752;19008;78;17.150778775;2021-12-06@20:22:09 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12852;4752;19008;75;16.572674042;2021-12-06@20:22:27 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12696;4735;18940;74;16.284344401;2021-12-06@20:22:45 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12684;4743;18972;77;16.958014917;2021-12-06@20:23:03 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12856;4766;19064;76;16.736716239;2021-12-06@20:23:21 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12708;4750;19000;76;16.724097892;2021-12-06@20:23:40 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12792;4758;19032;80;17.605004747;2021-12-06@20:23:59 +v2.0.8-163-ge813cd67;single;18;17783;;31744;12312;12812;4759;19036;79;17.394453901;2021-12-06@20:24:18 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;95888;336939;1347756;21;4.545320025;2021-12-06@20:24:36 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;95048;336946;1347784;23;5.043875112;2021-12-06@20:24:42 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;94508;336884;1347536;23;5.047837888;2021-12-06@20:24:49 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;94472;336843;1347372;23;5.044710659;2021-12-06@20:24:55 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;94740;336885;1347540;23;5.043086984;2021-12-06@20:25:02 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;94692;336878;1347512;24;5.263748739;2021-12-06@20:25:09 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;94916;336884;1347536;23;5.038846352;2021-12-06@20:25:15 +v2.0.8-163-ge813cd67;mulimex;18;17783;;31744;12312;95928;336986;1347944;23;5.045801500;2021-12-06@20:25:22 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;102200;568266;2273064;27;5.889785287;2021-12-06@20:25:48 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;103496;568251;2273004;33;7.254135647;2021-12-06@20:25:57 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;103060;568155;2272620;33;7.258544791;2021-12-06@20:26:05 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;103120;568281;2273124;33;7.259433077;2021-12-06@20:26:14 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;102232;568193;2272772;33;7.260431843;2021-12-06@20:26:23 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;103408;568131;2272524;33;7.257202648;2021-12-06@20:26:32 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;102248;568195;2272780;32;7.033852291;2021-12-06@20:26:40 +v2.0.8-163-ge813cd67;multi;32;17783;;32894;12393;103584;568271;2273084;33;7.250038875;2021-12-06@20:26:49 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71536;20480;81920;130;28.634837924;2021-12-06@20:27:41 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71828;20484;81936;127;27.989196410;2021-12-06@20:28:11 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71440;20475;81900;128;28.197493992;2021-12-06@20:28:41 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71676;20476;81904;129;28.434298098;2021-12-06@20:29:11 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71684;20463;81852;128;28.214101359;2021-12-06@20:29:41 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71496;20481;81924;127;28.004319718;2021-12-06@20:30:10 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71432;20462;81848;128;28.211302327;2021-12-06@20:30:40 +v2.0.8-163-ge813cd67;imex;32;17783;;32894;12393;71788;20486;81944;129;28.440937824;2021-12-06@20:31:10 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;13788;5053;20212;138;30.418650112;2021-12-06@20:32:03 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;14100;5056;20224;130;28.650281365;2021-12-06@20:32:33 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;14112;5053;20212;132;29.096653304;2021-12-06@20:33:04 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;13752;5054;20216;133;29.323401255;2021-12-06@20:33:34 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;14000;5051;20204;130;28.640473480;2021-12-06@20:34:06 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;13808;5053;20212;128;28.205272779;2021-12-06@20:34:36 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;13872;5066;20264;129;28.431496100;2021-12-06@20:35:06 +v2.0.8-163-ge813cd67;single;32;17783;;32894;12393;13728;5058;20232;134;29.531467650;2021-12-06@20:35:37 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;152396;581787;2327148;29;6.324602926;2021-12-06@20:36:04 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;153572;581745;2326980;34;7.484398688;2021-12-06@20:36:13 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;153808;581826;2327304;34;7.483412616;2021-12-06@20:36:22 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;153264;581724;2326896;34;7.485421176;2021-12-06@20:36:30 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;152912;581772;2327088;34;7.483382201;2021-12-06@20:36:39 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;153932;581737;2326948;34;7.481120459;2021-12-06@20:36:48 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;152668;581760;2327040;35;7.705629881;2021-12-06@20:36:58 +v2.0.8-163-ge813cd67;mulimex;32;17783;;32894;12393;153600;581890;2327560;33;7.260312320;2021-12-06@20:37:06 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;167480;978389;3913556;46;10.121600957;2021-12-06@20:37:47 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;167404;978474;3913896;51;11.233279854;2021-12-06@20:38:00 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;168088;978336;3913344;51;11.248628932;2021-12-06@20:38:13 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;168432;978410;3913640;52;11.469414891;2021-12-06@20:38:26 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;169440;978712;3914848;52;11.480132192;2021-12-06@20:38:39 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;168784;978638;3914552;53;11.694225140;2021-12-06@20:38:52 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;165712;978262;3913048;52;11.480888442;2021-12-06@20:39:05 +v2.0.8-163-ge813cd67;multi;56;17783;;33578;12411;166332;978182;3912728;51;11.254613500;2021-12-06@20:39:18 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108252;30352;121408;217;47.854685718;2021-12-06@20:40:46 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108100;30371;121484;222;48.990026257;2021-12-06@20:41:36 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108020;30302;121208;214;47.192668348;2021-12-06@20:42:25 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108024;30293;121172;211;46.537644076;2021-12-06@20:43:13 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108008;30312;121248;229;50.482256930;2021-12-06@20:44:05 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108220;30354;121416;220;48.511016057;2021-12-06@20:44:55 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108432;30316;121264;219;48.295091310;2021-12-06@20:45:45 +v2.0.8-163-ge813cd67;imex;56;17783;;33578;12411;108204;30357;121428;222;48.979834405;2021-12-06@20:46:35 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15228;5432;21728;240;52.940262022;2021-12-06@20:48:04 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15304;5435;21740;228;50.303076962;2021-12-06@20:48:56 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15548;5459;21836;220;48.554455486;2021-12-06@20:49:47 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15308;5444;21776;222;49.008509564;2021-12-06@20:50:38 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15428;5452;21808;221;48.781160247;2021-12-06@20:51:31 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15228;5455;21820;217;47.883786884;2021-12-06@20:52:21 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15288;5426;21704;258;56.958597239;2021-12-06@20:53:19 +v2.0.8-163-ge813cd67;single;56;17783;;33578;12411;15500;5451;21804;222;48.989794059;2021-12-06@20:54:09 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;250968;1001355;4005420;50;11.060636581;2021-12-06@20:54:56 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;252784;1001514;4006056;55;12.152078628;2021-12-06@20:55:10 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;252052;1001277;4005108;56;12.374197611;2021-12-06@20:55:26 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;250648;1001277;4005108;54;11.926556592;2021-12-06@20:55:41 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;250812;1001283;4005132;55;12.158614198;2021-12-06@20:55:57 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;252516;1001220;4004880;56;12.367040358;2021-12-06@20:56:13 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;252056;1001418;4005672;57;12.576237327;2021-12-06@20:56:29 +v2.0.8-163-ge813cd67;mulimex;56;17783;;33578;12411;252780;1001270;4005080;57;12.599500941;2021-12-06@20:56:45 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;286552;1730237;6920948;87;19.241520785;2021-12-06@20:58:01 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;288848;1731029;6924116;92;20.325654482;2021-12-06@20:58:23 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;288908;1730865;6923460;93;20.556020913;2021-12-06@20:58:45 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;284360;1730148;6920592;92;20.374314073;2021-12-06@20:59:08 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;289104;1730877;6923508;95;21.010108917;2021-12-06@20:59:31 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;288204;1731890;6927560;94;20.817362860;2021-12-06@20:59:54 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;289448;1731094;6924376;88;19.514472725;2021-12-06@21:00:16 +v2.0.8-163-ge813cd67;multi;100;17783;;33951;12499;284968;1730647;6922588;92;20.373575097;2021-12-06@21:00:39 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;175420;48437;193748;393;86.931550083;2021-12-06@21:03:13 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;175680;48530;194120;389;86.047319063;2021-12-06@21:04:40 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;174932;48333;193332;389;86.042806907;2021-12-06@21:06:08 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;175256;48409;193636;374;82.686947744;2021-12-06@21:07:32 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;174264;48227;192908;391;86.534293120;2021-12-06@21:09:00 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;175164;48321;193284;395;87.429189880;2021-12-06@21:10:29 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;174788;48355;193420;386;85.434448517;2021-12-06@21:11:56 +v2.0.8-163-ge813cd67;imex;100;17783;;33951;12499;174720;48348;193392;377;83.379206264;2021-12-06@21:13:21 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17808;6029;24116;409;90.665509842;2021-12-06@21:15:57 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17564;6010;24040;406;90.077951916;2021-12-06@21:17:29 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17840;6006;24024;409;90.699169533;2021-12-06@21:19:01 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17744;6054;24216;400;88.774242984;2021-12-06@21:20:32 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17516;5977;23908;420;93.218086288;2021-12-06@21:22:06 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17516;5966;23864;403;89.497769150;2021-12-06@21:23:37 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17776;6063;24252;409;90.766893951;2021-12-06@21:25:09 +v2.0.8-163-ge813cd67;single;100;17783;;33951;12499;17440;5989;23956;411;91.293720343;2021-12-06@21:26:42 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;434020;1770411;7081644;92;20.374003266;2021-12-06@21:28:07 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;433596;1770884;7083536;97;21.449047419;2021-12-06@21:28:29 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;432756;1770333;7081332;98;21.788378714;2021-12-06@21:28:53 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;432368;1770376;7081504;98;21.669638297;2021-12-06@21:29:16 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;432388;1770464;7081856;98;21.721247625;2021-12-06@21:29:39 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;433992;1770388;7081552;98;21.706651846;2021-12-06@21:30:02 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;432252;1770500;7082000;97;21.460912763;2021-12-06@21:30:25 +v2.0.8-163-ge813cd67;mulimex;100;17783;;33951;12499;431716;1770274;7081096;98;21.706835494;2021-12-06@21:30:48 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;493224;2211142;8844568;156;34.631798281;2021-12-06@21:33:00 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;495676;2211991;8847964;162;35.912034559;2021-12-06@21:33:37 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;491824;2211005;8844020;162;35.949107695;2021-12-06@21:34:15 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;493592;2211067;8844268;164;36.282322167;2021-12-06@21:34:52 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;499688;2212468;8849872;163;36.084842565;2021-12-06@21:35:30 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;499512;2212139;8848556;163;36.161966909;2021-12-06@21:36:08 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;498736;2212306;8849224;161;35.698107826;2021-12-06@21:36:45 +v2.0.8-163-ge813cd67;multi;178;17783;;34156;12582;497008;2212270;8849080;164;36.357921824;2021-12-06@21:37:23 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;294084;80485;321940;713;158.285876408;2021-12-06@21:42:00 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;292964;80178;320712;696;154.492668580;2021-12-06@21:44:36 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;294524;80562;322248;701;155.776462026;2021-12-06@21:47:14 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;293552;80305;321220;733;163.349949427;2021-12-06@21:49:58 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;295016;80685;322740;681;151.631866600;2021-12-06@21:52:32 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;292956;80171;320684;680;150.936769125;2021-12-06@21:55:04 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;294468;80503;322012;672;149.162057416;2021-12-06@21:57:35 +v2.0.8-163-ge813cd67;imex;178;17783;;34156;12582;294588;80557;322228;703;156.113833841;2021-12-06@22:00:12 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21020;6899;27596;750;166.981207920;2021-12-06@22:04:52 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21268;6893;27572;765;170.676730082;2021-12-06@22:07:44 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21032;6867;27468;724;161.047044019;2021-12-06@22:10:27 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21076;6867;27468;746;166.213513958;2021-12-06@22:13:14 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21252;6880;27520;702;156.176459554;2021-12-06@22:15:53 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21252;6906;27624;745;165.893160710;2021-12-06@22:18:41 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21328;6885;27540;716;159.211054427;2021-12-06@22:21:22 +v2.0.8-163-ge813cd67;single;178;17783;;34156;12582;21372;6912;27648;753;167.677584181;2021-12-06@22:24:12 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;761996;2283589;9134356;158;35.077615140;2021-12-06@22:26:35 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;757388;2282014;9128056;173;38.366688669;2021-12-06@22:27:15 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;760388;2283387;9133548;172;38.185089177;2021-12-06@22:27:54 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;754976;2281953;9127812;169;37.510356497;2021-12-06@22:28:33 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;757304;2282234;9128936;170;37.729832892;2021-12-06@22:29:13 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;756040;2281805;9127220;174;38.618450690;2021-12-06@22:29:53 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;760120;2283030;9132120;169;37.503847768;2021-12-06@22:30:32 +v2.0.8-163-ge813cd67;mulimex;178;17783;;34156;12582;761100;2283870;9135480;169;37.636358569;2021-12-06@22:31:11 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;876356;2311836;9247344;293;65.392363755;2021-12-06@22:35:12 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;867084;2310575;9242300;291;65.238325685;2021-12-06@22:36:20 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;876580;2311633;9246532;298;66.139101648;2021-12-06@22:37:27 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;864840;2309843;9239372;285;63.312026602;2021-12-06@22:38:32 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;875652;2311655;9246620;293;65.272628546;2021-12-06@22:39:39 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;864216;2309545;9238180;291;64.769997895;2021-12-06@22:40:45 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;858300;2308839;9235356;293;65.119980084;2021-12-06@22:41:52 +v2.0.8-163-ge813cd67;multi;316;17783;;34574;12710;873928;2311692;9246768;299;66.837449563;2021-12-06@22:43:01 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;508512;138198;552792;1319;294.454130761;2021-12-06@22:51:14 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;506972;137849;551396;1320;295.056020887;2021-12-06@22:56:11 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;515604;140023;560092;1291;288.661181754;2021-12-06@23:01:01 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;507152;137844;551376;1278;285.510122239;2021-12-06@23:06:03 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;506552;137709;550836;1321;294.923234433;2021-12-06@23:11:00 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;511720;138992;555968;1282;287.720350930;2021-12-06@23:15:49 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;506536;137741;550964;1340;300.026245249;2021-12-06@23:20:51 +v2.0.8-163-ge813cd67;imex;316;17783;;34574;12710;515792;139964;559856;1269;284.149055276;2021-12-06@23:25:36 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27536;8477;33908;1375;307.628461075;2021-12-06@23:34:13 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27484;8425;33700;1372;307.259035026;2021-12-06@23:39:22 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27340;8435;33740;1395;312.356857004;2021-12-06@23:44:36 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27532;8441;33764;1377;308.193987236;2021-12-06@23:49:46 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27304;8441;33764;1398;312.486161392;2021-12-06@23:55:00 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27448;8468;33872;1391;311.376580435;2021-12-07@00:00:12 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27408;8488;33952;1415;316.358885819;2021-12-07@00:05:30 +v2.0.8-163-ge813cd67;single;316;17783;;34574;12710;27208;8411;33644;1395;312.665274985;2021-12-07@00:10:44 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1345012;2437790;9751160;303;67.809636965;2021-12-07@00:15:15 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1343316;2438145;9752580;310;68.863045109;2021-12-07@00:16:26 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1333476;2436939;9747756;313;69.917316184;2021-12-07@00:17:37 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1339004;2437407;9749628;311;69.161857163;2021-12-07@00:18:49 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1339604;2436905;9747620;305;67.833680449;2021-12-07@00:19:58 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1340520;2438025;9752100;312;69.367867416;2021-12-07@00:21:09 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1350528;2439559;9758236;309;68.616446795;2021-12-07@00:22:20 +v2.0.8-163-ge813cd67;mulimex;316;17783;;34574;12710;1342512;2438486;9753944;311;69.337702741;2021-12-07@00:23:31 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76764;199484;797936;25;5.577802708;2021-12-07@00:25:14 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76900;199458;797832;26;5.788412392;2021-12-07@00:25:21 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76992;199475;797900;27;6.017407981;2021-12-07@00:25:29 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76676;199483;797932;27;6.020662961;2021-12-07@00:25:36 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76620;199481;797924;27;6.020711463;2021-12-07@00:25:44 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76740;199655;798620;26;5.783806234;2021-12-07@00:25:51 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76904;199527;798108;27;6.026277832;2021-12-07@00:25:58 +v2.0.8-163-ge813cd67;multi;10;31623;;54425;24231;76908;199483;797932;27;6.022514340;2021-12-07@00:26:06 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64644;18025;72100;77;17.226446663;2021-12-07@00:26:32 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64444;18016;72064;80;17.907633542;2021-12-07@00:26:51 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64408;18007;72028;80;17.926030627;2021-12-07@00:27:11 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64464;18013;72052;77;17.233416684;2021-12-07@00:27:29 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64412;18030;72120;79;17.687326143;2021-12-07@00:27:49 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64364;18005;72020;81;18.123377453;2021-12-07@00:28:08 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64440;18016;72064;82;18.364293282;2021-12-07@00:28:28 +v2.0.8-163-ge813cd67;imex;10;31623;;54425;24231;64492;18000;72000;76;17.009441753;2021-12-07@00:28:47 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18460;6167;24668;81;18.101138551;2021-12-07@00:29:13 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18504;6168;24672;78;17.458490078;2021-12-07@00:29:32 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18448;6194;24776;80;17.911326333;2021-12-07@00:29:51 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18436;6177;24708;81;18.146125845;2021-12-07@00:30:11 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18556;6169;24676;78;17.452083239;2021-12-07@00:30:30 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18480;6176;24704;79;17.677930319;2021-12-07@00:30:49 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18460;6170;24680;81;18.121845008;2021-12-07@00:31:09 +v2.0.8-163-ge813cd67;single;10;31623;;54425;24231;18420;6170;24680;79;17.678617968;2021-12-07@00:31:28 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;112172;208924;835696;25;5.495750793;2021-12-07@00:31:42 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;111604;208478;833912;27;6.021183973;2021-12-07@00:31:50 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;111644;208531;834124;27;6.019107549;2021-12-07@00:31:58 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;112108;208522;834088;27;6.025052294;2021-12-07@00:32:05 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;111708;208578;834312;27;6.023775603;2021-12-07@00:32:13 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;112200;208933;835732;28;6.250478218;2021-12-07@00:32:20 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;111696;208506;834024;27;6.022541521;2021-12-07@00:32:28 +v2.0.8-163-ge813cd67;mulimex;10;31623;;54425;24231;111580;208581;834324;27;6.020572487;2021-12-07@00:32:36 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;116424;340481;1361924;29;6.400766487;2021-12-07@00:32:53 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;116692;340571;1362284;33;7.379586660;2021-12-07@00:33:02 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;116604;340586;1362344;33;7.381501958;2021-12-07@00:33:11 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;116620;340566;1362264;33;7.381369677;2021-12-07@00:33:20 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;116664;340622;1362488;33;7.384938356;2021-12-07@00:33:28 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;117200;340835;1363340;33;7.373041717;2021-12-07@00:33:37 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;116348;340492;1361968;33;7.389839125;2021-12-07@00:33:46 +v2.0.8-163-ge813cd67;multi;18;31623;;57202;24252;116464;340628;1362512;33;7.379063027;2021-12-07@00:33:55 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88760;24347;97388;134;30.083540487;2021-12-07@00:34:38 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88600;24339;97356;135;30.306881467;2021-12-07@00:35:10 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88580;24323;97292;136;30.533026576;2021-12-07@00:35:42 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88728;24308;97232;145;32.572913302;2021-12-07@00:36:16 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88696;24332;97328;136;30.525377490;2021-12-07@00:36:48 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88808;24349;97396;131;29.401545980;2021-12-07@00:37:19 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88932;24318;97272;136;30.529224325;2021-12-07@00:37:51 +v2.0.8-163-ge813cd67;imex;18;31623;;57202;24252;88752;24336;97344;134;30.071959037;2021-12-07@00:38:22 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19868;6518;26072;136;30.477845740;2021-12-07@00:39:08 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19816;6510;26040;135;30.291957133;2021-12-07@00:39:40 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19812;6532;26128;134;30.058306208;2021-12-07@00:40:12 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19824;6503;26012;134;30.088943675;2021-12-07@00:40:43 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19744;6516;26064;137;30.729689582;2021-12-07@00:41:15 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19864;6523;26092;135;30.302545625;2021-12-07@00:41:47 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19836;6506;26024;135;30.299142632;2021-12-07@00:42:19 +v2.0.8-163-ge813cd67;single;18;31623;;57202;24252;19816;6503;26012;133;29.843170551;2021-12-07@00:42:50 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;173404;355989;1423956;31;6.885606766;2021-12-07@00:43:10 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;172724;355300;1421200;35;7.833118953;2021-12-07@00:43:19 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;173084;355306;1421224;35;7.847829123;2021-12-07@00:43:29 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;172604;355301;1421204;35;7.830660423;2021-12-07@00:43:38 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;173664;355321;1421284;34;7.621537193;2021-12-07@00:43:47 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;173276;355327;1421308;35;7.841106487;2021-12-07@00:43:56 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;173584;355401;1421604;35;7.844758686;2021-12-07@00:44:06 +v2.0.8-163-ge813cd67;mulimex;18;31623;;57202;24252;174300;355426;1421704;35;7.845561359;2021-12-07@00:44:15 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;182716;587123;2348492;47;10.579153166;2021-12-07@00:44:43 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;184024;587074;2348296;51;11.463146307;2021-12-07@00:44:56 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;184800;586959;2347836;53;11.911091684;2021-12-07@00:45:10 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;184032;587062;2348248;52;11.692711266;2021-12-07@00:45:23 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;186044;587247;2348988;52;11.695563953;2021-12-07@00:45:36 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;184560;588183;2352732;52;11.701077247;2021-12-07@00:45:49 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;185772;588521;2354084;52;11.689344915;2021-12-07@00:46:02 +v2.0.8-163-ge813cd67;multi;32;31623;;58800;24186;185504;587566;2350264;52;11.690960133;2021-12-07@00:46:16 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;128752;34758;139032;237;53.312138122;2021-12-07@00:47:30 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;128692;34777;139108;228;51.235768652;2021-12-07@00:48:23 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;129096;34807;139228;234;52.648362874;2021-12-07@00:49:17 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;128696;34741;138964;232;52.133848451;2021-12-07@00:50:10 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;128676;34775;139100;229;51.523463627;2021-12-07@00:51:03 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;128924;34766;139064;234;52.616067758;2021-12-07@00:51:58 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;128692;34772;139088;236;53.036605963;2021-12-07@00:52:52 +v2.0.8-163-ge813cd67;imex;32;31623;;58800;24186;129244;34831;139324;230;51.705653714;2021-12-07@00:53:45 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21324;6853;27412;245;55.167119391;2021-12-07@00:55:01 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21180;6852;27408;236;53.172595220;2021-12-07@00:55:56 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21072;6862;27448;242;54.524588789;2021-12-07@00:56:52 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21176;6855;27420;236;53.161102168;2021-12-07@00:57:47 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21320;6848;27392;232;52.259276307;2021-12-07@00:58:40 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21052;6873;27492;232;52.266579352;2021-12-07@00:59:34 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21088;6856;27424;233;52.467697369;2021-12-07@01:00:28 +v2.0.8-163-ge813cd67;single;32;31623;;58800;24186;21468;6856;27424;237;53.383505884;2021-12-07@01:01:23 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;278572;611752;2447008;52;11.653541875;2021-12-07@01:01:58 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;278404;612760;2451040;55;12.368262406;2021-12-07@01:02:12 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;279616;612778;2451112;53;11.911194767;2021-12-07@01:02:25 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;278372;611777;2447108;55;12.368401746;2021-12-07@01:02:39 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;278708;611731;2446924;55;12.381498100;2021-12-07@01:02:53 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;278272;611671;2446684;54;12.134275556;2021-12-07@01:03:07 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;278436;611804;2447216;55;12.363755621;2021-12-07@01:03:21 +v2.0.8-163-ge813cd67;mulimex;32;31623;;58800;24186;278860;611664;2446656;55;12.360190208;2021-12-07@01:03:35 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;300868;1011829;4047316;79;17.841550637;2021-12-07@01:04:22 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;301204;1009790;4039160;85;19.150460681;2021-12-07@01:04:42 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;301884;1009931;4039724;85;19.156987935;2021-12-07@01:05:03 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;303212;1011935;4047740;85;19.168569429;2021-12-07@01:05:24 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;302832;1009878;4039512;85;19.152589510;2021-12-07@01:05:44 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;301332;1009520;4038080;85;19.156489738;2021-12-07@01:06:05 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;304196;1010217;4040868;86;19.419680958;2021-12-07@01:06:26 +v2.0.8-163-ge813cd67;multi;56;31623;;59758;24285;302612;1010067;4040268;85;19.179914363;2021-12-07@01:06:47 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197272;52561;210244;411;92.667699750;2021-12-07@01:08:57 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197560;52712;210848;392;88.400097042;2021-12-07@01:10:27 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197332;52624;210496;393;88.657107917;2021-12-07@01:11:57 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197476;52616;210464;395;89.129672065;2021-12-07@01:13:28 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197100;52596;210384;410;92.483124898;2021-12-07@01:15:02 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197380;52585;210340;390;87.999132200;2021-12-07@01:16:32 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197308;52606;210424;388;87.613169021;2021-12-07@01:18:01 +v2.0.8-163-ge813cd67;imex;56;31623;;59758;24285;197384;52611;210444;389;87.708007347;2021-12-07@01:19:30 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;22788;7301;29204;418;94.321222264;2021-12-07@01:21:40 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;22700;7295;29180;410;92.585081943;2021-12-07@01:23:14 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;23016;7326;29304;411;92.781088585;2021-12-07@01:24:48 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;23032;7295;29180;403;90.937158773;2021-12-07@01:26:21 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;23032;7290;29160;411;92.785834058;2021-12-07@01:27:55 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;22900;7280;29120;402;90.704430019;2021-12-07@01:29:27 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;22868;7305;29220;407;91.854492069;2021-12-07@01:31:01 +v2.0.8-163-ge813cd67;single;56;31623;;59758;24285;22860;7289;29156;409;92.300350374;2021-12-07@01:32:34 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;461260;1053358;4213432;87;19.699291229;2021-12-07@01:33:30 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;460912;1051415;4205660;92;20.785507756;2021-12-07@01:33:53 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;461152;1053291;4213164;87;19.726254969;2021-12-07@01:34:29 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;459876;1051307;4205228;92;20.775752567;2021-12-07@01:34:51 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;460072;1051324;4205296;91;20.555773537;2021-12-07@01:35:13 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;460996;1053211;4212844;91;20.528760828;2021-12-07@01:35:35 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;459916;1051392;4205568;92;20.781245594;2021-12-07@01:35:58 +v2.0.8-163-ge813cd67;mulimex;56;31623;;59758;24285;459308;1053332;4213328;92;20.789469705;2021-12-07@01:36:20 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;517380;1784173;7136692;153;34.416894304;2021-12-07@01:37:47 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;515612;1787609;7150436;157;35.405846288;2021-12-07@01:38:24 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;515420;1787527;7150108;154;34.881555762;2021-12-07@01:39:01 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;516432;1784413;7137652;155;35.032807443;2021-12-07@01:39:37 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;513816;1783791;7135164;155;35.070035286;2021-12-07@01:40:14 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;515732;1788102;7152408;157;35.566989248;2021-12-07@01:40:51 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;515368;1787864;7151456;152;34.373459724;2021-12-07@01:41:27 +v2.0.8-163-ge813cd67;multi;100;31623;;60390;24233;514820;1783941;7135764;158;35.657870096;2021-12-07@01:42:04 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;320068;84718;338872;709;160.467395019;2021-12-07@01:45:52 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;320124;84653;338612;719;162.870144403;2021-12-07@01:48:36 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;320528;84778;339112;695;157.321678340;2021-12-07@01:51:15 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;321228;84926;339704;710;160.900969657;2021-12-07@01:53:57 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;320524;84759;339036;714;161.734540286;2021-12-07@01:56:41 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;321320;84981;339924;734;166.062190793;2021-12-07@01:59:28 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;319704;84544;338176;783;177.636056130;2021-12-07@02:02:27 +v2.0.8-163-ge813cd67;imex;100;31623;;60390;24233;320308;84771;339084;725;164.311025332;2021-12-07@02:05:13 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25224;7915;31660;737;166.981266478;2021-12-07@02:09:01 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25580;7940;31760;720;163.318866170;2021-12-07@02:11:46 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25420;7917;31668;723;164.033187130;2021-12-07@02:14:32 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25660;7925;31700;752;170.489334259;2021-12-07@02:17:24 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25400;7944;31776;725;164.360053176;2021-12-07@02:20:10 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25464;7929;31716;752;170.679482590;2021-12-07@02:23:02 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25596;7948;31792;743;168.575556044;2021-12-07@02:25:52 +v2.0.8-163-ge813cd67;single;100;31623;;60390;24233;25720;7942;31768;725;164.418423815;2021-12-07@02:28:38 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;789592;1856704;7426816;159;36.027172624;2021-12-07@02:30:20 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;793800;1857106;7428424;166;37.648918435;2021-12-07@02:30:59 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;794172;1857583;7430332;168;38.092730405;2021-12-07@02:31:39 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;793608;1857138;7428552;166;37.684220950;2021-12-07@02:32:18 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;791064;1856783;7427132;169;38.270562574;2021-12-07@02:32:58 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;792100;1856917;7427668;164;37.153277285;2021-12-07@02:33:37 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;794384;1857127;7428508;165;37.381511046;2021-12-07@02:34:15 +v2.0.8-163-ge813cd67;mulimex;100;31623;;60390;24233;792788;1860509;7442036;164;37.170801259;2021-12-07@02:34:54 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;898856;2306688;9226752;282;63.485166959;2021-12-07@02:37:34 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;896396;2306813;9227252;285;64.147590879;2021-12-07@02:38:40 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;894276;2305744;9222976;289;64.815940230;2021-12-07@02:39:46 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;892804;2305880;9223520;288;65.236804114;2021-12-07@02:40:53 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;891536;2305180;9220720;284;64.155186868;2021-12-07@02:41:58 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;896172;2306350;9225400;288;64.981371702;2021-12-07@02:43:05 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;899116;2306601;9226404;286;64.785877279;2021-12-07@02:44:11 +v2.0.8-163-ge813cd67;multi;178;31623;;60806;24427;894444;2305725;9222900;286;64.495302153;2021-12-07@02:45:17 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;545384;143317;573268;1322;301.218931787;2021-12-07@02:52:17 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;542120;142445;569780;1271;289.021362138;2021-12-07@02:57:07 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;540200;142046;568184;1246;283.686969006;2021-12-07@03:01:53 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;545040;143179;572716;1275;290.751202239;2021-12-07@03:06:45 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;541324;142212;568848;1296;294.315330410;2021-12-07@03:11:41 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;545964;143497;573988;1328;302.179796484;2021-12-07@03:16:44 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;541064;142169;568676;1219;277.303647172;2021-12-07@03:21:23 +v2.0.8-163-ge813cd67;imex;178;31623;;60806;24427;543708;142865;571460;1219;277.251790644;2021-12-07@03:26:02 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29568;8953;35812;1381;314.933741822;2021-12-07@03:33:15 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29852;8987;35948;1381;314.884152091;2021-12-07@03:38:31 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29464;8955;35820;1315;299.610186275;2021-12-07@03:43:32 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29584;9000;36000;1341;305.879601392;2021-12-07@03:48:40 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29728;8974;35896;1327;302.430958846;2021-12-07@03:53:44 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29684;8987;35948;1340;305.636436267;2021-12-07@03:58:51 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29636;8994;35976;1343;306.805716527;2021-12-07@04:03:59 +v2.0.8-163-ge813cd67;single;178;31623;;60806;24427;29812;8938;35752;1358;309.108867395;2021-12-07@04:09:10 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1383884;2433999;9735996;288;65.782263965;2021-12-07@04:12:09 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1387548;2440387;9761548;296;67.251556543;2021-12-07@04:13:18 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1380996;2439256;9757024;289;65.705329339;2021-12-07@04:14:25 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1386556;2434621;9738484;303;68.836958194;2021-12-07@04:15:36 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1394092;2436329;9745316;302;67.997739998;2021-12-07@04:16:46 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1390056;2435491;9741964;304;68.100654920;2021-12-07@04:17:56 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1386112;2434454;9737816;298;67.415780839;2021-12-07@04:19:05 +v2.0.8-163-ge813cd67;mulimex;178;31623;;60806;24427;1387456;2435007;9740028;298;67.383057018;2021-12-07@04:20:14 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1564356;2475939;9903756;494;113.081628459;2021-12-07@04:25:05 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1561796;2484820;9939280;499;114.135198074;2021-12-07@04:27:01 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1568952;2476967;9907868;495;113.669964646;2021-12-07@04:28:57 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1569832;2477406;9909624;497;113.601881144;2021-12-07@04:30:52 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1558324;2474234;9896936;496;113.867045197;2021-12-07@04:32:48 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1568836;2477533;9910132;497;113.782203885;2021-12-07@04:34:43 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1565824;2485586;9942344;501;114.821111493;2021-12-07@04:36:40 +v2.0.8-163-ge813cd67;multi;316;31623;;61364;24592;1565860;2485028;9940112;498;114.275231717;2021-12-07@04:38:36 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;936584;245198;980792;2427;554.549783224;2021-12-07@04:51:18 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;933528;244480;977920;2295;524.808953460;2021-12-07@05:00:04 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;932280;244185;976740;2408;550.042429410;2021-12-07@05:09:16 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;932532;244198;976792;2362;540.242527510;2021-12-07@05:18:17 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;934304;244660;978640;2331;532.427188477;2021-12-07@05:27:11 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;931900;244106;976424;2330;532.685504183;2021-12-07@05:36:06 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;933356;244429;977716;2394;547.319692755;2021-12-07@05:45:15 +v2.0.8-163-ge813cd67;imex;316;31623;;61364;24592;936964;245256;981024;2406;550.548794255;2021-12-07@05:54:27 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36176;10588;42352;2521;579.773663369;2021-12-07@06:07:43 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36132;10623;42492;2531;581.015621061;2021-12-07@06:17:26 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36304;10607;42428;2965;681.686200617;2021-12-07@06:28:49 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36136;10581;42324;2474;567.921765651;2021-12-07@06:38:19 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36072;10599;42396;2499;574.871336843;2021-12-07@06:47:56 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36136;10570;42280;2496;572.756761287;2021-12-07@06:57:31 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36184;10618;42472;2534;582.963105866;2021-12-07@07:07:15 +v2.0.8-163-ge813cd67;single;316;31623;;61364;24592;36368;10586;42344;2635;605.514667353;2021-12-07@07:17:23 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2440152;2704646;10818584;514;118.361277565;2021-12-07@07:22:53 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2440672;2704599;10818396;525;121.075359678;2021-12-07@07:24:56 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2438684;2703467;10813868;507;116.389648303;2021-12-07@07:26:55 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2438696;2703819;10815276;526;120.279403573;2021-12-07@07:28:58 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2436868;2704424;10817696;529;121.009792457;2021-12-07@07:31:02 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2434668;2703865;10815460;538;123.069886967;2021-12-07@07:33:08 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2442272;2704834;10819336;526;120.496355730;2021-12-07@07:35:11 +v2.0.8-163-ge813cd67;mulimex;316;31623;;61364;24592;2453404;2707315;10829260;530;121.430644180;2021-12-07@07:37:15 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;115296;208015;832060;35;7.964320633;2021-12-07@07:39:04 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;115392;208046;832184;40;9.061927972;2021-12-07@07:39:15 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;115584;208101;832404;39;8.861481273;2021-12-07@07:39:25 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;115808;208032;832128;40;9.087204592;2021-12-07@07:39:37 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;114916;207983;831932;39;8.854858360;2021-12-07@07:39:47 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;115424;208018;832072;37;8.382687618;2021-12-07@07:39:58 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;115200;208007;832028;39;8.853871091;2021-12-07@07:40:09 +v2.0.8-163-ge813cd67;multi;10;56234;;89496;27046;115720;208002;832008;39;8.843495349;2021-12-07@07:40:20 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88528;24052;96208;105;23.752201886;2021-12-07@07:40:54 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88760;24119;96476;103;23.401243122;2021-12-07@07:41:19 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88500;24097;96388;107;24.256920240;2021-12-07@07:41:46 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88764;24076;96304;107;24.270016785;2021-12-07@07:42:12 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88716;24084;96336;110;24.958964455;2021-12-07@07:42:39 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88480;24077;96308;107;24.241826574;2021-12-07@07:43:05 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88784;24091;96364;108;24.480520406;2021-12-07@07:43:31 +v2.0.8-163-ge813cd67;imex;10;56234;;89496;27046;88632;24079;96316;101;22.900378385;2021-12-07@07:43:56 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;26988;8345;33380;100;22.622742114;2021-12-07@07:44:31 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;26904;8347;33388;106;23.991849353;2021-12-07@07:44:56 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;27128;8372;33488;104;23.537595240;2021-12-07@07:45:22 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;27180;8370;33480;102;23.085544094;2021-12-07@07:45:47 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;27188;8338;33352;105;23.766262802;2021-12-07@07:46:13 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;26888;8316;33264;99;22.415114825;2021-12-07@07:46:37 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;27184;8353;33412;104;23.534318273;2021-12-07@07:47:03 +v2.0.8-163-ge813cd67;single;10;56234;;89496;27046;27276;8349;33396;101;22.862244797;2021-12-07@07:47:28 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;159200;219579;878316;35;7.880039793;2021-12-07@07:47:47 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;158340;219577;878308;39;8.847610285;2021-12-07@07:47:58 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;159336;219258;877032;39;8.847539392;2021-12-07@07:48:09 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;158620;219218;876872;39;8.842679114;2021-12-07@07:48:19 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;158760;219148;876592;39;8.850127546;2021-12-07@07:48:30 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;159036;219249;876996;39;8.859018830;2021-12-07@07:48:41 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;158632;219554;878216;40;9.080298570;2021-12-07@07:48:52 +v2.0.8-163-ge813cd67;mulimex;10;56234;;89496;27046;159272;219188;876752;40;9.066372626;2021-12-07@07:49:03 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;180020;355792;1423168;45;10.252088614;2021-12-07@07:49:27 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;179936;355063;1420252;50;11.404137946;2021-12-07@07:49:41 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;179824;355089;1420356;50;11.401650838;2021-12-07@07:49:55 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;179456;355171;1420684;50;11.398196284;2021-12-07@07:50:08 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;180000;355172;1420688;50;11.412623440;2021-12-07@07:50:22 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;180060;355055;1420220;52;11.851081639;2021-12-07@07:50:36 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;179836;355133;1420532;50;11.389664351;2021-12-07@07:50:49 +v2.0.8-163-ge813cd67;multi;18;56234;;97395;27048;179984;355132;1420528;50;11.394968365;2021-12-07@07:51:02 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118140;31722;126888;171;38.854328830;2021-12-07@07:51:55 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118220;31706;126824;179;40.659294292;2021-12-07@07:52:38 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118380;31713;126852;174;39.493578984;2021-12-07@07:53:19 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118156;31700;126800;160;36.332082074;2021-12-07@07:53:58 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118272;31718;126872;169;38.345543683;2021-12-07@07:54:39 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118144;31699;126796;199;45.193881902;2021-12-07@07:55:26 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118020;31693;126772;174;39.485973693;2021-12-07@07:56:07 +v2.0.8-163-ge813cd67;imex;18;56234;;97395;27048;118172;31714;126856;168;38.135301926;2021-12-07@07:56:47 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29640;9005;36020;170;38.655581121;2021-12-07@07:57:39 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29516;9008;36032;167;37.870671823;2021-12-07@07:58:19 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29532;9039;36156;171;38.780162600;2021-12-07@07:59:00 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29860;9012;36048;164;37.199583477;2021-12-07@07:59:39 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29868;8998;35992;166;37.707089883;2021-12-07@08:00:19 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29808;9031;36124;175;39.737176961;2021-12-07@08:01:01 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29848;9013;36052;169;38.326010963;2021-12-07@08:01:42 +v2.0.8-163-ge813cd67;single;18;56234;;97395;27048;29672;9030;36120;172;39.010369791;2021-12-07@08:02:23 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;248892;372880;1491520;51;11.584644703;2021-12-07@08:02:49 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;248688;372802;1491208;53;12.101671580;2021-12-07@08:03:03 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;248012;372784;1491136;53;12.091223498;2021-12-07@08:03:17 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;248136;372837;1491348;53;12.074483306;2021-12-07@08:03:31 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;248840;372806;1491224;54;12.316975303;2021-12-07@08:03:45 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;248908;372818;1491272;53;12.092308627;2021-12-07@08:03:59 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;247540;372767;1491068;54;12.316622179;2021-12-07@08:04:14 +v2.0.8-163-ge813cd67;mulimex;18;56234;;97395;27048;248624;372819;1491276;53;12.088563447;2021-12-07@08:04:28 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;288504;611952;2447808;76;17.420087629;2021-12-07@08:05:04 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;289068;611974;2447896;82;18.784716667;2021-12-07@08:05:26 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;288008;611915;2447660;82;18.757279283;2021-12-07@08:05:47 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;290032;612135;2448540;82;18.765192718;2021-12-07@08:06:07 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;289108;611925;2447700;83;19.006998407;2021-12-07@08:06:28 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;289168;612061;2448244;82;18.743200002;2021-12-07@08:06:49 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;289596;613163;2452652;82;18.786777734;2021-12-07@08:07:10 +v2.0.8-163-ge813cd67;multi;32;56234;;102105;27070;289352;611957;2447828;82;18.748140821;2021-12-07@08:07:31 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165428;60299;241196;298;67.814870127;2021-12-07@08:09:03 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165268;43900;175600;288;65.519232477;2021-12-07@08:10:10 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165328;43940;175760;285;64.825333771;2021-12-07@08:11:17 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165452;43944;175776;288;65.533378214;2021-12-07@08:12:24 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165556;43928;175712;281;63.928891601;2021-12-07@08:13:30 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165512;43906;175624;293;66.626425942;2021-12-07@08:14:39 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165300;43910;175640;292;66.444412953;2021-12-07@08:15:47 +v2.0.8-163-ge813cd67;imex;32;56234;;102105;27070;165232;43895;175580;279;63.615950535;2021-12-07@08:16:53 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32284;9704;38816;309;70.313180065;2021-12-07@08:18:25 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32480;9695;38780;286;65.090268007;2021-12-07@08:19:31 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32448;9709;38836;289;65.764155421;2021-12-07@08:20:39 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32572;9717;38868;298;67.857454848;2021-12-07@08:21:49 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32424;9707;38828;291;66.291291684;2021-12-07@08:22:57 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32332;9710;38840;290;65.987048128;2021-12-07@08:24:05 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32376;9711;38844;296;67.354610963;2021-12-07@08:25:14 +v2.0.8-162-ge38c4f44;single;32;56234;;102105;27070;32468;9705;38820;292;66.471219031;2021-12-07@08:26:23 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;399988;640722;2562888;80;18.315275334;2021-12-07@08:27:05 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;401032;640741;2562964;86;19.745304123;2021-12-07@08:27:27 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;400200;640797;2563188;86;19.709803601;2021-12-07@08:27:49 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;400900;640769;2563076;86;19.712992622;2021-12-07@08:28:10 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;401784;641902;2567608;86;19.730745754;2021-12-07@08:28:32 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;400576;640761;2563044;86;19.710070855;2021-12-07@08:28:53 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;400468;641860;2567440;86;19.725508931;2021-12-07@08:29:15 +v2.0.8-162-ge38c4f44;mulimex;32;56234;;102105;27070;400156;640855;2563420;86;19.685491649;2021-12-07@08:29:36 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;472304;1051438;4205752;128;29.336743114;2021-12-07@08:30:38 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;473268;1051443;4205772;138;31.614622262;2021-12-07@08:31:11 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;472792;1051883;4207532;138;31.621249639;2021-12-07@08:31:45 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;473360;1051686;4206744;138;31.650159319;2021-12-07@08:32:18 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;473688;1051629;4206516;140;32.054780508;2021-12-07@08:32:52 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;472152;1051817;4207268;138;31.622437078;2021-12-07@08:33:26 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;475672;1052130;4208520;138;31.608018473;2021-12-07@08:34:00 +v2.0.8-162-ge38c4f44;multi;56;56234;;104619;27074;472132;1053356;4213424;139;31.900961027;2021-12-07@08:34:34 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241460;63734;254936;517;118.660453291;2021-12-07@08:37:12 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241788;63721;254884;506;116.235448544;2021-12-07@08:39:10 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241892;63776;255104;506;116.242797190;2021-12-07@08:41:08 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241796;63694;254776;533;122.374036110;2021-12-07@08:43:13 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241788;63730;254920;514;118.073074039;2021-12-07@08:45:13 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241920;63719;254876;562;129.107215328;2021-12-07@08:47:24 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241844;63742;254968;511;117.165818186;2021-12-07@08:49:23 +v2.0.8-162-ge38c4f44;imex;56;56234;;104619;27074;241660;63742;254968;520;119.408303814;2021-12-07@08:51:25 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35340;10420;41680;536;123.008833467;2021-12-07@08:54:05 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35388;10398;41592;506;116.171004341;2021-12-07@08:56:03 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35412;10415;41660;513;117.736765623;2021-12-07@08:58:03 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35020;10396;41584;510;117.098314999;2021-12-07@09:00:02 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35344;10414;41656;508;116.602622562;2021-12-07@09:02:01 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35256;10407;41628;504;115.710585524;2021-12-07@09:03:59 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35440;10387;41548;515;118.263812415;2021-12-07@09:05:58 +v2.0.8-162-ge38c4f44;single;56;56234;;104619;27074;35212;10398;41592;510;117.028292461;2021-12-07@09:07:57 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;655972;1098948;4395792;142;32.684225987;2021-12-07@09:09:09 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;655612;1098977;4395908;146;33.577192836;2021-12-07@09:09:44 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;653968;1099129;4396516;147;33.828639319;2021-12-07@09:10:20 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;655868;1099055;4396220;146;33.566925798;2021-12-07@09:10:55 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;656528;1099089;4396356;149;34.258436479;2021-12-07@09:11:31 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;656372;1098978;4395912;148;34.008267792;2021-12-07@09:12:07 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;656588;1099016;4396064;147;33.823526772;2021-12-07@09:12:42 +v2.0.8-162-ge38c4f44;mulimex;56;56234;;104619;27074;656684;1098952;4395808;149;34.287582292;2021-12-07@09:13:18 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;811812;1857150;7428600;239;54.898294672;2021-12-07@09:15:05 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;811388;1857159;7428636;254;58.536591543;2021-12-07@09:16:06 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;815456;1858215;7432860;260;59.849598349;2021-12-07@09:17:07 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;815236;1858048;7432192;252;57.905388552;2021-12-07@09:18:06 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;813824;1858011;7432044;256;58.755406119;2021-12-07@09:19:07 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;816012;1858205;7432820;255;58.591220082;2021-12-07@09:20:07 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;815116;1858126;7432504;247;56.701459625;2021-12-07@09:21:05 +v2.0.8-162-ge38c4f44;multi;100;56234;;106396;27105;815352;1858086;7432344;255;58.561547021;2021-12-07@09:22:06 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380556;99721;398884;961;220.884274678;2021-12-07@09:26:53 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380972;99845;399380;945;217.191170823;2021-12-07@09:30:33 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380196;99712;398848;959;220.711569329;2021-12-07@09:34:16 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380916;99833;399332;926;213.098811443;2021-12-07@09:37:50 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380296;99674;398696;878;202.018146974;2021-12-07@09:41:15 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380496;99755;399020;883;203.084252840;2021-12-07@09:44:40 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380480;99710;398840;885;203.519919601;2021-12-07@09:48:05 +v2.0.8-162-ge38c4f44;imex;100;56234;;106396;27105;380476;99770;399080;884;203.587421629;2021-12-07@09:51:31 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39076;11373;45492;953;220.067987661;2021-12-07@09:56:16 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39044;11381;45524;934;216.386786625;2021-12-07@09:59:55 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39272;11404;45616;988;228.136814978;2021-12-07@10:03:45 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39180;11403;45612;936;216.213162155;2021-12-07@10:07:24 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39288;11402;45608;973;225.237349413;2021-12-07@10:11:11 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39356;11382;45528;1099;254.545626069;2021-12-07@10:15:28 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39172;11398;45592;987;228.431737878;2021-12-07@10:19:18 +v2.0.8-162-ge38c4f44;single;100;56234;;106396;27105;39092;11382;45528;1012;234.441023614;2021-12-07@10:23:15 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1126108;1939306;7757224;263;60.607039104;2021-12-07@10:25:20 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1125028;1939386;7757544;271;62.383007883;2021-12-07@10:26:24 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1127300;1939684;7758736;270;62.210629062;2021-12-07@10:27:29 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1126772;1939524;7758096;269;61.916333402;2021-12-07@10:28:33 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1126724;1939454;7757816;273;62.782615139;2021-12-07@10:29:38 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1126352;1939422;7757688;269;62.016610175;2021-12-07@10:30:43 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1127524;1939638;7758552;270;62.140153398;2021-12-07@10:31:47 +v2.0.8-162-ge38c4f44;mulimex;100;56234;;106396;27105;1125708;1939467;7757868;271;62.329646487;2021-12-07@10:32:52 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1412368;2434437;9737748;452;105.074120590;2021-12-07@10:36:13 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1415832;2435204;9740816;467;108.303104105;2021-12-07@10:38:03 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1408988;2433879;9735516;463;107.237797066;2021-12-07@10:39:53 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1407728;2433286;9733144;464;107.756021722;2021-12-07@10:41:43 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1415320;2435237;9740948;462;106.905256557;2021-12-07@10:43:32 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1415976;2435191;9740764;451;104.395056517;2021-12-07@10:45:18 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1411576;2434361;9737444;469;108.176752682;2021-12-07@10:47:09 +v2.0.8-162-ge38c4f44;multi;178;56234;;107469;27173;1410640;2433821;9735284;448;103.802910023;2021-12-07@10:48:55 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;626100;163488;653952;1741;401.900251645;2021-12-07@10:57:30 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;627192;163714;654856;1668;386.070966366;2021-12-07@11:03:59 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;626188;163543;654172;1717;396.937292355;2021-12-07@11:10:37 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;627636;163849;655396;1680;388.250700679;2021-12-07@11:17:07 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;626516;163584;654336;1732;400.114119250;2021-12-07@11:23:50 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;624596;163102;652408;1657;382.726705351;2021-12-07@11:30:14 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;625544;163339;653356;1691;390.508581231;2021-12-07@11:36:46 +v2.0.8-162-ge38c4f44;imex;178;56234;;107469;27173;625824;163424;653696;1764;406.973139329;2021-12-07@11:43:35 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;44764;12815;51260;1841;425.529791376;2021-12-07@11:52:32 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;45216;12848;51392;1623;375.002273344;2021-12-07@11:58:48 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;45104;12831;51324;1764;408.716158792;2021-12-07@12:05:39 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;45132;12832;51328;1731;400.106162572;2021-12-07@12:12:20 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;44740;12809;51236;1703;393.154512921;2021-12-07@12:18:55 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;44968;12843;51372;1622;374.993661126;2021-12-07@12:25:12 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;44808;12812;51248;1679;387.739081506;2021-12-07@12:31:41 +v2.0.8-162-ge38c4f44;single;178;56234;;107469;27173;45140;12834;51336;1608;372.600897973;2021-12-07@12:37:55 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1958068;2577134;10308536;464;107.705177588;2021-12-07@12:41:37 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1966572;2578759;10315036;468;107.885290589;2021-12-07@12:43:41 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1962524;2578133;10312532;462;107.137076114;2021-12-07@12:45:45 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1962600;2578118;10312472;462;107.717835010;2021-12-07@12:47:49 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1964796;2578613;10314452;471;108.260140050;2021-12-07@12:49:54 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1963108;2578321;10313284;468;108.541680736;2021-12-07@12:51:59 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1967808;2578469;10313876;468;108.124198194;2021-12-07@12:54:04 +v2.0.8-163-g542f2455;mulimex;178;56234;;107469;27173;1960704;2577558;10310232;461;107.053603216;2021-12-07@12:56:08 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2467048;2700998;10803992;797;186.242743507;2021-12-07@13:02:31 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2470612;2702008;10808032;807;188.098883875;2021-12-07@13:05:41 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2472368;2702141;10808564;805;187.476945648;2021-12-07@13:08:53 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2470252;2701710;10806840;870;199.900663013;2021-12-07@13:12:25 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2460868;2700067;10800268;1481;352.265841873;2021-12-07@13:22:30 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2477616;2703931;10815724;865;197.814149617;2021-12-07@13:25:50 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2474776;2703214;10812856;854;196.335411369;2021-12-07@13:29:10 +v2.0.8-163-g542f2455;multi;316;56234;;108134;27288;2477244;2703571;10814284;854;196.148596118;2021-12-07@13:32:30 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1061240;276415;1105660;2920;675.382400523;2021-12-07@13:46:56 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1062464;276715;1106860;2961;687.321316322;2021-12-07@13:58:40 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1059872;276093;1104372;2953;683.802078276;2021-12-07@14:10:21 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1060332;276189;1104756;2977;692.262919920;2021-12-07@14:21:55 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1059680;275992;1103968;2775;644.806413846;2021-12-07@14:32:41 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1063264;276848;1107392;2850;663.890224686;2021-12-07@14:43:47 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1062584;276719;1106876;2987;700.474717231;2021-12-07@14:55:29 +v2.0.8-163-g542f2455;imex;316;56234;;108134;27288;1059304;275920;1103680;3107;721.580970601;2021-12-07@15:07:32 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;53612;14987;59948;3183;738.604693202;2021-12-07@15:22:59 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;53816;15048;60192;3110;722.117562203;2021-12-07@15:35:03 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;53888;15090;60360;3088;715.782867904;2021-12-07@15:47:01 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;53684;14999;59996;3182;739.619192441;2021-12-07@15:59:22 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;52940;14884;59536;3155;732.560875442;2021-12-07@16:11:36 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;53484;14970;59880;3177;734.713577511;2021-12-07@16:23:53 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;53624;14993;59972;3197;742.265836951;2021-12-07@16:36:17 +v2.0.8-164-gf9e098c9;single;316;56234;;108134;27288;53708;14985;59940;3121;724.805608268;2021-12-07@16:48:24 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3441604;2954812;11819248;842;196.066017728;2021-12-07@16:55:02 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3455192;2958203;11832812;858;198.019355689;2021-12-07@16:58:23 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3449340;2955449;11821796;848;198.250545793;2021-12-07@17:01:44 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3441232;2954377;11817508;851;198.632521435;2021-12-07@17:05:05 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3436228;2953826;11815304;843;196.740888733;2021-12-07@17:08:24 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3441636;2954579;11818316;858;198.089961540;2021-12-07@17:11:45 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3455548;2957054;11828216;882;202.311464004;2021-12-07@17:15:10 +v2.0.8-164-gf9e098c9;mulimex;316;56234;;108134;27288;3441240;2954245;11816980;854;197.640247363;2021-12-07@17:18:31 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170308;220940;883760;43;10.211327539;2021-12-07@17:20:05 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170292;220949;883796;48;11.308742392;2021-12-07@17:20:18 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170792;220967;883868;49;11.507060210;2021-12-07@17:20:32 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170224;220977;883908;48;11.295528774;2021-12-07@17:20:45 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170200;220955;883820;48;11.298329429;2021-12-07@17:20:59 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170944;220957;883828;49;11.578842286;2021-12-07@17:21:12 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170532;220930;883720;49;11.571323611;2021-12-07@17:21:26 +v2.0.8-164-gf9e098c9;multi;10;100000;;126751;52385;170440;221038;884152;48;11.306314950;2021-12-07@17:21:40 +v2.0.8;multi;10;10000;;12925;6245;25444;7867;31468;34;7.380314168;2021-12-07@17:23:59 +v2.0.8;multi;10;10000;;12925;6245;25580;7913;31652;36;7.824798988;2021-12-07@17:24:09 +v2.0.8;multi;10;10000;;12925;6245;25632;7929;31716;37;8.046475550;2021-12-07@17:24:19 +v2.0.8;multi;10;10000;;12925;6245;25648;7940;31760;36;7.832148685;2021-12-07@17:24:30 +v2.0.8;multi;10;10000;;12925;6245;25468;7925;31700;36;7.822049309;2021-12-07@17:24:39 +v2.0.8;multi;10;10000;;12925;6245;25604;7916;31664;36;7.827260141;2021-12-07@17:24:49 +v2.0.8;multi;10;10000;;12925;6245;25480;7923;31692;37;8.038343117;2021-12-07@17:24:59 +v2.0.8;multi;10;10000;;12925;6245;25524;7921;31684;36;7.826981949;2021-12-07@17:25:09 +v2.0.8;imex;10;10000;;12925;6245;21536;6929;27716;26;5.636593475;2021-12-07@17:25:24 +v2.0.8;imex;10;10000;;12925;6245;21332;6929;27716;28;6.082880960;2021-12-07@17:25:32 +v2.0.8;imex;10;10000;;12925;6245;21516;6949;27796;28;6.087306502;2021-12-07@17:25:40 +v2.0.8;imex;10;10000;;12925;6245;21476;6948;27792;27;5.862203072;2021-12-07@17:25:48 +v2.0.8;imex;10;10000;;12925;6245;21844;6948;27792;28;6.077918495;2021-12-07@17:25:56 +v2.0.8;imex;10;10000;;12925;6245;21688;6929;27716;28;6.075490031;2021-12-07@17:26:04 +v2.0.8;imex;10;10000;;12925;6245;21416;6948;27792;28;6.079226967;2021-12-07@17:26:12 +v2.0.8;imex;10;10000;;12925;6245;21528;6942;27768;27;5.861203295;2021-12-07@17:26:20 +v2.0.8;single;10;10000;;12925;6245;;8332;3571;14284;27;5.834713449;2021-12-07@17:26:35 +v2.0.8;single;10;10000;;12925;6245;;8268;3602;14408;28;6.076678459;2021-12-07@17:26:43 +v2.0.8;single;10;10000;;12925;6245;;8280;3606;14424;29;6.295831071;2021-12-07@17:26:51 +v2.0.8;single;10;10000;;12925;6245;;8248;3610;14440;28;6.078340521;2021-12-07@17:26:59 +v2.0.8;single;10;10000;;12925;6245;;8488;3603;14412;29;6.298977355;2021-12-07@17:27:07 +v2.0.8;single;10;10000;;12925;6245;;8608;3627;14508;28;6.077737605;2021-12-07@17:27:15 +v2.0.8;single;10;10000;;12925;6245;;8172;3616;14464;28;6.082737448;2021-12-07@17:27:24 +v2.0.8;single;10;10000;;12925;6245;;8080;3611;14444;28;6.076552437;2021-12-07@17:27:32 +v2.0.8;mulimex;10;10000;;12925;6245;34896;10274;41096;34;7.375614892;2021-12-07@17:27:48 +v2.0.8;mulimex;10;10000;;12925;6245;35116;10319;41276;36;7.823989090;2021-12-07@17:27:58 +v2.0.8;mulimex;10;10000;;12925;6245;34828;10280;41120;36;7.824985758;2021-12-07@17:28:07 +v2.0.8;mulimex;10;10000;;12925;6245;35012;10264;41056;36;7.826464961;2021-12-07@17:28:17 +v2.0.8;mulimex;10;10000;;12925;6245;35172;10318;41272;37;8.047223876;2021-12-07@17:28:27 +v2.0.8;mulimex;10;10000;;12925;6245;34900;10281;41124;36;7.827521230;2021-12-07@17:28:38 +v2.0.8;mulimex;10;10000;;12925;6245;35060;10311;41244;36;7.829890366;2021-12-07@17:28:48 +v2.0.8;mulimex;10;10000;;12925;6245;34824;10284;41136;37;8.047637860;2021-12-07@17:28:58 +v2.0.8;multi;18;10000;;15314;6305;40776;11686;46744;62;13.504565080;2021-12-07@17:29:23 +v2.0.8;multi;18;10000;;15314;6305;40652;11669;46676;68;14.824318024;2021-12-07@17:29:40 +v2.0.8;multi;18;10000;;15314;6305;40536;11675;46700;69;15.039503816;2021-12-07@17:29:57 +v2.0.8;multi;18;10000;;15314;6305;40684;11651;46604;69;15.031619656;2021-12-07@17:30:14 +v2.0.8;multi;18;10000;;15314;6305;40720;11666;46664;68;14.813683180;2021-12-07@17:30:31 +v2.0.8;multi;18;10000;;15314;6305;40604;11669;46676;68;14.819970770;2021-12-07@17:30:48 +v2.0.8;multi;18;10000;;15314;6305;40572;11705;46820;68;14.827298019;2021-12-07@17:31:05 +v2.0.8;multi;18;10000;;15314;6305;40792;11671;46684;68;14.818596799;2021-12-07@17:31:21 +v2.0.8;imex;18;10000;;15314;6305;30360;9066;36264;41;8.899544407;2021-12-07@17:31:44 +v2.0.8;imex;18;10000;;15314;6305;29852;9072;36288;46;10.014288271;2021-12-07@17:31:56 +v2.0.8;imex;18;10000;;15314;6305;29916;9050;36200;45;9.795407102;2021-12-07@17:32:08 +v2.0.8;imex;18;10000;;15314;6305;29844;9045;36180;53;11.542368727;2021-12-07@17:32:22 +v2.0.8;imex;18;10000;;15314;6305;29852;9064;36256;44;9.577087526;2021-12-07@17:32:33 +v2.0.8;imex;18;10000;;15314;6305;29820;9076;36304;45;9.798265252;2021-12-07@17:32:45 +v2.0.8;imex;18;10000;;15314;6305;30148;9060;36240;44;9.575457314;2021-12-07@17:32:57 +v2.0.8;imex;18;10000;;15314;6305;30036;9066;36264;44;9.586265262;2021-12-07@17:33:08 +v2.0.8;single;18;10000;;15314;6305;;9212;3896;15584;45;9.787583814;2021-12-07@17:33:31 +v2.0.8;single;18;10000;;15314;6305;;9520;3946;15784;45;9.805247929;2021-12-07@17:33:43 +v2.0.8;single;18;10000;;15314;6305;;9568;3949;15796;45;9.798459050;2021-12-07@17:33:55 +v2.0.8;single;18;10000;;15314;6305;;9532;3955;15820;45;9.796275112;2021-12-07@17:34:06 +v2.0.8;single;18;10000;;15314;6305;;9588;3952;15808;45;9.796876411;2021-12-07@17:34:18 +v2.0.8;single;18;10000;;15314;6305;;9580;3929;15716;47;10.237484116;2021-12-07@17:34:30 +v2.0.8;single;18;10000;;15314;6305;;9796;3949;15796;46;10.021432268;2021-12-07@17:34:42 +v2.0.8;single;18;10000;;15314;6305;;9576;3965;15860;47;10.236391708;2021-12-07@17:34:55 +v2.0.8;mulimex;18;10000;;15314;6305;56628;15661;62644;62;13.504603988;2021-12-07@17:35:22 +v2.0.8;mulimex;18;10000;;15314;6305;56464;15687;62748;69;15.044574303;2021-12-07@17:35:38 +v2.0.8;mulimex;18;10000;;15314;6305;56692;15704;62816;69;15.045876645;2021-12-07@17:35:55 +v2.0.8;mulimex;18;10000;;15314;6305;56540;15693;62772;68;14.828177143;2021-12-07@17:36:12 +v2.0.8;mulimex;18;10000;;15314;6305;56720;15711;62844;69;15.049092830;2021-12-07@17:36:29 +v2.0.8;mulimex;18;10000;;15314;6305;56704;15682;62728;68;14.829925213;2021-12-07@17:36:46 +v2.0.8;mulimex;18;10000;;15314;6305;56632;15685;62740;69;15.041653552;2021-12-07@17:37:03 +v2.0.8;mulimex;18;10000;;15314;6305;56640;15690;62760;69;15.043299739;2021-12-07@17:37:20 +v2.0.8;multi;32;10000;;16892;6334;65688;18025;72100;120;26.180747823;2021-12-07@17:38:04 +v2.0.8;multi;32;10000;;16892;6334;65708;18023;72092;126;27.504178252;2021-12-07@17:38:33 +v2.0.8;multi;32;10000;;16892;6334;66120;18032;72128;127;27.724581477;2021-12-07@17:39:03 +v2.0.8;multi;32;10000;;16892;6334;65772;18016;72064;128;27.945166100;2021-12-07@17:39:33 +v2.0.8;multi;32;10000;;16892;6334;65960;18025;72100;127;27.691722123;2021-12-07@17:40:02 +v2.0.8;multi;32;10000;;16892;6334;65896;18022;72088;127;27.724702119;2021-12-07@17:40:32 +v2.0.8;multi;32;10000;;16892;6334;65876;18020;72080;127;27.721962977;2021-12-07@17:41:02 +v2.0.8;multi;32;10000;;16892;6334;65960;17997;71988;132;28.798228462;2021-12-07@17:41:32 +v2.0.8;imex;32;10000;;16892;6334;42900;12250;49000;71;15.460228949;2021-12-07@17:42:10 +v2.0.8;imex;32;10000;;16892;6334;42992;12277;49108;75;16.343240470;2021-12-07@17:42:28 +v2.0.8;imex;32;10000;;16892;6334;42632;12259;49036;73;15.915448156;2021-12-07@17:42:45 +v2.0.8;imex;32;10000;;16892;6334;42736;12264;49056;79;17.229714688;2021-12-07@17:43:05 +v2.0.8;imex;32;10000;;16892;6334;42892;12289;49156;74;16.132911122;2021-12-07@17:43:23 +v2.0.8;imex;32;10000;;16892;6334;42924;12270;49080;74;16.146408568;2021-12-07@17:43:41 +v2.0.8;imex;32;10000;;16892;6334;42740;12269;49076;73;15.900375136;2021-12-07@17:43:59 +v2.0.8;imex;32;10000;;16892;6334;42796;12260;49040;77;16.775149691;2021-12-07@17:44:18 +v2.0.8;single;32;10000;;16892;6334;10964;4276;17104;74;16.109957913;2021-12-07@17:44:56 +v2.0.8;single;32;10000;;16892;6334;11056;4247;16988;80;17.443506335;2021-12-07@17:45:15 +v2.0.8;single;32;10000;;16892;6334;10828;4272;17088;78;17.011778950;2021-12-07@17:45:35 +v2.0.8;single;32;10000;;16892;6334;10776;4262;17048;75;16.352763608;2021-12-07@17:45:53 +v2.0.8;single;32;10000;;16892;6334;10912;4264;17056;75;16.360266090;2021-12-07@17:46:11 +v2.0.8;single;32;10000;;16892;6334;11036;4268;17072;76;16.572461117;2021-12-07@17:46:30 +v2.0.8;single;32;10000;;16892;6334;10916;4253;17012;76;16.576309170;2021-12-07@17:46:49 +v2.0.8;single;32;10000;;16892;6334;10956;4292;17168;76;16.567794493;2021-12-07@17:47:08 +v2.0.8;mulimex;32;10000;;16892;6334;92792;24750;99000;122;26.624865322;2021-12-07@17:47:56 +v2.0.8;mulimex;32;10000;;16892;6334;92832;24742;98968;127;27.752408830;2021-12-07@17:48:26 +v2.0.8;mulimex;32;10000;;16892;6334;92836;24742;98968;127;27.669849538;2021-12-07@17:48:56 +v2.0.8;mulimex;32;10000;;16892;6334;92840;24769;99076;127;27.736230391;2021-12-07@17:49:26 +v2.0.8;mulimex;32;10000;;16892;6334;93364;24845;99380;127;27.730842065;2021-12-07@17:49:55 +v2.0.8;mulimex;32;10000;;16892;6334;92756;24745;98980;126;27.500146233;2021-12-07@17:50:25 +v2.0.8;mulimex;32;10000;;16892;6334;92752;24725;98900;127;27.748070578;2021-12-07@17:50:55 +v2.0.8;mulimex;32;10000;;16892;6334;92896;24759;99036;127;27.751678200;2021-12-07@17:51:24 +v2.0.8;multi;56;10000;;17919;6345;108424;28620;114480;222;48.428642973;2021-12-07@17:52:43 +v2.0.8;multi;56;10000;;17919;6345;115212;30384;121536;230;50.093778564;2021-12-07@17:53:35 +v2.0.8;multi;56;10000;;17919;6345;116512;30660;122640;229;49.944954914;2021-12-07@17:54:26 +v2.0.8;multi;56;10000;;17919;6345;118292;31090;124360;230;50.010190612;2021-12-07@17:55:20 +v2.0.8;multi;56;10000;;17919;6345;115404;30413;121652;229;50.077709971;2021-12-07@17:56:13 +v2.0.8;multi;56;10000;;17919;6345;122112;32020;128080;231;50.186404896;2021-12-07@17:57:05 +v2.0.8;multi;56;10000;;17919;6345;111028;29302;117208;229;49.958732992;2021-12-07@17:57:57 +v2.0.8;multi;56;10000;;17919;6345;116172;30616;122464;230;50.011137354;2021-12-07@17:58:49 +v2.0.8;imex;56;10000;;17919;6345;63352;17388;69552;123;26.857051411;2021-12-07@17:59:53 +v2.0.8;imex;56;10000;;17919;6345;63264;17387;69548;125;27.310443534;2021-12-07@18:00:22 +v2.0.8;imex;56;10000;;17919;6345;63308;17368;69472;124;27.085014487;2021-12-07@18:00:52 +v2.0.8;imex;56;10000;;17919;6345;63232;17376;69504;124;27.078020642;2021-12-07@18:01:20 +v2.0.8;imex;56;10000;;17919;6345;63080;17351;69404;124;27.093878431;2021-12-07@18:01:50 +v2.0.8;imex;56;10000;;17919;6345;63208;17360;69440;123;26.864556296;2021-12-07@18:02:19 +v2.0.8;imex;56;10000;;17919;6345;63272;17378;69512;124;27.080865302;2021-12-07@18:02:48 +v2.0.8;imex;56;10000;;17919;6345;63292;17401;69604;127;27.746670444;2021-12-07@18:03:17 +v2.0.8;single;56;10000;;17919;6345;12728;4720;18880;127;27.688249077;2021-12-07@18:04:23 +v2.0.8;single;56;10000;;17919;6345;12528;4671;18684;129;28.168592390;2021-12-07@18:04:53 +v2.0.8;single;56;10000;;17919;6345;12460;4667;18668;132;28.816726827;2021-12-07@18:05:24 +v2.0.8;single;56;10000;;17919;6345;13100;4730;18920;131;28.612050555;2021-12-07@18:05:55 +v2.0.8;single;56;10000;;17919;6345;12676;4734;18936;129;28.179133677;2021-12-07@18:06:25 +v2.0.8;single;56;10000;;17919;6345;12628;4677;18708;131;28.600839303;2021-12-07@18:06:56 +v2.0.8;single;56;10000;;17919;6345;12860;4724;18896;130;28.390482520;2021-12-07@18:07:27 +v2.0.8;single;56;10000;;17919;6345;12608;4682;18728;130;28.383924062;2021-12-07@18:07:58 +v2.0.8;mulimex;56;10000;;17919;6345;153720;39990;159960;222;48.558125010;2021-12-07@18:09:24 +v2.0.8;mulimex;56;10000;;17919;6345;173452;44911;179644;229;49.819159265;2021-12-07@18:10:17 +v2.0.8;mulimex;56;10000;;17919;6345;161572;41947;167788;229;50.037428440;2021-12-07@18:11:09 +v2.0.8;mulimex;56;10000;;17919;6345;162396;42118;168472;228;50.001794469;2021-12-07@18:12:01 +v2.0.8;mulimex;56;10000;;17919;6345;164284;42560;170240;229;49.904116030;2021-12-07@18:12:53 +v2.0.8;mulimex;56;10000;;17919;6345;165128;42803;171212;230;50.055860076;2021-12-07@18:13:45 +v2.0.8;mulimex;56;10000;;17919;6345;162508;42198;168792;229;49.858875896;2021-12-07@18:14:37 +v2.0.8;mulimex;56;10000;;17919;6345;163648;42476;169904;229;49.865987762;2021-12-07@18:15:29 +v2.0.8;multi;100;10000;;18563;6381;185588;47943;191772;407;89.336977668;2021-12-07@18:17:51 +v2.0.8;multi;100;10000;;18563;6381;185588;47911;191644;413;90.664938942;2021-12-07@18:19:24 +v2.0.8;multi;100;10000;;18563;6381;185424;47920;191680;408;89.688455243;2021-12-07@18:21:10 +v2.0.8;multi;100;10000;;18563;6381;185604;47932;191728;412;90.489482553;2021-12-07@18:22:44 +v2.0.8;multi;100;10000;;18563;6381;185740;47923;191692;412;90.682941706;2021-12-07@18:24:17 +v2.0.8;multi;100;10000;;18563;6381;185560;47922;191688;412;90.483333839;2021-12-07@18:25:50 +v2.0.8;multi;100;10000;;18563;6381;185712;47937;191748;413;90.720244470;2021-12-07@18:27:23 +v2.0.8;multi;100;10000;;18563;6381;185464;47917;191668;413;90.673136813;2021-12-07@18:28:56 +v2.0.8;imex;100;10000;;18563;6381;98996;26295;105180;228;49.772746849;2021-12-07@18:30:50 +v2.0.8;imex;100;10000;;18563;6381;99248;26325;105300;220;48.086806626;2021-12-07@18:31:41 +v2.0.8;imex;100;10000;;18563;6381;99472;26428;105712;225;49.166994106;2021-12-07@18:32:32 +v2.0.8;imex;100;10000;;18563;6381;99112;26308;105232;225;49.193044838;2021-12-07@18:33:24 +v2.0.8;imex;100;10000;;18563;6381;99320;26431;105724;218;48.375874157;2021-12-07@18:34:14 +v2.0.8;imex;100;10000;;18563;6381;99048;26358;105432;208;48.717563830;2021-12-07@18:35:06 +v2.0.8;imex;100;10000;;18563;6381;99212;26333;105332;220;48.239671818;2021-12-07@18:35:56 +v2.0.8;imex;100;10000;;18563;6381;98996;26308;105232;224;49.011486298;2021-12-07@18:36:47 +v2.0.8;single;100;10000;;18563;6381;14668;5186;20744;236;51.517662396;2021-12-07@18:38:41 +v2.0.8;single;100;10000;;18563;6381;14860;5231;20924;230;50.230503198;2021-12-07@18:39:33 +v2.0.8;single;100;10000;;18563;6381;14720;5196;20784;232;50.693102460;2021-12-07@18:40:26 +v2.0.8;single;100;10000;;18563;6381;14436;5152;20608;232;50.672487282;2021-12-07@18:41:18 +v2.0.8;single;100;10000;;18563;6381;14724;5223;20892;230;50.228859148;2021-12-07@18:42:11 +v2.0.8;single;100;10000;;18563;6381;14828;5214;20856;227;49.603268658;2021-12-07@18:43:02 +v2.0.8;single;100;10000;;18563;6381;14240;5155;20620;231;50.441219511;2021-12-07@18:43:55 +v2.0.8;single;100;10000;;18563;6381;14660;5165;20660;231;50.458875466;2021-12-07@18:44:47 +v2.0.8;mulimex;100;10000;;18563;6381;264512;67636;270544;408;89.573877427;2021-12-07@18:47:19 +v2.0.8;mulimex;100;10000;;18563;6381;264788;67650;270600;413;90.670580332;2021-12-07@18:48:52 +v2.0.8;mulimex;100;10000;;18563;6381;264352;67649;270596;415;91.098431472;2021-12-07@18:50:25 +v2.0.8;mulimex;100;10000;;18563;6381;264312;67631;270524;414;90.901171045;2021-12-07@18:51:58 +v2.0.8;mulimex;100;10000;;18563;6381;264372;67615;270460;415;91.116916145;2021-12-07@18:53:33 +v2.0.8;mulimex;100;10000;;18563;6381;264688;67689;270756;415;91.105156060;2021-12-07@18:55:07 +v2.0.8;mulimex;100;10000;;18563;6381;264284;67647;270588;415;91.104647172;2021-12-07@18:56:42 +v2.0.8;mulimex;100;10000;;18563;6381;264340;67658;270632;415;91.050613393;2021-12-07@18:58:15 +v2.0.8;multi;178;10000;;19033;6531;323688;82470;329880;737;162.869679370;2021-12-07@19:02:32 +v2.0.8;multi;178;10000;;19033;6531;323716;82499;329996;749;165.194929662;2021-12-07@19:05:19 +v2.0.8;multi;178;10000;;19033;6531;323924;82481;329924;747;164.678245748;2021-12-07@19:08:07 +v2.0.8;multi;178;10000;;19033;6531;323808;82478;329912;746;164.419681992;2021-12-07@19:10:54 +v2.0.8;multi;178;10000;;19033;6531;323624;82452;329808;747;164.624263970;2021-12-07@19:13:41 +v2.0.8;multi;178;10000;;19033;6531;323784;82445;329780;759;167.370843808;2021-12-07@19:16:31 +v2.0.8;multi;178;10000;;19033;6531;323920;82453;329812;747;164.679852071;2021-12-07@19:19:19 +v2.0.8;multi;178;10000;;19033;6531;323780;82453;329812;747;164.670572918;2021-12-07@19:22:06 +v2.0.8;imex;178;10000;;19033;6531;165764;43006;172024;440;96.468907431;2021-12-07@19:25:39 +v2.0.8;imex;178;10000;;19033;6531;166300;43086;172344;428;93.906688190;2021-12-07@19:27:15 +v2.0.8;imex;178;10000;;19033;6531;166080;43045;172180;422;92.639551274;2021-12-07@19:28:51 +v2.0.8;imex;178;10000;;19033;6531;165964;43079;172316;420;92.106320314;2021-12-07@19:30:25 +v2.0.8;imex;178;10000;;19033;6531;165980;43025;172100;419;92.364277190;2021-12-07@19:31:59 +v2.0.8;imex;178;10000;;19033;6531;165724;43023;172092;431;94.616677919;2021-12-07@19:33:36 +v2.0.8;imex;178;10000;;19033;6531;165772;43017;172068;426;93.487271096;2021-12-07@19:35:12 +v2.0.8;imex;178;10000;;19033;6531;166000;43072;172288;424;93.016330089;2021-12-07@19:36:47 +v2.0.8;single;178;10000;;19033;6531;18628;6166;24664;457;100.596303996;2021-12-07@19:40:12 +v2.0.8;single;178;10000;;19033;6531;19388;6353;25412;442;97.137298157;2021-12-07@19:41:52 +v2.0.8;single;178;10000;;19033;6531;19456;6384;25536;441;96.715050045;2021-12-07@19:43:30 +v2.0.8;single;178;10000;;19033;6531;19164;6313;25252;439;96.296256213;2021-12-07@19:45:09 +v2.0.8;single;178;10000;;19033;6531;19184;6364;25456;446;97.855675289;2021-12-07@19:46:49 +v2.0.8;single;178;10000;;19033;6531;19396;6347;25388;450;98.684735012;2021-12-07@19:48:29 +v2.0.8;single;178;10000;;19033;6531;19348;6328;25312;450;98.714682791;2021-12-07@19:50:10 +v2.0.8;single;178;10000;;19033;6531;19448;6444;25776;447;98.041881310;2021-12-07@19:51:50 +v2.0.8;mulimex;178;10000;;19033;6531;465480;117905;471620;741;163.492284571;2021-12-07@19:56:20 +v2.0.8;mulimex;178;10000;;19033;6531;465412;117893;471572;752;165.821125164;2021-12-07@19:59:08 +v2.0.8;mulimex;178;10000;;19033;6531;465540;117933;471732;749;165.118394046;2021-12-07@20:01:56 +v2.0.8;mulimex;178;10000;;19033;6531;485248;122800;491200;753;165.268371616;2021-12-07@20:04:43 +v2.0.8;mulimex;178;10000;;19033;6531;465580;117927;471708;749;165.185756859;2021-12-07@20:07:31 +v2.0.8;mulimex;178;10000;;19033;6531;465668;117924;471696;748;164.826062978;2021-12-07@20:10:18 +v2.0.8;mulimex;178;10000;;19033;6531;465340;117874;471496;749;165.087836049;2021-12-07@20:13:06 +v2.0.8;mulimex;178;10000;;19033;6531;465368;117911;471644;748;164.828776379;2021-12-07@20:15:53 +v2.0.8;multi;316;10000;;19553;6638;576412;145619;582476;1364;301.437332047;2021-12-07@20:23:50 +v2.0.8;multi;316;10000;;19553;6638;576120;145565;582260;1391;308.026027526;2021-12-07@20:29:01 +v2.0.8;multi;316;10000;;19553;6638;576256;145604;582416;1381;305.780135037;2021-12-07@20:34:10 +v2.0.8;multi;316;10000;;19553;6638;576476;145702;582808;1392;308.944966159;2021-12-07@20:39:22 +v2.0.8;multi;316;10000;;19553;6638;576724;145735;582940;1391;307.443356777;2021-12-07@20:44:32 +v2.0.8;multi;316;10000;;19553;6638;575996;145566;582264;1390;307.199272424;2021-12-07@20:49:43 +v2.0.8;multi;316;10000;;19553;6638;576796;145727;582908;1389;308.110883414;2021-12-07@20:54:54 +v2.0.8;multi;316;10000;;19553;6638;576512;145707;582828;1381;306.679623567;2021-12-07@21:00:04 +v2.0.8;imex;316;10000;;19553;6638;284180;72597;290388;1162;256.999570973;2021-12-07@21:07:58 +v2.0.8;imex;316;10000;;19553;6638;284108;72602;290408;837;185.575683561;2021-12-07@21:11:05 +v2.0.8;imex;316;10000;;19553;6638;284504;72667;290668;853;188.174848726;2021-12-07@21:14:16 +v2.0.8;imex;316;10000;;19553;6638;284452;72603;290412;858;189.552162419;2021-12-07@21:17:28 +v2.0.8;imex;316;10000;;19553;6638;283988;72571;290284;855;188.332651908;2021-12-07@21:20:38 +v2.0.8;imex;316;10000;;19553;6638;284152;72613;290452;865;190.454185115;2021-12-07@21:23:51 +v2.0.8;imex;316;10000;;19553;6638;284128;72581;290324;859;190.103712198;2021-12-07@21:27:04 +v2.0.8;imex;316;10000;;19553;6638;284100;72595;290380;843;186.871446146;2021-12-07@21:30:14 +v2.0.8;single;316;10000;;19553;6638;29668;8937;35748;1209;268.291747662;2021-12-07@21:38:13 +v2.0.8;single;316;10000;;19553;6638;29456;8870;35480;1157;256.729359927;2021-12-07@21:43:14 +v2.0.8;single;316;10000;;19553;6638;29436;8865;35460;897;198.441831717;2021-12-07@21:46:50 +v2.0.8;single;316;10000;;19553;6638;29372;8809;35236;1186;263.235567951;2021-12-07@21:51:55 +v2.0.8;single;316;10000;;19553;6638;29676;8923;35692;1138;251.997587950;2021-12-07@21:57:04 +v2.0.8;single;316;10000;;19553;6638;29676;8910;35640;1196;264.643353842;2021-12-07@22:03:48 +v2.0.8;single;316;10000;;19553;6638;28636;8658;34632;871;192.101982736;2021-12-07@22:07:02 +v2.0.8;single;316;10000;;19553;6638;28784;8702;34808;862;190.054995406;2021-12-07@22:10:14 +v2.0.8;mulimex;316;10000;;19553;6638;829616;208913;835652;1379;305.101826067;2021-12-07@22:18:30 +v2.0.8;mulimex;316;10000;;19553;6638;829692;208922;835688;1391;308.145536069;2021-12-07@22:23:43 +v2.0.8;mulimex;316;10000;;19553;6638;829244;208802;835208;1388;307.228636267;2021-12-07@22:28:53 +v2.0.8;mulimex;316;10000;;19553;6638;829120;208850;835400;1387;306.742807045;2021-12-07@22:34:03 +v2.0.8;mulimex;316;10000;;19553;6638;829216;208817;835268;1392;308.302072501;2021-12-07@22:39:15 +v2.0.8;mulimex;316;10000;;19553;6638;829748;208921;835684;1388;307.362917314;2021-12-07@22:44:25 +v2.0.8;mulimex;316;10000;;19553;6638;829208;208807;835228;1395;308.999325562;2021-12-07@22:49:38 +v2.0.8;mulimex;316;10000;;19553;6638;828944;208780;835120;1388;307.283899217;2021-12-07@22:54:49 +v2.0.8;multi;10;17783;;29952;12293;52600;14689;58756;68;14.981767531;2021-12-07@22:56:42 +v2.0.8;multi;10;17783;;29952;12293;52684;14698;58792;73;16.061628388;2021-12-07@22:57:00 +v2.0.8;multi;10;17783;;29952;12293;52960;14697;58788;73;16.066075809;2021-12-07@22:57:18 +v2.0.8;multi;10;17783;;29952;12293;52668;14714;58856;73;16.062526647;2021-12-07@22:57:36 +v2.0.8;multi;10;17783;;29952;12293;52928;14714;58856;73;16.069427309;2021-12-07@22:57:54 +v2.0.8;multi;10;17783;;29952;12293;52596;14700;58800;73;16.059685774;2021-12-07@22:58:12 +v2.0.8;multi;10;17783;;29952;12293;52448;14710;58840;73;16.065174594;2021-12-07@22:58:30 +v2.0.8;multi;10;17783;;29952;12293;52784;14706;58824;73;16.059399392;2021-12-07@22:58:48 +v2.0.8;imex;10;17783;;29952;12293;42096;12074;48296;47;10.279322481;2021-12-07@22:59:07 +v2.0.8;imex;10;17783;;29952;12293;42148;12090;48360;47;10.316036647;2021-12-07@22:59:19 +v2.0.8;imex;10;17783;;29952;12293;41948;12076;48304;48;10.541208636;2021-12-07@22:59:32 +v2.0.8;imex;10;17783;;29952;12293;42196;12091;48364;48;10.535102474;2021-12-07@22:59:44 +v2.0.8;imex;10;17783;;29952;12293;41976;12086;48344;48;10.536781737;2021-12-07@22:59:57 +v2.0.8;imex;10;17783;;29952;12293;42168;12098;48392;48;10.547518191;2021-12-07@23:00:09 +v2.0.8;imex;10;17783;;29952;12293;41840;12076;48304;48;10.536963398;2021-12-07@23:00:22 +v2.0.8;imex;10;17783;;29952;12293;42264;12092;48368;48;10.539528854;2021-12-07@23:00:34 +v2.0.8;single;10;17783;;29952;12293;14076;5068;20272;47;10.281822482;2021-12-07@23:00:54 +v2.0.8;single;10;17783;;29952;12293;14220;5098;20392;49;10.756095546;2021-12-07@23:01:07 +v2.0.8;single;10;17783;;29952;12293;14360;5123;20492;58;12.731911771;2021-12-07@23:01:22 +v2.0.8;single;10;17783;;29952;12293;14244;5121;20484;49;10.759502433;2021-12-07@23:01:34 +v2.0.8;single;10;17783;;29952;12293;14120;5090;20360;49;10.757031388;2021-12-07@23:01:47 +v2.0.8;single;10;17783;;29952;12293;14384;5101;20404;50;10.986469990;2021-12-07@23:02:00 +v2.0.8;single;10;17783;;29952;12293;14400;5092;20368;49;10.756130568;2021-12-07@23:02:13 +v2.0.8;single;10;17783;;29952;12293;14204;5121;20484;50;10.981860948;2021-12-07@23:02:26 +v2.0.8;mulimex;10;17783;;29952;12293;72284;19560;78240;68;14.956576090;2021-12-07@23:02:49 +v2.0.8;mulimex;10;17783;;29952;12293;72000;19545;78180;73;16.057923166;2021-12-07@23:03:08 +v2.0.8;mulimex;10;17783;;29952;12293;71980;19563;78252;73;16.062892732;2021-12-07@23:03:27 +v2.0.8;mulimex;10;17783;;29952;12293;72200;19545;78180;73;16.065900423;2021-12-07@23:03:45 +v2.0.8;mulimex;10;17783;;29952;12293;72108;19569;78276;73;16.065647037;2021-12-07@23:04:02 +v2.0.8;mulimex;10;17783;;29952;12293;72220;19553;78212;73;16.060401545;2021-12-07@23:04:20 +v2.0.8;mulimex;10;17783;;29952;12293;72092;19564;78256;73;16.065845448;2021-12-07@23:04:39 +v2.0.8;mulimex;10;17783;;29952;12293;71912;19576;78304;73;16.081175264;2021-12-07@23:04:57 +v2.0.8;multi;18;17783;;31744;12312;78796;21239;84956;128;28.210543448;2021-12-07@23:05:37 +v2.0.8;multi;18;17783;;31744;12312;78892;21236;84944;133;29.313785115;2021-12-07@23:06:08 +v2.0.8;multi;18;17783;;31744;12312;78860;21259;85036;132;29.089304016;2021-12-07@23:06:39 +v2.0.8;multi;18;17783;;31744;12312;78896;21234;84936;132;29.110696357;2021-12-07@23:07:10 +v2.0.8;multi;18;17783;;31744;12312;78660;21242;84968;132;29.113043019;2021-12-07@23:07:41 +v2.0.8;multi;18;17783;;31744;12312;78820;21249;84996;132;29.098402324;2021-12-07@23:08:12 +v2.0.8;multi;18;17783;;31744;12312;79132;21254;85016;132;29.090970554;2021-12-07@23:08:43 +v2.0.8;multi;18;17783;;31744;12312;78908;21241;84964;132;29.089269354;2021-12-07@23:09:14 +v2.0.8;imex;18;17783;;31744;12312;56100;15548;62192;77;16.934970885;2021-12-07@23:09:45 +v2.0.8;imex;18;17783;;31744;12312;55912;15563;62252;78;17.183357570;2021-12-07@23:10:04 +v2.0.8;imex;18;17783;;31744;12312;55996;15558;62232;78;17.176566163;2021-12-07@23:10:23 +v2.0.8;imex;18;17783;;31744;12312;56012;15556;62224;79;17.397583590;2021-12-07@23:10:43 +v2.0.8;imex;18;17783;;31744;12312;56136;15556;62224;79;17.389481949;2021-12-07@23:11:02 +v2.0.8;imex;18;17783;;31744;12312;56132;15552;62208;78;17.179822082;2021-12-07@23:11:21 +v2.0.8;imex;18;17783;;31744;12312;56024;15544;62176;79;17.384813456;2021-12-07@23:11:41 +v2.0.8;imex;18;17783;;31744;12312;56252;15565;62260;79;17.396236983;2021-12-07@23:12:00 +v2.0.8;single;18;17783;;31744;12312;15444;5423;21692;82;18.007927575;2021-12-07@23:12:32 +v2.0.8;single;18;17783;;31744;12312;15464;5421;21684;82;18.045997939;2021-12-07@23:12:52 +v2.0.8;single;18;17783;;31744;12312;15400;5422;21688;82;18.033770437;2021-12-07@23:13:12 +v2.0.8;single;18;17783;;31744;12312;15596;5421;21684;81;17.816019585;2021-12-07@23:13:31 +v2.0.8;single;18;17783;;31744;12312;15820;5431;21724;82;18.047372275;2021-12-07@23:13:52 +v2.0.8;single;18;17783;;31744;12312;15744;5429;21716;82;18.047277703;2021-12-07@23:14:12 +v2.0.8;single;18;17783;;31744;12312;15704;5434;21736;81;17.802801021;2021-12-07@23:14:32 +v2.0.8;single;18;17783;;31744;12312;15572;5426;21704;84;18.500995532;2021-12-07@23:14:52 +v2.0.8;mulimex;18;17783;;31744;12312;110216;29085;116340;126;27.863846148;2021-12-07@23:15:34 +v2.0.8;mulimex;18;17783;;31744;12312;110224;29085;116340;132;29.110628931;2021-12-07@23:16:06 +v2.0.8;mulimex;18;17783;;31744;12312;110368;29059;116236;139;30.647025097;2021-12-07@23:16:38 +v2.0.8;mulimex;18;17783;;31744;12312;110052;29075;116300;132;29.103343787;2021-12-07@23:17:09 +v2.0.8;mulimex;18;17783;;31744;12312;110556;29070;116280;132;29.119485992;2021-12-07@23:17:40 +v2.0.8;mulimex;18;17783;;31744;12312;110232;29079;116316;132;29.107660152;2021-12-07@23:18:11 +v2.0.8;mulimex;18;17783;;31744;12312;110112;29070;116280;132;29.122564770;2021-12-07@23:18:43 +v2.0.8;mulimex;18;17783;;31744;12312;110428;29097;116388;133;29.338075528;2021-12-07@23:19:15 +v2.0.8;multi;32;17783;;32894;12393;123888;32500;130000;231;51.025597980;2021-12-07@23:20:26 +v2.0.8;multi;32;17783;;32894;12393;123880;32488;129952;236;52.143516022;2021-12-07@23:21:20 +v2.0.8;multi;32;17783;;32894;12393;124060;32473;129892;236;52.153346593;2021-12-07@23:22:14 +v2.0.8;multi;32;17783;;32894;12393;123876;32480;129920;236;52.130243958;2021-12-07@23:23:08 +v2.0.8;multi;32;17783;;32894;12393;123792;32499;129996;236;52.117796364;2021-12-07@23:24:02 +v2.0.8;multi;32;17783;;32894;12393;123876;32501;130004;236;52.127692385;2021-12-07@23:24:55 +v2.0.8;multi;32;17783;;32894;12393;123756;32485;129940;236;52.132111841;2021-12-07@23:25:49 +v2.0.8;multi;32;17783;;32894;12393;123744;32491;129964;236;52.154881655;2021-12-07@23:26:43 +v2.0.8;imex;32;17783;;32894;12393;78876;21268;85072;133;29.291456317;2021-12-07@23:27:35 +v2.0.8;imex;32;17783;;32894;12393;78948;21281;85124;133;29.312148251;2021-12-07@23:28:06 +v2.0.8;imex;32;17783;;32894;12393;79072;21251;85004;134;29.546412078;2021-12-07@23:28:37 +v2.0.8;imex;32;17783;;32894;12393;78724;21291;85164;133;29.311076537;2021-12-07@23:29:09 +v2.0.8;imex;32;17783;;32894;12393;79204;21285;85140;133;29.322697128;2021-12-07@23:29:40 +v2.0.8;imex;32;17783;;32894;12393;78736;21266;85064;133;29.330312741;2021-12-07@23:30:11 +v2.0.8;imex;32;17783;;32894;12393;78648;21252;85008;134;29.545044913;2021-12-07@23:30:43 +v2.0.8;imex;32;17783;;32894;12393;78892;21269;85076;132;29.095177339;2021-12-07@23:31:14 +v2.0.8;single;32;17783;;32894;12393;16984;5785;23140;139;30.613300128;2021-12-07@23:32:07 +v2.0.8;single;32;17783;;32894;12393;17048;5794;23176;138;30.414009306;2021-12-07@23:32:39 +v2.0.8;single;32;17783;;32894;12393;16940;5800;23200;140;30.867701920;2021-12-07@23:33:12 +v2.0.8;single;32;17783;;32894;12393;16812;5793;23172;138;30.428487508;2021-12-07@23:33:45 +v2.0.8;single;32;17783;;32894;12393;16980;5783;23132;137;30.204243693;2021-12-07@23:34:17 +v2.0.8;single;32;17783;;32894;12393;16856;5789;23156;140;30.853967292;2021-12-07@23:34:50 +v2.0.8;single;32;17783;;32894;12393;17232;5817;23268;140;30.865001918;2021-12-07@23:35:23 +v2.0.8;single;32;17783;;32894;12393;16964;5823;23292;138;30.411766172;2021-12-07@23:35:55 +v2.0.8;mulimex;32;17783;;32894;12393;176232;45555;182220;233;51.453441508;2021-12-07@23:37:08 +v2.0.8;mulimex;32;17783;;32894;12393;176196;45575;182300;237;52.306092287;2021-12-07@23:38:03 +v2.0.8;mulimex;32;17783;;32894;12393;176072;45557;182228;237;52.360627637;2021-12-07@23:38:57 +v2.0.8;mulimex;32;17783;;32894;12393;176280;45563;182252;237;52.329645269;2021-12-07@23:39:52 +v2.0.8;mulimex;32;17783;;32894;12393;176180;45571;182284;237;52.334679469;2021-12-07@23:40:46 +v2.0.8;mulimex;32;17783;;32894;12393;175936;45551;182204;237;52.266928195;2021-12-07@23:41:41 +v2.0.8;mulimex;32;17783;;32894;12393;176080;45553;182212;237;52.361941135;2021-12-07@23:42:35 +v2.0.8;mulimex;32;17783;;32894;12393;176028;45536;182144;237;52.321463733;2021-12-07@23:43:29 +v2.0.8;multi;56;17783;;33578;12411;199572;51429;205716;410;90.709436502;2021-12-07@23:45:30 +v2.0.8;multi;56;17783;;33578;12411;202084;52088;208352;416;92.071376431;2021-12-07@23:47:05 +v2.0.8;multi;56;17783;;33578;12411;201836;52041;208164;427;94.509872876;2021-12-07@23:48:42 +v2.0.8;multi;56;17783;;33578;12411;202932;52236;208944;418;92.201157262;2021-12-07@23:50:16 +v2.0.8;multi;56;17783;;33578;12411;203200;52327;209308;517;114.540884193;2021-12-07@23:52:13 +v2.0.8;multi;56;17783;;33578;12411;201932;51992;207968;417;92.309036585;2021-12-07@23:53:47 +v2.0.8;multi;56;17783;;33578;12411;199492;51395;205580;416;92.126528983;2021-12-07@23:55:22 +v2.0.8;multi;56;17783;;33578;12411;205612;52988;211952;518;114.650827855;2021-12-07@23:57:19 +v2.0.8;imex;56;17783;;33578;12411;116292;30634;122536;227;50.028597511;2021-12-07@23:58:47 +v2.0.8;imex;56;17783;;33578;12411;116492;30623;122492;224;49.454963126;2021-12-07@23:59:39 +v2.0.8;imex;56;17783;;33578;12411;116344;30604;122416;224;49.430264565;2021-12-08@00:00:30 +v2.0.8;imex;56;17783;;33578;12411;116284;30628;122512;232;51.179389336;2021-12-08@00:01:24 +v2.0.8;imex;56;17783;;33578;12411;116292;30635;122540;232;51.184033626;2021-12-08@00:02:17 +v2.0.8;imex;56;17783;;33578;12411;116428;30608;122432;226;49.924571322;2021-12-08@00:03:09 +v2.0.8;imex;56;17783;;33578;12411;116116;30601;122404;225;49.642720352;2021-12-08@00:04:00 +v2.0.8;imex;56;17783;;33578;12411;116044;30611;122444;231;50.976736588;2021-12-08@00:04:54 +v2.0.8;single;56;17783;;33578;12411;18916;6267;25068;238;52.491792706;2021-12-08@00:06:23 +v2.0.8;single;56;17783;;33578;12411;18848;6246;24984;234;51.655042864;2021-12-08@00:07:17 +v2.0.8;single;56;17783;;33578;12411;18964;6281;25124;236;52.079049228;2021-12-08@00:08:11 +v2.0.8;single;56;17783;;33578;12411;18848;6248;24992;233;51.420948875;2021-12-08@00:09:04 +v2.0.8;single;56;17783;;33578;12411;18876;6311;25244;234;51.679661905;2021-12-08@00:09:58 +v2.0.8;single;56;17783;;33578;12411;18568;6235;24940;234;51.654971494;2021-12-08@00:10:51 +v2.0.8;single;56;17783;;33578;12411;19012;6241;24964;244;53.827829759;2021-12-08@00:11:47 +v2.0.8;single;56;17783;;33578;12411;18868;6268;25072;234;51.656667337;2021-12-08@00:12:41 +v2.0.8;mulimex;56;17783;;33578;12411;291268;74389;297556;411;90.828744403;2021-12-08@00:14:49 +v2.0.8;mulimex;56;17783;;33578;12411;288980;73784;295136;416;92.179599742;2021-12-08@00:16:23 +v2.0.8;mulimex;56;17783;;33578;12411;288208;73555;294220;417;92.196572792;2021-12-08@00:17:58 +v2.0.8;mulimex;56;17783;;33578;12411;290044;74056;296224;416;92.284506559;2021-12-08@00:19:33 +v2.0.8;mulimex;56;17783;;33578;12411;291748;74514;298056;417;92.166442512;2021-12-08@00:21:07 +v2.0.8;mulimex;56;17783;;33578;12411;290960;74215;296860;417;92.124233886;2021-12-08@00:22:42 +v2.0.8;mulimex;56;17783;;33578;12411;288884;73789;295156;417;92.200807914;2021-12-08@00:24:16 +v2.0.8;mulimex;56;17783;;33578;12411;290012;74004;296016;418;92.364635929;2021-12-08@00:25:51 +v2.0.8;multi;100;17783;;33951;12499;337100;85821;343284;737;163.537420557;2021-12-08@00:29:28 +v2.0.8;multi;100;17783;;33951;12499;380704;96687;386748;754;166.096816473;2021-12-08@00:32:16 +v2.0.8;multi;100;17783;;33951;12499;374888;95225;380900;751;166.000934873;2021-12-08@00:35:05 +v2.0.8;multi;100;17783;;33951;12499;375784;95511;382044;751;165.967554851;2021-12-08@00:37:57 +v2.0.8;multi;100;17783;;33951;12499;338612;86179;344716;739;163.883216607;2021-12-08@00:40:57 +v2.0.8;multi;100;17783;;33951;12499;375588;95469;381876;454;100.667637280;2021-12-08@00:43:46 +v2.0.8;multi;100;17783;;33951;12499;375468;95448;381792;752;166.204631750;2021-12-08@00:46:35 +v2.0.8;multi;100;17783;;33951;12499;368620;93677;374708;452;100.455355480;2021-12-08@00:49:23 +v2.0.8;imex;100;17783;;33951;12499;184684;47733;190932;417;92.236102418;2021-12-08@00:52:00 +v2.0.8;imex;100;17783;;33951;12499;184704;47737;190948;411;91.056504584;2021-12-08@00:53:34 +v2.0.8;imex;100;17783;;33951;12499;184752;47729;190916;419;92.899060561;2021-12-08@00:55:08 +v2.0.8;imex;100;17783;;33951;12499;184808;47731;190924;417;92.277196693;2021-12-08@00:56:42 +v2.0.8;imex;100;17783;;33951;12499;184832;47742;190968;414;91.628513044;2021-12-08@00:58:16 +v2.0.8;imex;100;17783;;33951;12499;184684;47746;190984;412;91.207421323;2021-12-08@00:59:49 +v2.0.8;imex;100;17783;;33951;12499;184824;47744;190976;406;89.856505062;2021-12-08@01:01:21 +v2.0.8;imex;100;17783;;33951;12499;184500;47701;190804;408;90.295013261;2021-12-08@01:02:53 +v2.0.8;single;100;17783;;33951;12499;21448;6903;27612;432;95.548503721;2021-12-08@01:05:30 +v2.0.8;single;100;17783;;33951;12499;21712;6932;27728;438;96.989551953;2021-12-08@01:07:09 +v2.0.8;single;100;17783;;33951;12499;21444;6930;27720;422;93.446610267;2021-12-08@01:08:44 +v2.0.8;single;100;17783;;33951;12499;21764;6951;27804;420;92.991451896;2021-12-08@01:10:19 +v2.0.8;single;100;17783;;33951;12499;21408;6932;27728;424;93.896027936;2021-12-08@01:11:55 +v2.0.8;single;100;17783;;33951;12499;21408;6879;27516;428;94.736583603;2021-12-08@01:13:32 +v2.0.8;single;100;17783;;33951;12499;21588;6928;27712;422;93.414625593;2021-12-08@01:15:07 +v2.0.8;single;100;17783;;33951;12499;21120;6867;27468;428;94.866818099;2021-12-08@01:16:44 +v2.0.8;mulimex;100;17783;;33951;12499;490120;124106;496424;738;164.269970695;2021-12-08@01:20:29 +v2.0.8;mulimex;100;17783;;33951;12499;549020;138821;555284;455;99.813440827;2021-12-08@01:23:18 +v2.0.8;mulimex;100;17783;;33951;12499;545632;137969;551876;455;100.004036697;2021-12-08@01:26:06 +v2.0.8;mulimex;100;17783;;33951;12499;541404;136885;547540;451;100.153725312;2021-12-08@01:28:55 +v2.0.8;mulimex;100;17783;;33951;12499;540380;136614;546456;452;100.417072126;2021-12-08@01:31:43 +v2.0.8;mulimex;100;17783;;33951;12499;541076;136810;547240;455;100.331728609;2021-12-08@01:34:32 +v2.0.8;mulimex;100;17783;;33951;12499;544456;137644;550576;453;100.029168896;2021-12-08@01:37:20 +v2.0.8;mulimex;100;17783;;33951;12499;540284;136511;546044;449;100.207336557;2021-12-08@01:40:08 +v2.0.8;multi;178;17783;;34156;12582;580792;146746;586984;1370;306.165294100;2021-12-08@01:46:51 +v2.0.8;multi;178;17783;;34156;12582;580356;146654;586616;1551;347.958025333;2021-12-08@01:52:42 +v2.0.8;multi;178;17783;;34156;12582;580288;146613;586452;1556;348.992405464;2021-12-08@01:58:33 +v2.0.8;multi;178;17783;;34156;12582;583052;147269;589076;1360;303.963340554;2021-12-08@02:03:40 +v2.0.8;multi;178;17783;;34156;12582;580092;146595;586380;1358;303.804616588;2021-12-08@02:08:47 +v2.0.8;multi;178;17783;;34156;12582;580392;146626;586504;1360;303.917704343;2021-12-08@02:13:54 +v2.0.8;multi;178;17783;;34156;12582;580660;146639;586556;1360;304.079597270;2021-12-08@02:19:01 +v2.0.8;multi;178;17783;;34156;12582;580336;146615;586460;1355;302.795462304;2021-12-08@02:24:06 +v2.0.8;imex;178;17783;;34156;12582;305088;77832;311328;777;172.466694965;2021-12-08@02:28:57 +v2.0.8;imex;178;17783;;34156;12582;305332;77899;311596;733;162.621100016;2021-12-08@02:31:42 +v2.0.8;imex;178;17783;;34156;12582;305312;77923;311692;757;168.184861654;2021-12-08@02:34:33 +v2.0.8;imex;178;17783;;34156;12582;305420;77906;311624;749;166.092477614;2021-12-08@02:37:21 +v2.0.8;imex;178;17783;;34156;12582;305408;77912;311648;720;159.758593918;2021-12-08@02:40:03 +v2.0.8;imex;178;17783;;34156;12582;305424;77912;311648;759;168.331348384;2021-12-08@02:42:53 +v2.0.8;imex;178;17783;;34156;12582;305576;77921;311684;769;170.463281707;2021-12-08@02:45:46 +v2.0.8;imex;178;17783;;34156;12582;305560;77939;311756;742;164.691179161;2021-12-08@02:48:33 +v2.0.8;single;178;17783;;34156;12582;25572;7941;31764;801;178.038615746;2021-12-08@02:53:15 +v2.0.8;single;178;17783;;34156;12582;25516;7942;31768;746;166.299921978;2021-12-08@02:56:03 +v2.0.8;single;178;17783;;34156;12582;25436;7898;31592;762;169.129662317;2021-12-08@02:58:54 +v2.0.8;single;178;17783;;34156;12582;25392;7848;31392;774;171.813185791;2021-12-08@03:01:48 +v2.0.8;single;178;17783;;34156;12582;25468;7888;31552;763;169.405391931;2021-12-08@03:04:40 +v2.0.8;single;178;17783;;34156;12582;25984;7995;31980;762;169.368580681;2021-12-08@03:07:31 +v2.0.8;single;178;17783;;34156;12582;25324;7862;31448;767;170.154276387;2021-12-08@03:10:23 +v2.0.8;single;178;17783;;34156;12582;26012;7989;31956;771;171.319508879;2021-12-08@03:13:16 +v2.0.8;mulimex;178;17783;;34156;12582;850224;214038;856152;1352;302.580194981;2021-12-08@03:20:04 +v2.0.8;mulimex;178;17783;;34156;12582;849928;213999;855996;1368;305.720676944;2021-12-08@03:25:13 +v2.0.8;mulimex;178;17783;;34156;12582;849720;214004;856016;1359;304.264812969;2021-12-08@03:30:21 +v2.0.8;mulimex;178;17783;;34156;12582;849892;214041;856164;1364;304.962652047;2021-12-08@03:35:30 +v2.0.8;mulimex;178;17783;;34156;12582;850140;214062;856248;1359;303.756259177;2021-12-08@03:40:36 +v2.0.8;mulimex;178;17783;;34156;12582;850068;214111;856444;1356;303.201357056;2021-12-08@03:45:43 +v2.0.8;mulimex;178;17783;;34156;12582;849968;214056;856224;461;101.320614191;2021-12-08@03:50:50 +v2.0.8;mulimex;178;17783;;34156;12582;850000;214075;856300;1360;302.886359514;2021-12-08@03:55:56 +v2.0.8;multi;316;17783;;34574;12710;1017108;255823;1023292;2936;661.001866960;2021-12-08@04:09:53 +v2.0.8;multi;316;17783;;34574;12710;1017140;255807;1023228;2376;533.394451529;2021-12-08@04:18:51 +v2.0.8;multi;316;17783;;34574;12710;1017404;255856;1023424;2381;535.857541385;2021-12-08@04:27:51 +v2.0.8;multi;316;17783;;34574;12710;1017200;255855;1023420;2361;529.716934273;2021-12-08@04:36:45 +v2.0.8;multi;316;17783;;34574;12710;1017236;255840;1023360;2375;533.717903276;2021-12-08@04:45:42 +v2.0.8;multi;316;17783;;34574;12710;1017072;255794;1023176;2385;536.516402253;2021-12-08@04:54:43 +v2.0.8;multi;316;17783;;34574;12710;1017548;255881;1023524;2372;534.219961926;2021-12-08@05:03:41 +v2.0.8;multi;316;17783;;34574;12710;1017056;255853;1023412;2378;535.330217231;2021-12-08@05:12:40 +v2.0.8;imex;316;17783;;34574;12710;524428;132600;530400;1574;353.785847633;2021-12-08@05:22:13 +v2.0.8;imex;316;17783;;34574;12710;522096;132027;528108;1452;324.136401718;2021-12-08@05:27:39 +v2.0.8;imex;316;17783;;34574;12710;521836;131995;527980;1675;376.266049845;2021-12-08@05:33:58 +v2.0.8;imex;316;17783;;34574;12710;521712;131998;527992;1735;388.377112181;2021-12-08@05:40:29 +v2.0.8;imex;316;17783;;34574;12710;522420;132138;528552;1441;321.706880621;2021-12-08@05:45:53 +v2.0.8;imex;316;17783;;34574;12710;521548;131975;527900;1448;323.891415141;2021-12-08@05:51:19 +v2.0.8;imex;316;17783;;34574;12710;522536;132211;528844;1439;321.977521834;2021-12-08@05:56:44 +v2.0.8;imex;316;17783;;34574;12710;521972;132022;528088;1645;368.637133013;2021-12-08@06:02:55 +v2.0.8;single;316;17783;;34574;12710;32900;9744;38976;1648;369.471450326;2021-12-08@06:12:16 +v2.0.8;single;316;17783;;34574;12710;35192;10379;41516;1522;341.706180958;2021-12-08@06:18:00 +v2.0.8;single;316;17783;;34574;12710;31904;9478;37912;1738;389.427384637;2021-12-08@06:24:31 +v2.0.8;single;316;17783;;34574;12710;33784;9933;39732;1523;339.824139183;2021-12-08@06:30:13 +v2.0.8;single;316;17783;;34574;12710;32060;9574;38296;1538;343.246728738;2021-12-08@06:35:58 +v2.0.8;single;316;17783;;34574;12710;32748;9677;38708;1734;387.685683063;2021-12-08@06:42:27 +v2.0.8;single;316;17783;;34574;12710;34916;10277;41108;1759;394.525265765;2021-12-08@06:49:04 +v2.0.8;single;316;17783;;34574;12710;34888;10247;40988;1512;336.965964558;2021-12-08@06:54:43 +v2.0.8;mulimex;316;17783;;34574;12710;1496456;375646;1502584;2606;586.344607203;2021-12-08@07:07:40 +v2.0.8;mulimex;316;17783;;34574;12710;1496384;375639;1502556;1368;305.659665680;2021-12-08@07:16:39 +v2.0.8;mulimex;316;17783;;34574;12710;1496368;375586;1502344;2393;535.508388759;2021-12-08@07:25:40 +v2.0.8;mulimex;316;17783;;34574;12710;1496864;375710;1502840;2364;531.096878086;2021-12-08@07:34:36 +v2.0.8;mulimex;316;17783;;34574;12710;1496264;375640;1502560;2402;537.854836184;2021-12-08@07:43:39 +v2.0.8;mulimex;316;17783;;34574;12710;1496384;375654;1502616;2386;535.118076245;2021-12-08@07:52:39 +v2.0.8;mulimex;316;17783;;34574;12710;1496504;375617;1502468;2396;536.890438660;2021-12-08@08:01:41 +v2.0.8;mulimex;316;17783;;34574;12710;1496752;375656;1502624;2390;535.641925649;2021-12-08@08:10:41 +v2.0.8;multi;10;31623;;54425;24231;92936;24807;99228;117;26.368815866;2021-12-08@08:12:47 +v2.0.8;multi;10;31623;;54425;24231;93212;24810;99240;124;27.874472220;2021-12-08@08:13:18 +v2.0.8;multi;10;31623;;54425;24231;93140;24805;99220;124;27.872004940;2021-12-08@08:13:49 +v2.0.8;multi;10;31623;;54425;24231;92888;24813;99252;124;27.874111579;2021-12-08@08:14:21 +v2.0.8;multi;10;31623;;54425;24231;93128;24817;99268;124;27.884302590;2021-12-08@08:14:50 +v2.0.8;multi;10;31623;;54425;24231;93116;24796;99184;124;27.880850887;2021-12-08@08:15:20 +v2.0.8;multi;10;31623;;54425;24231;93092;24815;99260;124;27.897365927;2021-12-08@08:15:50 +v2.0.8;multi;10;31623;;54425;24231;92968;24810;99240;124;27.894408342;2021-12-08@08:16:20 +v2.0.8;imex;10;31623;;54425;24231;74980;20308;81232;78;17.411649311;2021-12-08@08:16:47 +v2.0.8;imex;10;31623;;54425;24231;75116;20318;81272;79;17.675904803;2021-12-08@08:17:06 +v2.0.8;imex;10;31623;;54425;24231;75088;20335;81340;80;17.889252699;2021-12-08@08:17:26 +v2.0.8;imex;10;31623;;54425;24231;74984;20340;81360;80;17.906519849;2021-12-08@08:17:46 +v2.0.8;imex;10;31623;;54425;24231;75232;20319;81276;81;18.117705616;2021-12-08@08:18:06 +v2.0.8;imex;10;31623;;54425;24231;75168;20343;81372;81;18.113592541;2021-12-08@08:18:26 +v2.0.8;imex;10;31623;;54425;24231;75096;20326;81304;80;17.896420267;2021-12-08@08:18:46 +v2.0.8;imex;10;31623;;54425;24231;74964;20316;81264;80;17.891189684;2021-12-08@08:19:05 +v2.0.8;single;10;31623;;54425;24231;23080;7273;29092;81;18.109329035;2021-12-08@08:19:34 +v2.0.8;single;10;31623;;54425;24231;22916;7291;29164;85;19.012704115;2021-12-08@08:19:55 +v2.0.8;single;10;31623;;54425;24231;23080;7303;29212;83;18.569219327;2021-12-08@08:20:16 +v2.0.8;single;10;31623;;54425;24231;22892;7305;29220;84;18.806752030;2021-12-08@08:20:37 +v2.0.8;single;10;31623;;54425;24231;22984;7316;29264;84;18.790478733;2021-12-08@08:20:57 +v2.0.8;single;10;31623;;54425;24231;22956;7287;29148;84;18.773703064;2021-12-08@08:21:18 +v2.0.8;single;10;31623;;54425;24231;23016;7298;29192;83;18.562210551;2021-12-08@08:21:38 +v2.0.8;single;10;31623;;54425;24231;22972;7321;29284;84;18.787685160;2021-12-08@08:21:59 +v2.0.8;mulimex;10;31623;;54425;24231;130520;34147;136588;119;26.772858515;2021-12-08@08:22:35 +v2.0.8;mulimex;10;31623;;54425;24231;130304;34167;136668;124;27.888189765;2021-12-08@08:23:05 +v2.0.8;mulimex;10;31623;;54425;24231;130364;34166;136664;124;27.899722939;2021-12-08@08:23:35 +v2.0.8;mulimex;10;31623;;54425;24231;130504;34162;136648;124;27.890003623;2021-12-08@08:24:05 +v2.0.8;mulimex;10;31623;;54425;24231;130420;34140;136560;124;27.884966947;2021-12-08@08:24:34 +v2.0.8;mulimex;10;31623;;54425;24231;130460;34144;136576;124;27.913979295;2021-12-08@08:25:04 +v2.0.8;mulimex;10;31623;;54425;24231;130512;34131;136524;124;27.880353152;2021-12-08@08:25:34 +v2.0.8;mulimex;10;31623;;54425;24231;130412;34144;136576;124;27.910615175;2021-12-08@08:26:03 +v2.0.8;multi;18;31623;;57202;24252;140280;36577;146308;221;49.758809558;2021-12-08@08:27:06 +v2.0.8;multi;18;31623;;57202;24252;140092;36550;146200;227;51.115461302;2021-12-08@08:27:59 +v2.0.8;multi;18;31623;;57202;24252;139988;36551;146204;227;51.180680346;2021-12-08@08:28:52 +v2.0.8;multi;18;31623;;57202;24252;140024;36539;146156;227;51.120056180;2021-12-08@08:29:45 +v2.0.8;multi;18;31623;;57202;24252;140192;36536;146144;227;51.170873079;2021-12-08@08:30:38 +v2.0.8;multi;18;31623;;57202;24252;140168;36560;146240;227;51.129130081;2021-12-08@08:31:31 +v2.0.8;multi;18;31623;;57202;24252;139892;36541;146164;227;51.134665995;2021-12-08@08:32:24 +v2.0.8;multi;18;31623;;57202;24252;140144;36546;146184;227;51.143209579;2021-12-08@08:33:17 +v2.0.8;imex;18;31623;;57202;24252;101252;26842;107368;137;30.709332501;2021-12-08@08:34:02 +v2.0.8;imex;18;31623;;57202;24252;101180;26880;107520;136;30.519154328;2021-12-08@08:34:34 +v2.0.8;imex;18;31623;;57202;24252;101280;26867;107468;139;31.184962472;2021-12-08@08:35:07 +v2.0.8;imex;18;31623;;57202;24252;101304;26871;107484;137;30.719885872;2021-12-08@08:35:40 +v2.0.8;imex;18;31623;;57202;24252;101452;26898;107592;138;30.967063958;2021-12-08@08:36:13 +v2.0.8;imex;18;31623;;57202;24252;101128;26872;107488;135;30.354842019;2021-12-08@08:36:45 +v2.0.8;imex;18;31623;;57202;24252;101180;26883;107532;139;31.176447833;2021-12-08@08:37:19 +v2.0.8;imex;18;31623;;57202;24252;101452;26865;107460;138;30.961640338;2021-12-08@08:37:53 +v2.0.8;single;18;31623;;57202;24252;24944;7793;31172;142;31.858615482;2021-12-08@08:38:40 +v2.0.8;single;18;31623;;57202;24252;24756;7760;31040;141;31.651175859;2021-12-08@08:39:14 +v2.0.8;single;18;31623;;57202;24252;24872;7778;31112;140;31.411432215;2021-12-08@08:39:47 +v2.0.8;single;18;31623;;57202;24252;24924;7775;31100;141;31.683337224;2021-12-08@08:40:21 +v2.0.8;single;18;31623;;57202;24252;24840;7774;31096;144;32.310743085;2021-12-08@08:40:55 +v2.0.8;single;18;31623;;57202;24252;24672;7774;31096;145;32.517663232;2021-12-08@08:41:29 +v2.0.8;single;18;31623;;57202;24252;24928;7769;31076;143;32.093836004;2021-12-08@08:42:03 +v2.0.8;single;18;31623;;57202;24252;25088;7802;31208;143;32.075427826;2021-12-08@08:42:37 +v2.0.8;mulimex;18;31623;;57202;24252;200848;51737;206948;223;50.241548010;2021-12-08@08:43:41 +v2.0.8;mulimex;18;31623;;57202;24252;200536;51708;206832;228;51.381461937;2021-12-08@08:44:34 +v2.0.8;mulimex;18;31623;;57202;24252;200512;51700;206800;228;51.383348517;2021-12-08@08:45:28 +v2.0.8;mulimex;18;31623;;57202;24252;200948;51709;206836;228;51.373186780;2021-12-08@08:46:21 +v2.0.8;mulimex;18;31623;;57202;24252;200620;51731;206924;228;51.385071771;2021-12-08@08:47:15 +v2.0.8;mulimex;18;31623;;57202;24252;200988;51716;206864;228;51.366925804;2021-12-08@08:48:08 +v2.0.8;mulimex;18;31623;;57202;24252;200900;51729;206916;282;63.554246095;2021-12-08@08:49:14 +v2.0.8;mulimex;18;31623;;57202;24252;200568;51726;206904;228;51.413892532;2021-12-08@08:50:07 +v2.0.8;multi;32;31623;;58800;24186;219516;56418;225672;401;90.740418873;2021-12-08@08:51:57 +v2.0.8;multi;32;31623;;58800;24186;219696;56439;225756;407;92.058291025;2021-12-08@08:53:32 +v2.0.8;multi;32;31623;;58800;24186;219404;56407;225628;407;92.057910792;2021-12-08@08:55:06 +v2.0.8;multi;32;31623;;58800;24186;219400;56454;225816;408;92.147867635;2021-12-08@08:56:40 +v2.0.8;multi;32;31623;;58800;24186;219516;56422;225688;407;92.059075489;2021-12-08@08:58:14 +v2.0.8;multi;32;31623;;58800;24186;219464;56448;225792;408;92.235867675;2021-12-08@08:59:49 +v2.0.8;multi;32;31623;;58800;24186;219820;56422;225688;407;92.062971760;2021-12-08@09:01:23 +v2.0.8;multi;32;31623;;58800;24186;219684;56458;225832;408;92.163197655;2021-12-08@09:02:57 +v2.0.8;imex;32;31623;;58800;24186;143972;37508;150032;239;53.774679488;2021-12-08@09:04:14 +v2.0.8;imex;32;31623;;58800;24186;143696;37491;149964;237;53.290428657;2021-12-08@09:05:09 +v2.0.8;imex;32;31623;;58800;24186;143900;37493;149972;236;53.082550593;2021-12-08@09:06:04 +v2.0.8;imex;32;31623;;58800;24186;143824;37539;150156;240;53.956441175;2021-12-08@09:07:00 +v2.0.8;imex;32;31623;;58800;24186;143620;37503;150012;236;53.063892848;2021-12-08@09:07:55 +v2.0.8;imex;32;31623;;58800;24186;143804;37497;149988;237;53.310853415;2021-12-08@09:08:50 +v2.0.8;imex;32;31623;;58800;24186;143736;37487;149948;239;53.768769316;2021-12-08@09:09:46 +v2.0.8;imex;32;31623;;58800;24186;143488;37487;149948;236;53.037211619;2021-12-08@09:10:41 +v2.0.8;single;32;31623;;58800;24186;26920;8188;32752;248;55.736633070;2021-12-08@09:11:59 +v2.0.8;single;32;31623;;58800;24186;26496;8218;32872;246;55.345376662;2021-12-08@09:12:56 +v2.0.8;single;32;31623;;58800;24186;26452;8203;32812;245;55.150783123;2021-12-08@09:13:53 +v2.0.8;single;32;31623;;58800;24186;26528;8201;32804;247;55.586255114;2021-12-08@09:14:51 +v2.0.8;single;32;31623;;58800;24186;26828;8224;32896;246;55.392899527;2021-12-08@09:15:48 +v2.0.8;single;32;31623;;58800;24186;26568;8213;32852;246;55.352709793;2021-12-08@09:16:46 +v2.0.8;single;32;31623;;58800;24186;26512;8212;32848;245;55.098677291;2021-12-08@09:17:42 +v2.0.8;single;32;31623;;58800;24186;26876;8250;33000;251;56.466797543;2021-12-08@09:18:41 +v2.0.8;mulimex;32;31623;;58800;24186;320124;81618;326472;402;90.828775110;2021-12-08@09:20:35 +v2.0.8;mulimex;32;31623;;58800;24186;320184;81579;326316;409;92.431039536;2021-12-08@09:22:10 +v2.0.8;mulimex;32;31623;;58800;24186;320260;81559;326236;414;93.547639538;2021-12-08@09:23:45 +v2.0.8;mulimex;32;31623;;58800;24186;320064;81571;326284;409;92.398306196;2021-12-08@09:25:20 +v2.0.8;mulimex;32;31623;;58800;24186;320328;81576;326304;409;92.358358164;2021-12-08@09:26:55 +v2.0.8;mulimex;32;31623;;58800;24186;319840;81548;326192;409;92.453979741;2021-12-08@09:28:29 +v2.0.8;mulimex;32;31623;;58800;24186;319872;81540;326160;409;92.411970574;2021-12-08@09:30:04 +v2.0.8;mulimex;32;31623;;58800;24186;319924;81569;326276;409;92.389010192;2021-12-08@09:31:39 +v2.0.8;multi;56;31623;;59758;24285;354688;90225;360900;715;161.882937029;2021-12-08@09:34:52 +v2.0.8;multi;56;31623;;59758;24285;354564;90209;360836;717;162.852905444;2021-12-08@09:37:37 +v2.0.8;multi;56;31623;;59758;24285;354656;90227;360908;719;162.936763656;2021-12-08@09:40:22 +v2.0.8;multi;56;31623;;59758;24285;354896;90267;361068;718;162.651194717;2021-12-08@09:43:06 +v2.0.8;multi;56;31623;;59758;24285;354808;90262;361048;717;162.643113485;2021-12-08@09:45:51 +v2.0.8;multi;56;31623;;59758;24285;354728;90229;360916;717;162.501436861;2021-12-08@09:48:35 +v2.0.8;multi;56;31623;;59758;24285;354612;90231;360924;715;162.355517930;2021-12-08@09:51:19 +v2.0.8;multi;56;31623;;59758;24285;354792;90236;360944;719;163.046735284;2021-12-08@09:54:04 +v2.0.8;imex;56;31623;;59758;24285;215568;55455;221820;414;93.411149256;2021-12-08@09:56:14 +v2.0.8;imex;56;31623;;59758;24285;215940;55529;222116;407;91.881835857;2021-12-08@09:57:48 +v2.0.8;imex;56;31623;;59758;24285;215916;55494;221976;408;92.077494877;2021-12-08@09:59:23 +v2.0.8;imex;56;31623;;59758;24285;215576;55475;221900;409;92.359334313;2021-12-08@10:00:57 +v2.0.8;imex;56;31623;;59758;24285;215524;55487;221948;409;92.385897340;2021-12-08@10:02:32 +v2.0.8;imex;56;31623;;59758;24285;215732;55497;221988;441;99.625898648;2021-12-08@10:04:14 +v2.0.8;imex;56;31623;;59758;24285;215648;55511;222044;406;91.681708219;2021-12-08@10:05:48 +v2.0.8;imex;56;31623;;59758;24285;215984;55486;221944;408;92.104395465;2021-12-08@10:07:23 +v2.0.8;single;56;31623;;59758;24285;28992;8831;35324;433;97.751392193;2021-12-08@10:09:37 +v2.0.8;single;56;31623;;59758;24285;29156;8814;35256;427;96.402729970;2021-12-08@10:11:16 +v2.0.8;single;56;31623;;59758;24285;29028;8788;35152;427;96.397286775;2021-12-08@10:12:54 +v2.0.8;single;56;31623;;59758;24285;29072;8852;35408;425;95.973269697;2021-12-08@10:14:33 +v2.0.8;single;56;31623;;59758;24285;28788;8754;35016;420;94.841135407;2021-12-08@10:16:10 +v2.0.8;single;56;31623;;59758;24285;28968;8787;35148;429;96.885858180;2021-12-08@10:17:49 +v2.0.8;single;56;31623;;59758;24285;29084;8815;35260;430;97.200190073;2021-12-08@10:19:28 +v2.0.8;single;56;31623;;59758;24285;29232;8816;35264;422;95.318085951;2021-12-08@10:21:05 +v2.0.8;mulimex;56;31623;;59758;24285;524208;132570;530280;716;161.948015906;2021-12-08@10:24:24 +v2.0.8;mulimex;56;31623;;59758;24285;524228;132589;530356;718;163.061667192;2021-12-08@10:27:10 +v2.0.8;mulimex;56;31623;;59758;24285;524176;132580;530320;720;162.980213869;2021-12-08@10:29:55 +v2.0.8;mulimex;56;31623;;59758;24285;524280;132610;530440;720;163.224117724;2021-12-08@10:32:42 +v2.0.8;mulimex;56;31623;;59758;24285;525116;132875;531500;720;163.238807927;2021-12-08@10:35:28 +v2.0.8;mulimex;56;31623;;59758;24285;524284;132589;530356;719;163.292320790;2021-12-08@10:38:14 +v2.0.8;mulimex;56;31623;;59758;24285;524116;132572;530288;720;163.067451590;2021-12-08@10:41:00 +v2.0.8;mulimex;56;31623;;59758;24285;524388;132639;530556;720;163.206057859;2021-12-08@10:43:47 +v2.0.8;multi;100;31623;;60390;24233;601344;151833;607332;1513;344.883511822;2021-12-08@10:50:26 +v2.0.8;multi;100;31623;;60390;24233;624400;157645;630580;1165;268.780763274;2021-12-08@10:56:03 +v2.0.8;multi;100;31623;;60390;24233;623584;157430;629720;1377;313.470390920;2021-12-08@11:01:21 +v2.0.8;multi;100;31623;;60390;24233;621912;156976;627904;1245;288.208275459;2021-12-08@11:07:17 +v2.0.8;multi;100;31623;;60390;24233;617444;155895;623580;1230;283.293392755;2021-12-08@11:13:09 +v2.0.8;multi;100;31623;;60390;24233;612280;154578;618312;1496;341.697738777;2021-12-08@11:18:54 +v2.0.8;multi;100;31623;;60390;24233;616216;155535;622140;1219;282.210077287;2021-12-08@11:24:44 +v2.0.8;multi;100;31623;;60390;24233;610368;154137;616548;1346;307.148932535;2021-12-08@11:29:55 +v2.0.8;imex;100;31623;;60390;24233;344400;87650;350600;756;171.078737269;2021-12-08@11:33:53 +v2.0.8;imex;100;31623;;60390;24233;344428;87661;350644;734;166.334170376;2021-12-08@11:36:42 +v2.0.8;imex;100;31623;;60390;24233;344884;87749;350996;734;166.429060944;2021-12-08@11:39:30 +v2.0.8;imex;100;31623;;60390;24233;343984;87595;350380;729;165.233216802;2021-12-08@11:42:18 +v2.0.8;imex;100;31623;;60390;24233;344632;87697;350788;733;166.079004371;2021-12-08@11:45:06 +v2.0.8;imex;100;31623;;60390;24233;344520;87649;350596;728;165.027959638;2021-12-08@11:47:53 +v2.0.8;imex;100;31623;;60390;24233;344444;87679;350716;734;166.313666528;2021-12-08@11:50:42 +v2.0.8;imex;100;31623;;60390;24233;344544;87714;350856;742;168.042208415;2021-12-08@11:53:33 +v2.0.8;single;100;31623;;60390;24233;32512;9612;38448;775;175.458575666;2021-12-08@11:57:31 +v2.0.8;single;100;31623;;60390;24233;31908;9567;38268;761;172.604947657;2021-12-08@12:00:25 +v2.0.8;single;100;31623;;60390;24233;32140;9563;38252;752;170.498530623;2021-12-08@12:03:17 +v2.0.8;single;100;31623;;60390;24233;31796;9534;38136;756;171.451913360;2021-12-08@12:06:10 +v2.0.8;single;100;31623;;60390;24233;32340;9575;38300;761;172.493975304;2021-12-08@12:09:04 +v2.0.8;single;100;31623;;60390;24233;32116;9537;38148;760;172.280034669;2021-12-08@12:11:58 +v2.0.8;single;100;31623;;60390;24233;32516;9591;38364;763;172.952830973;2021-12-08@12:14:53 +v2.0.8;single;100;31623;;60390;24233;32552;9629;38516;788;178.451672749;2021-12-08@12:17:53 +v2.0.8;mulimex;100;31623;;60390;24233;916628;230687;922748;1425;325.732562101;2021-12-08@12:24:19 +v2.0.8;mulimex;100;31623;;60390;24233;932556;234656;938624;1161;267.619864333;2021-12-08@12:29:55 +v2.0.8;mulimex;100;31623;;60390;24233;926924;233216;932864;1461;333.811069823;2021-12-08@12:35:32 +v2.0.8;mulimex;100;31623;;60390;24233;929632;233976;935904;1100;253.630645455;2021-12-08@12:40:54 +v2.0.8;mulimex;100;31623;;60390;24233;928928;233787;935148;1260;290.833195027;2021-12-08@12:46:53 +v2.0.8;mulimex;100;31623;;60390;24233;913212;229879;919516;1492;341.626259673;2021-12-08@12:52:38 +v2.0.8;mulimex;100;31623;;60390;24233;917728;230936;923744;1384;315.460346912;2021-12-08@12:57:56 +v2.0.8;mulimex;100;31623;;60390;24233;911564;229425;917700;1529;349.187945591;2021-12-08@13:03:49 +v2.0.8;multi;178;31623;;60806;24427;1036860;260700;1042800;2811;646.972937321;2021-12-08@13:16:10 +v2.0.8;multi;178;31623;;60806;24427;1115996;280495;1121980;2035;468.615300090;2021-12-08@13:30:04 +v2.0.8;multi;178;31623;;60806;24427;1057076;265842;1063368;2272;525.077433498;2021-12-08@13:41:04 +v2.0.8;multi;178;31623;;60806;24427;1054104;265004;1060016;1940;445.469762335;2021-12-08@13:50:44 +v2.0.8;multi;178;31623;;60806;24427;1086820;273284;1093136;2527;576.678661107;2021-12-08@14:00:24 +v2.0.8;multi;178;31623;;60806;24427;1059064;266319;1065276;2347;539.411727796;2021-12-08@14:11:38 +v2.0.8;multi;178;31623;;60806;24427;1081752;271998;1087992;2553;581.829601419;2021-12-08@14:21:23 +v2.0.8;multi;178;31623;;60806;24427;1036788;260791;1043164;2642;609.570901367;2021-12-08@14:32:43 +v2.0.8;imex;178;31623;;60806;24427;575180;145340;581360;1481;336.651069271;2021-12-08@14:40:17 +v2.0.8;imex;178;31623;;60806;24427;575636;145445;581780;1338;304.413725084;2021-12-08@14:45:24 +v2.0.8;imex;178;31623;;60806;24427;575072;145286;581144;1542;354.070235998;2021-12-08@14:51:21 +v2.0.8;imex;178;31623;;60806;24427;575116;145294;581176;1355;308.168328292;2021-12-08@14:56:31 +v2.0.8;imex;178;31623;;60806;24427;575432;145462;581848;1326;301.578053363;2021-12-08@15:01:35 +v2.0.8;imex;178;31623;;60806;24427;575928;145487;581948;1500;341.408838533;2021-12-08@15:07:19 +v2.0.8;imex;178;31623;;60806;24427;574992;145311;581244;1420;322.526401057;2021-12-08@15:12:44 +v2.0.8;imex;178;31623;;60806;24427;575280;145320;581280;1414;321.301434860;2021-12-08@15:18:08 +v2.0.8;single;178;31623;;60806;24427;38084;11030;44120;1515;345.289206830;2021-12-08@15:25:40 +v2.0.8;single;178;31623;;60806;24427;37424;10899;43596;1477;336.220475529;2021-12-08@15:31:18 +v2.0.8;single;178;31623;;60806;24427;37612;10900;43600;1598;366.610944496;2021-12-08@15:37:28 +v2.0.8;single;178;31623;;60806;24427;37036;10827;43308;1398;318.190836344;2021-12-08@15:42:48 +v2.0.8;single;178;31623;;60806;24427;37712;10921;43684;1415;321.954921245;2021-12-08@15:48:12 +v2.0.8;single;178;31623;;60806;24427;37636;10960;43840;1400;318.999584610;2021-12-08@15:53:33 +v2.0.8;single;178;31623;;60806;24427;37396;10804;43216;1373;312.399063378;2021-12-08@15:58:47 +v2.0.8;single;178;31623;;60806;24427;37112;10810;43240;1430;325.478678293;2021-12-08@16:04:15 +v2.0.8;mulimex;178;31623;;60806;24427;1653752;414960;1659840;2441;555.587873145;2021-12-08@16:15:18 +v2.0.8;mulimex;178;31623;;60806;24427;1593268;399833;1599332;2624;604.267722703;2021-12-08@16:25:27 +v2.0.8;mulimex;178;31623;;60806;24427;1587784;398538;1594152;2413;554.220212078;2021-12-08@16:34:46 +v2.0.8;mulimex;178;31623;;60806;24427;1557124;390857;1563428;2570;590.778817550;2021-12-08@16:44:42 +v2.0.8;mulimex;178;31623;;60806;24427;1597168;400850;1603400;2602;598.768923543;2021-12-08@16:54:46 +v2.0.8;mulimex;178;31623;;60806;24427;1599004;401244;1604976;1649;379.945126131;2021-12-08@17:04:46 +v2.0.8;mulimex;178;31623;;60806;24427;1584080;397580;1590320;2595;594.474870469;2021-12-08@17:14:45 +v2.0.8;mulimex;178;31623;;60806;24427;1591412;399453;1597812;2635;602.206602542;2021-12-08@17:24:53 +v2.0.8;multi;316;31623;;61364;24592;1817408;455885;1823540;4079;940.025651339;2021-12-08@17:43:31 +v2.0.8;multi;316;31623;;61364;24592;1817380;455894;1823576;4088;940.532223102;2021-12-08@17:59:17 +v2.0.8;multi;316;31623;;61364;24592;1817440;455922;1823688;4108;944.262149929;2021-12-08@18:15:07 +v2.0.8;multi;316;31623;;61364;24592;1817604;455939;1823756;4096;940.907790885;2021-12-08@18:30:54 +v2.0.8;multi;316;31623;;61364;24592;1817448;455894;1823576;4060;938.543177206;2021-12-08@18:46:39 +v2.0.8;multi;316;31623;;61364;24592;1817976;456045;1824180;4107;945.413934584;2021-12-08@19:02:31 +v2.0.8;multi;316;31623;;61364;24592;1817576;455937;1823748;4079;940.798451355;2021-12-08@19:18:17 +v2.0.8;multi;316;31623;;61364;24592;1847160;463294;1853176;3885;890.428946122;2021-12-08@19:44:11 +v2.0.8;imex;316;31623;;61364;24592;984560;247723;990892;2788;637.817606902;2021-12-08@19:58:29 +v2.0.8;imex;316;31623;;61364;24592;983444;247418;989672;2572;588.050905900;2021-12-08@20:08:21 +v2.0.8;imex;316;31623;;61364;24592;984100;247575;990300;2601;592.756215016;2021-12-08@20:18:16 +v2.0.8;imex;316;31623;;61364;24592;983544;247402;989608;2595;592.259221360;2021-12-08@20:28:11 +v2.0.8;imex;316;31623;;61364;24592;983388;247426;989704;2538;582.954021881;2021-12-08@20:37:56 +v2.0.8;imex;316;31623;;61364;24592;983320;247400;989600;2879;660.381515865;2021-12-08@20:48:59 +v2.0.8;imex;316;31623;;61364;24592;984616;247705;990820;2628;600.677377561;2021-12-08@20:59:02 +v2.0.8;imex;316;31623;;61364;24592;984460;247649;990596;2436;556.008854656;2021-12-08@21:08:21 +v2.0.8;single;316;31623;;61364;24592;43508;12472;49888;2884;657.314033765;2021-12-08@21:22:35 +v2.0.8;single;316;31623;;61364;24592;44604;12646;50584;2721;621.343520425;2021-12-08@21:32:58 +v2.0.8;single;316;31623;;61364;24592;44628;12665;50660;2978;684.008287181;2021-12-08@21:44:24 +v2.0.8;single;316;31623;;61364;24592;44908;12694;50776;2821;646.099908630;2021-12-08@21:55:13 +v2.0.8;single;316;31623;;61364;24592;44468;12599;50396;2640;601.153770946;2021-12-08@22:05:16 +v2.0.8;single;316;31623;;61364;24592;44252;12564;50256;2851;650.704807431;2021-12-08@23:00:27 +v2.0.8;single;316;31623;;61364;24592;43760;12441;49764;2778;633.012615648;2021-12-08@23:11:03 +v2.0.8;single;316;31623;;61364;24592;43780;12507;50028;2715;622.376185441;2021-12-08@23:21:30 +v2.0.8;mulimex;316;31623;;61364;24592;2738700;686198;2744792;4099;943.279600625;2021-12-08@23:40:30 +v2.0.8;mulimex;316;31623;;61364;24592;2738684;686236;2744944;4114;946.200502053;2021-12-08@23:56:25 +v2.0.8;mulimex;316;31623;;61364;24592;2740232;686610;2746440;4118;947.239619301;2021-12-09@00:12:22 +v2.0.8;mulimex;316;31623;;61364;24592;2740516;686654;2746616;4092;941.296457258;2021-12-09@00:28:13 +v2.0.8;mulimex;316;31623;;61364;24592;2739040;686269;2745076;4110;946.010538983;2021-12-09@00:44:08 +v2.0.8;mulimex;316;31623;;61364;24592;2738760;686210;2744840;4095;947.546542409;2021-12-09@01:00:05 +v2.0.8;mulimex;316;31623;;61364;24592;2739056;686287;2745148;4107;945.049324731;2021-12-09@01:16:00 +v2.0.8;mulimex;316;31623;;61364;24592;2739096;686292;2745168;3286;759.732133419;2021-12-09@01:31:57 +v2.0.8;multi;10;56234;;89496;27046;142696;37222;148888;187;43.076460176;2021-12-09@01:34:17 +v2.0.8;multi;10;56234;;89496;27046;142692;37217;148868;194;44.610942153;2021-12-09@01:35:05 +v2.0.8;multi;10;56234;;89496;27046;142788;37191;148764;193;44.413673811;2021-12-09@01:35:53 +v2.0.8;multi;10;56234;;89496;27046;142752;37212;148848;193;44.373585357;2021-12-09@01:36:41 +v2.0.8;multi;10;56234;;89496;27046;142580;37212;148848;193;44.385711186;2021-12-09@01:37:29 +v2.0.8;multi;10;56234;;89496;27046;142560;37221;148884;193;44.464580964;2021-12-09@01:38:17 +v2.0.8;multi;10;56234;;89496;27046;142652;37204;148816;193;44.421442437;2021-12-09@01:39:05 +v2.0.8;multi;10;56234;;89496;27046;142916;37200;148800;193;44.419207089;2021-12-09@01:39:53 +v2.0.8;imex;10;56234;;89496;27046;105644;27935;111740;100;22.747002538;2021-12-09@01:40:30 +v2.0.8;imex;10;56234;;89496;27046;105388;27918;111672;105;23.876284289;2021-12-09@01:40:57 +v2.0.8;imex;10;56234;;89496;27046;105468;27907;111628;102;23.225126913;2021-12-09@01:41:24 +v2.0.8;imex;10;56234;;89496;27046;105416;27891;111564;123;27.996070618;2021-12-09@01:41:56 +v2.0.8;imex;10;56234;;89496;27046;105440;27907;111628;106;24.117040667;2021-12-09@01:42:24 +v2.0.8;imex;10;56234;;89496;27046;105388;27937;111748;106;24.162428072;2021-12-09@01:42:51 +v2.0.8;imex;10;56234;;89496;27046;105340;27892;111568;101;22.983686177;2021-12-09@01:43:18 +v2.0.8;imex;10;56234;;89496;27046;105516;27945;111780;104;23.691121022;2021-12-09@01:43:46 +v2.0.8;single;10;56234;;89496;27046;34692;10169;40676;102;23.223283493;2021-12-09@01:44:23 +v2.0.8;single;10;56234;;89496;27046;34636;10185;40740;109;24.834617029;2021-12-09@01:44:51 +v2.0.8;single;10;56234;;89496;27046;34708;10197;40788;108;24.619898505;2021-12-09@01:45:19 +v2.0.8;single;10;56234;;89496;27046;34840;10184;40736;109;24.840966122;2021-12-09@01:45:48 +v2.0.8;single;10;56234;;89496;27046;34500;10189;40756;110;25.044417012;2021-12-09@01:46:17 +v2.0.8;single;10;56234;;89496;27046;34560;10245;40980;103;23.456869516;2021-12-09@01:46:44 +v2.0.8;single;10;56234;;89496;27046;34680;10208;40832;105;23.928176334;2021-12-09@01:47:11 +v2.0.8;single;10;56234;;89496;27046;34760;10202;40808;106;24.150051670;2021-12-09@01:47:39 +v2.0.8;mulimex;10;56234;;89496;27046;189712;49024;196096;188;43.298753541;2021-12-09@01:48:36 +v2.0.8;mulimex;10;56234;;89496;27046;189896;48995;195980;194;44.652787154;2021-12-09@01:49:24 +v2.0.8;mulimex;10;56234;;89496;27046;189904;49007;196028;194;44.645481839;2021-12-09@01:50:13 +v2.0.8;mulimex;10;56234;;89496;27046;189740;49000;196000;195;44.824372111;2021-12-09@01:51:01 +v2.0.8;mulimex;10;56234;;89496;27046;189940;49022;196088;194;44.643335669;2021-12-09@01:51:50 +v2.0.8;mulimex;10;56234;;89496;27046;190076;49002;196008;194;44.636194098;2021-12-09@01:52:38 +v2.0.8;mulimex;10;56234;;89496;27046;189788;49011;196044;195;44.867962880;2021-12-09@01:53:27 +v2.0.8;mulimex;10;56234;;89496;27046;189940;49000;196000;194;44.655484407;2021-12-09@01:54:15 +v2.0.8;multi;18;56234;;97395;27048;220856;56793;227172;365;84.242500491;2021-12-09@01:55:56 +v2.0.8;multi;18;56234;;97395;27048;220944;56797;227188;370;85.299469540;2021-12-09@01:57:25 +v2.0.8;multi;18;56234;;97395;27048;221188;56814;227256;371;85.548376086;2021-12-09@01:58:55 +v2.0.8;multi;18;56234;;97395;27048;221100;56785;227140;370;85.294809720;2021-12-09@02:00:24 +v2.0.8;multi;18;56234;;97395;27048;221248;56814;227256;370;85.313751432;2021-12-09@02:01:53 +v2.0.8;multi;18;56234;;97395;27048;220880;56800;227200;370;85.418382552;2021-12-09@02:03:23 +v2.0.8;multi;18;56234;;97395;27048;220896;56802;227208;371;85.577290839;2021-12-09@02:04:52 +v2.0.8;multi;18;56234;;97395;27048;220920;56825;227300;371;85.487229091;2021-12-09@02:06:22 +v2.0.8;imex;18;56234;;97395;27048;138196;36107;144428;174;39.863860580;2021-12-09@02:07:19 +v2.0.8;imex;18;56234;;97395;27048;138404;36103;144412;177;40.446869642;2021-12-09@02:08:03 +v2.0.8;imex;18;56234;;97395;27048;138384;36080;144320;170;38.827586332;2021-12-09@02:08:46 +v2.0.8;imex;18;56234;;97395;27048;138184;36077;144308;173;39.501912238;2021-12-09@02:09:29 +v2.0.8;imex;18;56234;;97395;27048;138212;36113;144452;167;38.114783504;2021-12-09@02:10:11 +v2.0.8;imex;18;56234;;97395;27048;138320;36096;144384;172;39.300459014;2021-12-09@02:10:54 +v2.0.8;imex;18;56234;;97395;27048;138060;36080;144320;172;39.272846939;2021-12-09@02:11:37 +v2.0.8;imex;18;56234;;97395;27048;138360;36110;144440;172;39.298479669;2021-12-09@02:12:20 +v2.0.8;single;18;56234;;97395;27048;38092;11076;44304;177;40.405539648;2021-12-09@02:13:17 +v2.0.8;single;18;56234;;97395;27048;38104;11088;44352;179;40.849114995;2021-12-09@02:14:02 +v2.0.8;single;18;56234;;97395;27048;38236;11085;44340;179;40.876602451;2021-12-09@02:14:46 +v2.0.8;single;18;56234;;97395;27048;38228;11092;44368;171;39.087449223;2021-12-09@02:15:29 +v2.0.8;single;18;56234;;97395;27048;38348;11091;44364;177;40.396962129;2021-12-09@02:16:13 +v2.0.8;single;18;56234;;97395;27048;38184;11086;44344;219;49.987324884;2021-12-09@02:17:07 +v2.0.8;single;18;56234;;97395;27048;38188;11098;44392;177;40.386130004;2021-12-09@02:17:51 +v2.0.8;single;18;56234;;97395;27048;38240;11095;44380;179;40.872568173;2021-12-09@02:18:35 +v2.0.8;mulimex;18;56234;;97395;27048;294980;75268;301072;368;84.825866604;2021-12-09@02:20:17 +v2.0.8;mulimex;18;56234;;97395;27048;294844;75261;301044;378;87.147814315;2021-12-09@02:21:48 +v2.0.8;mulimex;18;56234;;97395;27048;295148;75264;301056;372;85.749551801;2021-12-09@02:23:18 +v2.0.8;mulimex;18;56234;;97395;27048;294764;75250;301000;373;86.102422714;2021-12-09@02:24:48 +v2.0.8;mulimex;18;56234;;97395;27048;294928;75268;301072;372;85.750423733;2021-12-09@02:26:18 +v2.0.8;mulimex;18;56234;;97395;27048;294912;75279;301116;376;86.669508184;2021-12-09@02:27:48 +v2.0.8;mulimex;18;56234;;97395;27048;294756;75265;301060;372;85.807379445;2021-12-09@02:29:18 +v2.0.8;mulimex;18;56234;;97395;27048;294792;75240;300960;372;85.856400355;2021-12-09@02:30:48 +v2.0.8;multi;32;56234;;102105;27070;352820;89735;358940;679;156.724959930;2021-12-09@02:33:47 +v2.0.8;multi;32;56234;;102105;27070;352900;89768;359072;685;158.022130913;2021-12-09@02:36:29 +v2.0.8;multi;32;56234;;102105;27070;352724;89723;358892;682;157.578659868;2021-12-09@02:39:11 +v2.0.8;multi;32;56234;;102105;27070;352744;89750;359000;684;157.865235335;2021-12-09@02:41:53 +v2.0.8;multi;32;56234;;102105;27070;352952;89751;359004;683;157.586056618;2021-12-09@02:44:35 +v2.0.8;multi;32;56234;;102105;27070;352916;89739;358956;684;157.860741247;2021-12-09@02:47:17 +v2.0.8;multi;32;56234;;102105;27070;352768;89712;358848;684;157.830419550;2021-12-09@02:49:59 +v2.0.8;multi;32;56234;;102105;27070;353076;89768;359072;685;157.965420749;2021-12-09@02:52:41 +v2.0.8;imex;32;56234;;102105;27070;188048;48630;194520;301;68.873587314;2021-12-09@02:54:14 +v2.0.8;imex;32;56234;;102105;27070;188232;48612;194448;296;67.775865835;2021-12-09@02:55:25 +v2.0.8;imex;32;56234;;102105;27070;188232;48635;194540;296;67.713965671;2021-12-09@02:56:37 +v2.0.8;imex;32;56234;;102105;27070;188308;48595;194380;294;67.300473996;2021-12-09@02:57:48 +v2.0.8;imex;32;56234;;102105;27070;188196;48604;194416;297;67.966663623;2021-12-09@02:59:00 +v2.0.8;imex;32;56234;;102105;27070;188212;48593;194372;299;68.506095881;2021-12-09@03:00:12 +v2.0.8;imex;32;56234;;102105;27070;188260;48609;194436;296;67.765148938;2021-12-09@03:01:23 +v2.0.8;imex;32;56234;;102105;27070;188300;48624;194496;295;67.689075853;2021-12-09@03:02:35 +v2.0.8;single;32;56234;;102105;27070;41008;11845;47380;314;71.812278825;2021-12-09@03:04:09 +v2.0.8;single;32;56234;;102105;27070;41468;11856;47424;307;70.206675400;2021-12-09@03:05:23 +v2.0.8;single;32;56234;;102105;27070;41492;11852;47408;308;70.420815006;2021-12-09@03:06:37 +v2.0.8;single;32;56234;;102105;27070;41152;11852;47408;307;70.215297931;2021-12-09@03:07:51 +v2.0.8;single;32;56234;;102105;27070;41376;11871;47484;303;69.291348679;2021-12-09@03:09:04 +v2.0.8;single;32;56234;;102105;27070;41360;11840;47360;305;69.714203976;2021-12-09@03:10:18 +v2.0.8;single;32;56234;;102105;27070;41244;11849;47396;311;71.085526216;2021-12-09@03:11:32 +v2.0.8;single;32;56234;;102105;27070;41204;11869;47476;318;72.743029581;2021-12-09@03:12:49 +v2.0.8;mulimex;32;56234;;102105;27070;471504;119440;477760;682;157.544319694;2021-12-09@03:15:49 +v2.0.8;mulimex;32;56234;;102105;27070;471784;119493;477972;687;158.368983008;2021-12-09@03:18:32 +v2.0.8;mulimex;32;56234;;102105;27070;471812;119469;477876;687;158.386644390;2021-12-09@03:21:14 +v2.0.8;mulimex;32;56234;;102105;27070;471448;119446;477784;686;158.194186036;2021-12-09@03:23:57 +v2.0.8;mulimex;32;56234;;102105;27070;471520;119452;477808;686;158.401780461;2021-12-09@03:26:40 +v2.0.8;mulimex;32;56234;;102105;27070;471372;119407;477628;685;158.096997037;2021-12-09@03:29:22 +v2.0.8;mulimex;32;56234;;102105;27070;471640;119440;477760;687;158.530682276;2021-12-09@03:32:05 +v2.0.8;mulimex;32;56234;;102105;27070;471584;119483;477932;687;158.541124463;2021-12-09@03:34:48 +v2.0.8;multi;56;56234;;104619;27074;573900;145002;580008;1215;280.398077334;2021-12-09@03:40:00 +v2.0.8;multi;56;56234;;104619;27074;574072;145022;580088;1221;281.968479587;2021-12-09@03:44:47 +v2.0.8;multi;56;56234;;104619;27074;574052;145064;580256;1219;281.554420970;2021-12-09@03:49:33 +v2.0.8;multi;56;56234;;104619;27074;573896;145016;580064;1222;282.031923643;2021-12-09@03:54:20 +v2.0.8;multi;56;56234;;104619;27074;573760;145016;580064;1224;281.715512892;2021-12-09@03:59:06 +v2.0.8;multi;56;56234;;104619;27074;573972;145030;580120;1221;281.836097055;2021-12-09@04:03:53 +v2.0.8;multi;56;56234;;104619;27074;573812;144986;579944;1223;281.716253380;2021-12-09@04:08:39 +v2.0.8;multi;56;56234;;104619;27074;573896;144994;579976;1224;281.764882679;2021-12-09@04:13:26 +v2.0.8;imex;56;56234;;104619;27074;269332;68886;275544;535;122.765102225;2021-12-09@04:16:06 +v2.0.8;imex;56;56234;;104619;27074;269336;68875;275500;517;118.646660092;2021-12-09@04:18:08 +v2.0.8;imex;56;56234;;104619;27074;269236;68805;275220;512;117.515803180;2021-12-09@04:20:10 +v2.0.8;imex;56;56234;;104619;27074;269028;68830;275320;528;121.109350145;2021-12-09@04:22:15 +v2.0.8;imex;56;56234;;104619;27074;269552;68896;275584;514;117.963522291;2021-12-09@04:24:16 +v2.0.8;imex;56;56234;;104619;27074;268996;68817;275268;512;117.517070977;2021-12-09@04:26:18 +v2.0.8;imex;56;56234;;104619;27074;269356;68844;275376;511;117.291517129;2021-12-09@04:28:19 +v2.0.8;imex;56;56234;;104619;27074;269040;68836;275344;506;116.166008720;2021-12-09@04:30:19 +v2.0.8;single;56;56234;;104619;27074;44348;12626;50504;547;125.502251256;2021-12-09@04:33:01 +v2.0.8;single;56;56234;;104619;27074;44324;12620;50480;531;121.833247022;2021-12-09@04:35:07 +v2.0.8;single;56;56234;;104619;27074;44236;12633;50532;533;122.328728488;2021-12-09@04:37:13 +v2.0.8;single;56;56234;;104619;27074;44172;12622;50488;531;121.834201032;2021-12-09@04:39:18 +v2.0.8;single;56;56234;;104619;27074;44068;12602;50408;533;122.345329114;2021-12-09@04:41:24 +v2.0.8;single;56;56234;;104619;27074;44580;12638;50552;539;123.687200017;2021-12-09@04:43:31 +v2.0.8;single;56;56234;;104619;27074;44320;12611;50444;536;123.013284308;2021-12-09@04:45:38 +v2.0.8;single;56;56234;;104619;27074;44476;12642;50568;537;123.200488144;2021-12-09@04:47:45 +v2.0.8;mulimex;56;56234;;104619;27074;769032;193772;775088;1220;281.545428608;2021-12-09@04:53:03 +v2.0.8;mulimex;56;56234;;104619;27074;768960;193794;775176;1224;282.635176476;2021-12-09@04:57:51 +v2.0.8;mulimex;56;56234;;104619;27074;768900;193760;775040;1223;282.262223337;2021-12-09@05:02:38 +v2.0.8;mulimex;56;56234;;104619;27074;768984;193788;775152;1224;282.584243331;2021-12-09@05:07:26 +v2.0.8;mulimex;56;56234;;104619;27074;769200;193764;775056;1224;282.551595087;2021-12-09@05:12:14 +v2.0.8;mulimex;56;56234;;104619;27074;769172;193781;775124;1223;282.503140269;2021-12-09@05:17:01 +v2.0.8;mulimex;56;56234;;104619;27074;769276;193781;775124;1223;282.592751664;2021-12-09@05:21:49 +v2.0.8;mulimex;56;56234;;104619;27074;769024;193813;775252;1225;282.793893968;2021-12-09@05:26:37 +v2.0.8;multi;100;56234;;106396;27105;977892;246003;984012;2267;523.926696862;2021-12-09@05:36:14 +v2.0.8;multi;100;56234;;106396;27105;978152;246036;984144;2246;519.895864941;2021-12-09@05:45:00 +v2.0.8;multi;100;56234;;106396;27105;978028;245997;983988;2253;520.939267948;2021-12-09@05:53:46 +v2.0.8;multi;100;56234;;106396;27105;977692;245961;983844;2884;667.416313194;2021-12-09@06:05:00 +v2.0.8;multi;100;56234;;106396;27105;978300;246075;984300;2235;517.771739337;2021-12-09@06:13:43 +v2.0.8;multi;100;56234;;106396;27105;978076;246075;984300;2506;584.021092673;2021-12-09@06:23:33 +v2.0.8;multi;100;56234;;106396;27105;977724;246026;984104;2244;520.745145256;2021-12-09@06:32:19 +v2.0.8;multi;100;56234;;106396;27105;977984;246053;984212;2254;522.625676561;2021-12-09@06:41:08 +v2.0.8;imex;100;56234;;106396;27105;415120;105351;421404;954;219.437474200;2021-12-09@06:45:52 +v2.0.8;imex;100;56234;;106396;27105;414796;105291;421164;911;209.829620794;2021-12-09@06:49:26 +v2.0.8;imex;100;56234;;106396;27105;414812;105273;421092;904;208.213314639;2021-12-09@06:52:58 +v2.0.8;imex;100;56234;;106396;27105;414948;105270;421080;914;210.435671631;2021-12-09@06:56:33 +v2.0.8;imex;100;56234;;106396;27105;414520;105205;420820;907;208.992456957;2021-12-09@07:00:06 +v2.0.8;imex;100;56234;;106396;27105;415104;105332;421328;914;210.359145899;2021-12-09@07:03:40 +v2.0.8;imex;100;56234;;106396;27105;414936;105244;420976;916;210.990511293;2021-12-09@07:07:15 +v2.0.8;imex;100;56234;;106396;27105;415236;105322;421288;924;212.864787628;2021-12-09@07:10:52 +v2.0.8;single;100;56234;;106396;27105;48900;13689;54756;988;227.390902194;2021-12-09@07:15:39 +v2.0.8;single;100;56234;;106396;27105;48656;13705;54820;947;218.178643096;2021-12-09@07:19:21 +v2.0.8;single;100;56234;;106396;27105;48380;13698;54792;953;219.684324595;2021-12-09@07:23:04 +v2.0.8;single;100;56234;;106396;27105;48636;13725;54900;964;221.989491097;2021-12-09@07:26:50 +v2.0.8;single;100;56234;;106396;27105;48760;13733;54932;947;218.187075110;2021-12-09@07:30:32 +v2.0.8;single;100;56234;;106396;27105;48632;13708;54832;942;216.990504838;2021-12-09@07:34:13 +v2.0.8;single;100;56234;;106396;27105;48592;13712;54848;950;219.023368140;2021-12-09@07:37:55 +v2.0.8;single;100;56234;;106396;27105;48516;13691;54764;950;218.824297595;2021-12-09@07:41:38 +v2.0.8;mulimex;100;56234;;106396;27105;1312932;329766;1319064;2241;519.681333288;2021-12-09@07:51:17 +v2.0.8;mulimex;100;56234;;106396;27105;1312936;329759;1319036;2261;524.523725959;2021-12-09@08:00:08 +v2.0.8;mulimex;100;56234;;106396;27105;1316704;330724;1322896;2271;526.067791525;2021-12-09@08:09:00 +v2.0.8;mulimex;100;56234;;106396;27105;1313008;329725;1318900;2287;529.322695333;2021-12-09@08:17:56 +v2.0.8;mulimex;100;56234;;106396;27105;1312720;329748;1318992;2261;523.545375331;2021-12-09@08:26:46 +v2.0.8;mulimex;100;56234;;106396;27105;1312852;329770;1319080;2253;522.250997383;2021-12-09@08:35:34 +v2.0.8;mulimex;100;56234;;106396;27105;1312464;329681;1318724;2317;536.325125812;2021-12-09@08:44:37 +v2.0.8;mulimex;100;56234;;106396;27105;1313072;329739;1318956;1224;281.675648409;2021-12-09@08:53:27 +v2.0.8;multi;178;56234;;107469;27173;1699680;426462;1705848;4006;933.640679377;2021-12-09@09:10:35 +v2.0.8;multi;178;56234;;107469;27173;1691980;424509;1698036;4202;975.480960960;2021-12-09@09:30:38 +v2.0.8;multi;178;56234;;107469;27173;1692056;424544;1698176;4393;1022.885671784;2021-12-09@09:47:48 +v2.0.8;multi;178;56234;;107469;27173;1692112;424567;1698268;4216;980.464629212;2021-12-09@10:04:17 +v2.0.8;multi;178;56234;;107469;27173;1726688;433203;1732812;4271;990.155722730;2021-12-09@10:20:55 +v2.0.8;multi;178;56234;;107469;27173;1704360;427584;1710336;4171;967.545148543;2021-12-09@10:37:10 +v2.0.8;multi;178;56234;;107469;27173;1692696;424629;1698516;4215;979.130552574;2021-12-09@10:53:37 +v2.0.8;multi;178;56234;;107469;27173;1722412;432171;1728684;4198;972.045807733;2021-12-09@11:09:57 +v2.0.8;imex;178;56234;;107469;27173;670004;169039;676156;1776;410.361222060;2021-12-09@11:18:44 +v2.0.8;imex;178;56234;;107469;27173;671636;169440;677760;1800;415.415158479;2021-12-09@11:25:44 +v2.0.8;imex;178;56234;;107469;27173;669736;169005;676020;1682;388.352625214;2021-12-09@11:32:17 +v2.0.8;imex;178;56234;;107469;27173;669652;168949;675796;1636;378.256284325;2021-12-09@11:38:39 +v2.0.8;imex;178;56234;;107469;27173;670184;169133;676532;1638;378.576819179;2021-12-09@11:45:02 +v2.0.8;imex;178;56234;;107469;27173;670484;169175;676700;1685;389.622863147;2021-12-09@11:51:36 +v2.0.8;imex;178;56234;;107469;27173;670116;169067;676268;1645;380.161810258;2021-12-09@11:58:00 +v2.0.8;imex;178;56234;;107469;27173;670572;169206;676824;1670;386.420856463;2021-12-09@12:04:31 +v2.0.8;single;178;56234;;107469;27173;54052;14997;59988;1852;426.861549282;2021-12-09@12:13:23 +v2.0.8;single;178;56234;;107469;27173;53752;15012;60048;1713;395.364411427;2021-12-09@12:20:02 +v2.0.8;single;178;56234;;107469;27173;53760;14989;59956;1733;400.594933515;2021-12-09@12:26:46 +v2.0.8;single;178;56234;;107469;27173;53684;14977;59908;1694;391.627934647;2021-12-09@12:33:22 +v2.0.8;single;178;56234;;107469;27173;53544;14904;59616;1709;394.959460176;2021-12-09@12:40:00 +v2.0.8;single;178;56234;;107469;27173;53800;15024;60096;1722;397.759941738;2021-12-09@12:46:42 +v2.0.8;single;178;56234;;107469;27173;53860;15046;60184;1692;391.594794102;2021-12-09@12:53:17 +v2.0.8;single;178;56234;;107469;27173;53880;14950;59800;1692;390.680971967;2021-12-09@12:59:51 +v2.0.8;mulimex;178;56234;;107469;27173;2338648;586215;2344860;4023;936.017445147;2021-12-09@13:17:13 +v2.0.8;mulimex;178;56234;;107469;27173;2337864;585988;2343952;4041;940.990618366;2021-12-09@13:33:02 +v2.0.8;mulimex;178;56234;;107469;27173;2277424;570870;2283480;4039;936.371529079;2021-12-09@13:48:47 +v2.0.8;mulimex;178;56234;;107469;27173;2290092;574028;2296112;3700;865.724553463;2021-12-09@14:07:01 +v2.0.8;mulimex;178;56234;;107469;27173;2280928;571721;2286884;4165;975.368066359;2021-12-09@14:23:25 +v2.0.8;mulimex;178;56234;;107469;27173;2275232;570339;2281356;4037;942.583291360;2021-12-09@14:39:16 +v2.0.8;mulimex;178;56234;;107469;27173;2275076;570335;2281340;4059;948.040280054;2021-12-09@14:55:13 +v2.0.8;mulimex;178;56234;;107469;27173;2317016;580747;2322988;4008;934.728786925;2021-12-09@15:10:57 +v2.0.8;multi;316;56234;;108134;27288;2982428;747178;2988712;7069;1645.763423739;2021-12-09@15:41:16 +v2.0.8;multi;316;56234;;108134;27288;3021724;756982;3027928;7376;1723.483024098;2021-12-09@16:10:11 +v2.0.8;multi;316;56234;;108134;27288;2975452;745339;2981356;7056;1651.955468093;2021-12-09@16:37:54 +v2.0.8;multi;316;56234;;108134;27288;2954964;740284;2961136;7524;1755.771361504;2021-12-09@17:15:26 +v2.0.8;multi;316;56234;;108134;27288;3039412;761385;3045540;6925;1620.905531083;2021-12-09@17:52:38 +v2.0.8;multi;316;56234;;108134;27288;2954708;740192;2960768;7259;1695.924555217;2021-12-09@18:37:24 +v2.0.8;multi;316;56234;;108134;27288;2954412;740119;2960476;7262;1695.996853746;2021-12-09@19:05:52 +v2.0.8;multi;316;56234;;108134;27288;3008228;753575;3014300;6266;1465.627322403;2021-12-09@19:34:09 +v2.0.8;imex;316;56234;;108134;27288;1120464;281684;1126736;3427;796.432706981;2021-12-09@19:51:03 +v2.0.8;imex;316;56234;;108134;27288;1121164;281804;1127216;3342;772.079363080;2021-12-09@20:04:00 +v2.0.8;imex;316;56234;;108134;27288;1120360;281661;1126644;3204;742.032109509;2021-12-09@20:16:27 +v2.0.8;imex;316;56234;;108134;27288;1121228;281838;1127352;3501;810.484132004;2021-12-09@20:30:02 +v2.0.8;imex;316;56234;;108134;27288;1120212;281561;1126244;3423;790.724751814;2021-12-09@20:43:18 +v2.0.8;imex;316;56234;;108134;27288;1121304;281817;1127268;3132;724.962736472;2021-12-09@20:55:28 +v2.0.8;imex;316;56234;;108134;27288;1120072;281579;1126316;3276;756.808201315;2021-12-09@21:08:10 +v2.0.8;imex;316;56234;;108134;27288;1121312;281887;1127548;3423;791.096044598;2021-12-09@21:21:26 +v2.0.8;single;316;56234;;108134;27288;63192;17328;69312;3526;820.375490660;2021-12-09@21:38:21 +v2.0.8;single;316;56234;;108134;27288;62120;17071;68284;3767;872.360939351;2021-12-09@21:52:58 +v2.0.8;single;316;56234;;108134;27288;64692;17647;70588;3222;746.195466983;2021-12-09@22:05:28 +v2.0.8;single;316;56234;;108134;27288;64096;17519;70076;3208;743.262772134;2021-12-09@22:17:55 +v2.0.8;single;316;56234;;108134;27288;63044;17309;69236;3366;778.216515068;2021-12-09@22:30:57 +v2.0.8;single;316;56234;;108134;27288;63388;17308;69232;3357;777.614062224;2021-12-09@22:43:59 +v2.0.8;single;316;56234;;108134;27288;63032;17284;69136;3472;803.113882265;2021-12-09@22:57:26 +v2.0.8;single;316;56234;;108134;27288;63600;17364;69456;3264;756.433835243;2021-12-09@23:10:06 +v2.0.8;mulimex;316;56234;;108134;27288;4015060;1005320;4021280;6419;1498.673693895;2021-12-09@23:45:35 +v2.0.8;mulimex;316;56234;;108134;27288;4023336;1007386;4029544;7218;1693.209656559;2021-12-10@00:14:02 +v2.0.8;mulimex;316;56234;;108134;27288;4145892;1038037;4152148;7027;1647.495303583;2021-12-10@00:52:48 +v2.0.8;mulimex;316;56234;;108134;27288;4067076;1018274;4073096;6475;1515.253848630;2021-12-10@01:52:01 +v2.0.8;mulimex;316;56234;;108134;27288;4013040;1004824;4019296;7229;1691.493952292;2021-12-10@02:20:26 +v2.0.8;mulimex;316;56234;;108134;27288;4077188;1020810;4083240;7200;1691.526918780;2021-12-10@02:48:50 +v2.0.8;mulimex;316;56234;;108134;27288;3979960;996552;3986208;7216;1689.235991590;2021-12-10@03:17:14 +v2.0.8;mulimex;316;56234;;108134;27288;4064952;1017712;4070848;7188;1684.878327657;2021-12-10@03:45:32 +v2.0.8;multi;10;100000;;126751;52385;207180;53289;213156;256;61.422379061;2021-12-10@08:03:21 +v2.0.8;multi;10;100000;;126751;52385;207144;53317;213268;261;62.598339789;2021-12-10@08:04:27 +v2.0.8;multi;10;100000;;126751;52385;207032;53330;213320;263;63.008616201;2021-12-10@08:05:34 +v2.0.8;multi;10;100000;;126751;52385;206892;53313;213252;261;62.584602089;2021-12-10@08:06:40 +v2.0.8;multi;10;100000;;126751;52385;207060;53318;213272;262;62.757940658;2021-12-10@08:07:46 +v2.0.8;multi;10;100000;;126751;52385;207268;53372;213488;261;62.566563125;2021-12-10@08:08:53 +v2.0.8;multi;10;100000;;126751;52385;206964;53300;213200;261;62.595160233;2021-12-10@08:09:59 +v2.0.8;multi;10;100000;;126751;52385;207200;53363;213452;262;62.802329236;2021-12-10@08:11:06 +v2.0.8;imex;10;100000;;126751;52385;163032;42279;169116;163;38.670607040;2021-12-10@08:11:59 +v2.0.8;imex;10;100000;;126751;52385;162836;42296;169184;168;39.786845618;2021-12-10@08:12:43 +v2.0.8;imex;10;100000;;126751;52385;163364;42331;169324;166;39.358378706;2021-12-10@08:13:26 +v2.0.8;imex;10;100000;;126751;52385;163460;42382;169528;161;38.226190489;2021-12-10@08:14:08 +v2.0.8;imex;10;100000;;126751;52385;162920;42306;169224;166;39.361520596;2021-12-10@08:14:51 +v2.0.8;imex;10;100000;;126751;52385;163020;42303;169212;163;38.668722105;2021-12-10@08:15:34 +v2.0.8;imex;10;100000;;126751;52385;163000;42309;169236;163;38.630222251;2021-12-10@08:16:16 +v2.0.8;imex;10;100000;;126751;52385;163076;42274;169096;164;38.886226544;2021-12-10@08:16:59 +v2.0.8;single;10;100000;;126751;52385;47720;13496;53984;168;39.846843594;2021-12-10@08:17:54 +v2.0.8;single;10;100000;;126751;52385;48836;13737;54948;175;41.502038251;2021-12-10@08:18:39 +v2.0.8;single;10;100000;;126751;52385;48248;13613;54452;170;40.374474078;2021-12-10@08:19:23 +v2.0.8;single;10;100000;;126751;52385;48360;13602;54408;169;40.093415604;2021-12-10@08:20:07 +v2.0.8;single;10;100000;;126751;52385;48164;13594;54376;172;40.808377893;2021-12-10@08:20:51 +v2.0.8;single;10;100000;;126751;52385;48628;13649;54596;171;40.625741570;2021-12-10@08:21:36 +v2.0.8;single;10;100000;;126751;52385;48124;13611;54444;172;40.809039751;2021-12-10@08:22:20 +v2.0.8;single;10;100000;;126751;52385;48320;13623;54492;172;40.768748838;2021-12-10@08:23:04 +v2.0.8;mulimex;10;100000;;126751;52385;289288;73837;295348;257;61.657009484;2021-12-10@08:24:21 +v2.0.8;mulimex;10;100000;;126751;52385;289728;73964;295856;263;63.020400014;2021-12-10@08:25:28 +v2.0.8;mulimex;10;100000;;126751;52385;289948;74059;296236;262;62.793233488;2021-12-10@08:26:35 +v2.0.8;mulimex;10;100000;;126751;52385;289432;73931;295724;262;62.793702903;2021-12-10@08:27:42 +v2.0.8;mulimex;10;100000;;126751;52385;289612;73934;295736;262;62.771316175;2021-12-10@08:28:49 +v2.0.8;mulimex;10;100000;;126751;52385;289408;73867;295468;262;62.766908632;2021-12-10@08:29:55 +v2.0.8;mulimex;10;100000;;126751;52385;289184;73836;295344;262;62.795311186;2021-12-10@08:31:02 +v2.0.8;mulimex;10;100000;;126751;52385;289220;73862;295448;262;62.817272413;2021-12-10@08:32:09 +v2.0.8;multi;18;100000;;150528;52561;348400;88666;354664;540;130.489324272;2021-12-10@08:34:38 +v2.0.8;multi;18;100000;;150528;52561;348608;88660;354640;544;131.511207725;2021-12-10@08:36:54 +v2.0.8;multi;18;100000;;150528;52561;348680;88699;354796;545;131.566437230;2021-12-10@08:39:09 +v2.0.8;multi;18;100000;;150528;52561;348608;88663;354652;543;131.309395814;2021-12-10@08:41:25 +v2.0.8;multi;18;100000;;150528;52561;348844;88705;354820;543;131.294875974;2021-12-10@08:43:40 +v2.0.8;multi;18;100000;;150528;52561;348508;88681;354724;545;131.490671970;2021-12-10@08:45:55 +v2.0.8;multi;18;100000;;150528;52561;348424;88666;354664;545;131.502302428;2021-12-10@08:48:11 +v2.0.8;multi;18;100000;;150528;52561;348624;88687;354748;544;131.281956301;2021-12-10@08:50:26 +v2.0.8;imex;18;100000;;150528;52561;233052;59800;239200;287;68.101859285;2021-12-10@08:51:53 +v2.0.8;imex;18;100000;;150528;52561;232932;59756;239024;286;67.936193661;2021-12-10@08:53:05 +v2.0.8;imex;18;100000;;150528;52561;232692;59766;239064;286;67.894212419;2021-12-10@08:54:17 +v2.0.8;imex;18;100000;;150528;52561;232976;59748;238992;280;66.430536606;2021-12-10@08:55:27 +v2.0.8;imex;18;100000;;150528;52561;232712;59779;239116;292;69.312523916;2021-12-10@08:56:40 +v2.0.8;imex;18;100000;;150528;52561;233000;59786;239144;278;65.995881891;2021-12-10@08:57:50 +v2.0.8;imex;18;100000;;150528;52561;232784;59760;239040;282;66.905117325;2021-12-10@08:59:01 +v2.0.8;imex;18;100000;;150528;52561;232896;59776;239104;284;67.408694626;2021-12-10@09:00:12 +v2.0.8;single;18;100000;;150528;52561;57280;15824;63296;303;71.852114708;2021-12-10@09:01:42 +v2.0.8;single;18;100000;;150528;52561;56848;15818;63272;290;68.923846922;2021-12-10@09:02:55 +v2.0.8;single;18;100000;;150528;52561;57160;15822;63288;297;70.530116768;2021-12-10@09:04:09 +v2.0.8;single;18;100000;;150528;52561;57368;15846;63384;299;71.041915062;2021-12-10@09:05:24 +v2.0.8;single;18;100000;;150528;52561;57188;15830;63320;294;69.797519612;2021-12-10@09:06:37 +v2.0.8;single;18;100000;;150528;52561;57468;15876;63504;291;69.110833238;2021-12-10@09:07:50 +v2.0.8;single;18;100000;;150528;52561;57140;15847;63388;289;68.639002772;2021-12-10@09:09:02 +v2.0.8;single;18;100000;;150528;52561;57388;15846;63384;290;68.913569080;2021-12-10@09:10:15 +v2.0.8;mulimex;18;100000;;150528;52561;484416;122686;490744;540;130.553837694;2021-12-10@09:12:44 +v2.0.8;mulimex;18;100000;;150528;52561;484712;122676;490704;546;131.948042431;2021-12-10@09:15:00 +v2.0.8;mulimex;18;100000;;150528;52561;484424;122663;490652;546;132.018527845;2021-12-10@09:17:17 +v2.0.8;mulimex;18;100000;;150528;52561;484424;122686;490744;547;132.011198043;2021-12-10@09:19:33 +v2.0.8;mulimex;18;100000;;150528;52561;484524;122668;490672;547;132.034137333;2021-12-10@09:21:50 +v2.0.8;mulimex;18;100000;;150528;52561;484512;122680;490720;546;132.034952063;2021-12-10@09:24:06 +v2.0.8;mulimex;18;100000;;150528;52561;484608;122668;490672;545;131.694100866;2021-12-10@09:26:22 +v2.0.8;mulimex;18;100000;;150528;52561;484616;122655;490620;546;131.893197937;2021-12-10@09:28:38 +v2.0.8;multi;32;100000;;166748;52697;585500;147903;591612;1051;255.532629359;2021-12-10@09:33:18 +v2.0.8;multi;32;100000;;166748;52697;585464;147907;591628;1058;257.244346435;2021-12-10@09:37:40 +v2.0.8;multi;32;100000;;166748;52697;585556;147929;591716;1059;256.961497659;2021-12-10@09:42:01 +v2.0.8;multi;32;100000;;166748;52697;585472;147906;591624;1058;257.112199514;2021-12-10@09:46:23 +v2.0.8;multi;32;100000;;166748;52697;585676;147898;591592;1058;257.040912680;2021-12-10@09:50:45 +v2.0.8;multi;32;100000;;166748;52697;585416;147907;591628;1061;257.313266247;2021-12-10@09:55:07 +v2.0.8;multi;32;100000;;166748;52697;585332;147915;591660;1058;256.777926722;2021-12-10@09:59:28 +v2.0.8;multi;32;100000;;166748;52697;585756;147908;591632;1057;256.952081928;2021-12-10@10:03:50 +v2.0.8;imex;32;100000;;166748;52697;335240;85338;341352;506;120.751991642;2021-12-10@10:06:15 +v2.0.8;imex;32;100000;;166748;52697;334992;85336;341344;490;117.016551039;2021-12-10@10:08:16 +v2.0.8;imex;32;100000;;166748;52697;334944;85283;341132;486;115.992463342;2021-12-10@10:10:16 +v2.0.8;imex;32;100000;;166748;52697;335032;85300;341200;489;116.814684184;2021-12-10@10:12:17 +v2.0.8;imex;32;100000;;166748;52697;335028;85344;341376;510;121.783557333;2021-12-10@10:14:23 +v2.0.8;imex;32;100000;;166748;52697;334816;85284;341136;501;119.658040761;2021-12-10@10:16:26 +v2.0.8;imex;32;100000;;166748;52697;335148;85322;341288;497;118.590825695;2021-12-10@10:18:29 +v2.0.8;imex;32;100000;;166748;52697;334820;85317;341268;494;117.897587677;2021-12-10@10:20:31 +v2.0.8;single;32;100000;;166748;52697;64448;17675;70700;516;123.079794369;2021-12-10@10:22:58 +v2.0.8;single;32;100000;;166748;52697;64328;17686;70744;515;122.978158421;2021-12-10@10:25:05 +v2.0.8;single;32;100000;;166748;52697;64728;17699;70796;511;122.010874127;2021-12-10@10:27:11 +v2.0.8;single;32;100000;;166748;52697;64688;17656;70624;518;123.722731594;2021-12-10@10:29:18 +v2.0.8;single;32;100000;;166748;52697;64608;17704;70816;516;123.277272432;2021-12-10@10:31:25 +v2.0.8;single;32;100000;;166748;52697;64692;17693;70772;538;128.386620522;2021-12-10@10:33:37 +v2.0.8;single;32;100000;;166748;52697;64284;17685;70740;523;125.010457036;2021-12-10@10:35:46 +v2.0.8;single;32;100000;;166748;52697;64428;17712;70848;521;124.345744319;2021-12-10@10:37:54 +v2.0.8;mulimex;32;100000;;166748;52697;810824;204281;817124;1055;256.240882417;2021-12-10@10:42:34 +v2.0.8;mulimex;32;100000;;166748;52697;811256;204335;817340;1064;258.026130780;2021-12-10@10:46:57 +v2.0.8;mulimex;32;100000;;166748;52697;810872;204263;817052;1081;262.522433325;2021-12-10@10:51:24 +v2.0.8;mulimex;32;100000;;166748;52697;811256;204320;817280;1065;258.100905345;2021-12-10@10:55:47 +v2.0.8;mulimex;32;100000;;166748;52697;810972;204291;817164;1064;258.029904559;2021-12-10@11:00:10 +v2.0.8;mulimex;32;100000;;166748;52697;811280;204341;817364;1064;257.757754508;2021-12-10@11:04:33 +v2.0.8;mulimex;32;100000;;166748;52697;810812;204298;817192;1060;257.598729297;2021-12-10@11:08:55 +v2.0.8;mulimex;32;100000;;166748;52697;810768;204268;817072;1060;257.926354176;2021-12-10@11:13:18 +v2.0.8;multi;56;100000;;177146;52712;984564;247728;990912;1951;474.865680018;2021-12-10@11:21:47 +v2.0.8;multi;56;100000;;177146;52712;984792;247753;991012;1957;476.222394144;2021-12-10@11:29:49 +v2.0.8;multi;56;100000;;177146;52712;984952;247740;990960;1956;475.570635848;2021-12-10@11:37:50 +v2.0.8;multi;56;100000;;177146;52712;984900;247746;990984;1955;475.899522714;2021-12-10@11:45:51 +v2.0.8;multi;56;100000;;177146;52712;984628;247704;990816;1951;475.012637040;2021-12-10@11:53:52 +v2.0.8;multi;56;100000;;177146;52712;985000;247761;991044;1957;475.580521729;2021-12-10@12:01:53 +v2.0.8;multi;56;100000;;177146;52712;984672;247748;990992;1956;475.372066988;2021-12-10@12:09:53 +v2.0.8;multi;56;100000;;177146;52712;984836;247726;990904;1953;475.463190947;2021-12-10@12:17:54 +v2.0.8;imex;56;100000;;177146;52712;495208;125394;501576;898;215.066937792;2021-12-10@12:22:08 +v2.0.8;imex;56;100000;;177146;52712;495296;125334;501336;875;209.791672186;2021-12-10@12:25:42 +v2.0.8;imex;56;100000;;177146;52712;495128;125329;501316;873;209.367687161;2021-12-10@12:29:15 +v2.0.8;imex;56;100000;;177146;52712;495064;125323;501292;879;210.591852481;2021-12-10@12:32:50 +v2.0.8;imex;56;100000;;177146;52712;494948;125370;501480;876;209.979222487;2021-12-10@12:36:24 +v2.0.8;imex;56;100000;;177146;52712;495420;125388;501552;894;214.054642518;2021-12-10@12:40:02 +v2.0.8;imex;56;100000;;177146;52712;495308;125350;501400;878;210.553936800;2021-12-10@12:43:37 +v2.0.8;imex;56;100000;;177146;52712;495248;125352;501408;877;210.188968617;2021-12-10@12:47:11 +v2.0.8;single;56;100000;;177146;52712;71520;19403;77612;928;222.447862729;2021-12-10@12:51:30 +v2.0.8;single;56;100000;;177146;52712;71228;19345;77380;914;219.315697798;2021-12-10@12:55:13 +v2.0.8;single;56;100000;;177146;52712;71064;19374;77496;912;218.750667689;2021-12-10@12:58:56 +v2.0.8;single;56;100000;;177146;52712;71480;19388;77552;907;217.554823544;2021-12-10@13:02:37 +v2.0.8;single;56;100000;;177146;52712;71136;19358;77432;911;218.584815827;2021-12-10@13:06:19 +v2.0.8;single;56;100000;;177146;52712;71192;19369;77476;904;216.944834447;2021-12-10@13:10:00 +v2.0.8;single;56;100000;;177146;52712;71256;19381;77524;903;216.670268354;2021-12-10@13:13:40 +v2.0.8;single;56;100000;;177146;52712;71444;19349;77396;912;218.676539322;2021-12-10@13:17:23 +v2.0.8;mulimex;56;100000;;177146;52712;1359512;341403;1365612;1965;477.751909531;2021-12-10@13:25:57 +v2.0.8;mulimex;56;100000;;177146;52712;1359596;341468;1365872;1962;476.908165405;2021-12-10@13:34:00 +v2.0.8;mulimex;56;100000;;177146;52712;1359608;341465;1365860;1963;477.112198559;2021-12-10@13:42:04 +v2.0.8;mulimex;56;100000;;177146;52712;1359636;341419;1365676;1963;476.525478906;2021-12-10@13:50:07 +v2.0.8;mulimex;56;100000;;177146;52712;1359428;341396;1365584;1966;476.638603907;2021-12-10@13:58:09 +v2.0.8;mulimex;56;100000;;177146;52712;1359680;341480;1365920;1963;476.941676967;2021-12-10@14:06:13 +v2.0.8;mulimex;56;100000;;177146;52712;1359436;341431;1365724;1972;478.641511070;2021-12-10@14:14:17 +v2.0.8;mulimex;56;100000;;177146;52712;1359416;341414;1365656;1966;476.988056518;2021-12-10@14:22:21 +v2.0.8;multi;100;100000;;184114;52738;1713960;430068;1720272;3688;895.861507387;2021-12-10@14:38:11 +v2.0.8;multi;100;100000;;184114;52738;1713576;429937;1719748;3701;897.690082290;2021-12-10@14:53:16 +v2.0.8;multi;100;100000;;184114;52738;1713444;429927;1719708;3727;902.033207677;2021-12-10@15:08:26 +v2.0.8;multi;100;100000;;184114;52738;1713916;429976;1719904;3738;908.741144363;2021-12-10@15:23:42 +v2.0.8;multi;100;100000;;184114;52738;1714008;430015;1720060;3671;892.649549515;2021-12-10@15:38:42 +v2.0.8;multi;100;100000;;184114;52738;1713960;430016;1720064;3680;894.382012011;2021-12-10@15:53:44 +v2.0.8;multi;100;100000;;184114;52738;1713612;429948;1719792;3681;892.489340561;2021-12-10@16:08:44 +v2.0.8;multi;100;100000;;184114;52738;1713832;430002;1720008;3680;894.560041411;2021-12-10@16:23:46 +v2.0.8;imex;100;100000;;184114;52738;778156;196073;784292;1642;395.720583362;2021-12-10@16:31:28 +v2.0.8;imex;100;100000;;184114;52738;778344;196093;784372;1576;380.524516560;2021-12-10@16:37:53 +v2.0.8;imex;100;100000;;184114;52738;778224;196098;784392;1597;385.367538117;2021-12-10@16:44:23 +v2.0.8;imex;100;100000;;184114;52738;778408;196116;784464;1591;384.051018325;2021-12-10@16:50:51 +v2.0.8;imex;100;100000;;184114;52738;778596;196153;784612;1580;381.401205690;2021-12-10@16:57:17 +v2.0.8;imex;100;100000;;184114;52738;778212;196060;784240;1586;382.982382295;2021-12-10@17:03:45 +v2.0.8;imex;100;100000;;184114;52738;778436;196138;784552;1598;385.675210805;2021-12-10@17:10:14 +v2.0.8;imex;100;100000;;184114;52738;778044;196084;784336;1589;383.708026852;2021-12-10@17:16:42 +v2.0.8;single;100;100000;;184114;52738;78280;21122;84488;1719;414.137664009;2021-12-10@17:24:39 +v2.0.8;single;100;100000;;184114;52738;78364;21131;84524;1660;400.072047378;2021-12-10@17:31:23 +v2.0.8;single;100;100000;;184114;52738;78500;21173;84692;1652;398.091613145;2021-12-10@17:38:04 +v2.0.8;single;100;100000;;184114;52738;78536;21153;84612;1665;401.142907638;2021-12-10@17:44:49 +v2.0.8;single;100;100000;;184114;52738;78692;21221;84884;1655;398.961981207;2021-12-10@17:51:32 +v2.0.8;single;100;100000;;184114;52738;78540;21170;84680;1667;401.595647385;2021-12-10@17:58:17 +v2.0.8;single;100;100000;;184114;52738;78584;21170;84680;1733;417.163083625;2021-12-10@18:05:18 +v2.0.8;single;100;100000;;184114;52738;78460;21182;84728;1721;414.237359917;2021-12-10@18:12:16 +v2.0.8;mulimex;100;100000;;184114;52738;2360520;591716;2366864;3696;899.183164293;2021-12-10@18:28:17 +v2.0.8;mulimex;100;100000;;184114;52738;2360316;591657;2366628;3702;900.338327312;2021-12-10@18:43:26 +v2.0.8;mulimex;100;100000;;184114;52738;2360372;591623;2366492;3705;899.887592162;2021-12-10@18:58:35 +v2.0.8;mulimex;100;100000;;184114;52738;2359972;591564;2366256;3720;902.877746033;2021-12-10@19:13:46 +v2.0.8;mulimex;100;100000;;184114;52738;2360200;591648;2366592;3668;892.048996541;2021-12-10@19:28:47 +v2.0.8;mulimex;100;100000;;184114;52738;2360332;591607;2366428;2819;691.300006844;2021-12-10@19:43:51 +v2.0.8;mulimex;100;100000;;184114;52738;2360680;591670;2366680;3704;899.521566115;2021-12-10@19:59:00 +v2.0.8;mulimex;100;100000;;184114;52738;2360496;591655;2366620;3686;893.689240674;2021-12-10@20:14:02 +v2.0.8;multi;178;100000;;188000;52864;3018772;756202;3024808;6621;1622.367983252;2021-12-10@20:42:40 +v2.0.8;multi;178;100000;;188000;52864;2996336;750655;3002620;6889;1676.610774038;2021-12-10@21:10:48 +v2.0.8;multi;178;100000;;188000;52864;3005988;753029;3012116;5803;1423.152525213;2021-12-10@21:38:02 diff --git a/filter/config.Y b/filter/config.Y index 8916ea97..15b77761 100644 --- a/filter/config.Y +++ b/filter/config.Y @@ -248,10 +248,6 @@ assert_assign(struct f_lval *lval, struct f_inst *expr, const char *start, const setter = f_new_inst(FI_VAR_SET, expr, lval->sym); getter = f_new_inst(FI_VAR_GET, lval->sym); break; - case F_LVAL_PREFERENCE: - setter = f_new_inst(FI_PREF_SET, expr); - getter = f_new_inst(FI_PREF_GET); - break; case F_LVAL_SA: setter = f_new_inst(FI_RTA_SET, expr, lval->sa); getter = f_new_inst(FI_RTA_GET, lval->sa); @@ -290,7 +286,7 @@ CF_KEYWORDS(FUNCTION, PRINT, PRINTN, UNSET, RETURN, MIN, MAX, EMPTY, FILTER, WHERE, EVAL, ATTRIBUTE, - BT_ASSERT, BT_TEST_SUITE, BT_CHECK_ASSIGN, BT_TEST_SAME, FORMAT) + BT_ASSERT, BT_TEST_SUITE, BT_CHECK_ASSIGN, BT_TEST_SAME, FORMAT, STACKS) %nonassoc THEN %nonassoc ELSE @@ -314,6 +310,12 @@ CF_KEYWORDS(FUNCTION, PRINT, PRINTN, UNSET, RETURN, CF_GRAMMAR +conf: FILTER STACKS expr expr ';' { + new_config->filter_vstk = $3; + new_config->filter_estk = $4; + } + ; + conf: filter_def ; filter_def: FILTER symbol { $2 = cf_define_symbol($2, SYM_FILTER, filter, NULL); cf_push_scope( $2 ); } @@ -753,6 +755,7 @@ static_attr: | IFNAME { $$ = f_new_static_attr(T_STRING, SA_IFNAME, 0); } | IFINDEX { $$ = f_new_static_attr(T_INT, SA_IFINDEX, 1); } | WEIGHT { $$ = f_new_static_attr(T_INT, SA_WEIGHT, 0); } + | PREFERENCE { $$ = f_new_static_attr(T_INT, SA_PREF, 0); } | GW_MPLS { $$ = f_new_static_attr(T_INT, SA_GW_MPLS, 0); } ; @@ -779,8 +782,6 @@ term: | constant { $$ = $1; } | constructor { $$ = $1; } - | PREFERENCE { $$ = f_new_inst(FI_PREF_GET); } - | static_attr { $$ = f_new_inst(FI_RTA_GET, $1); } | dynamic_attr { $$ = f_new_inst(FI_EA_GET, $1); } @@ -878,9 +879,6 @@ cmd: cf_error( "This static attribute is read-only."); $$ = f_new_inst(FI_RTA_SET, $3, $1); } - | PREFERENCE '=' term ';' { - $$ = f_new_inst(FI_PREF_SET, $3); - } | UNSET '(' dynamic_attr ')' ';' { $$ = f_new_inst(FI_EA_UNSET, $3); } @@ -923,7 +921,6 @@ get_cf_position: lvalue: CF_SYM_KNOWN { cf_assert_symbol($1, SYM_VARIABLE); $$ = (struct f_lval) { .type = F_LVAL_VARIABLE, .sym = $1 }; } - | PREFERENCE { $$ = (struct f_lval) { .type = F_LVAL_PREFERENCE }; } | static_attr { $$ = (struct f_lval) { .type = F_LVAL_SA, .sa = $1 }; } | dynamic_attr { $$ = (struct f_lval) { .type = F_LVAL_EA, .da = $1 }; }; diff --git a/filter/data.h b/filter/data.h index bb815c34..6d4ca333 100644 --- a/filter/data.h +++ b/filter/data.h @@ -100,6 +100,7 @@ enum f_sa_code { SA_IFNAME, SA_IFINDEX, SA_WEIGHT, + SA_PREF, SA_GW_MPLS, } PACKED; diff --git a/filter/decl.m4 b/filter/decl.m4 index 5242c04c..44537aaa 100644 --- a/filter/decl.m4 +++ b/filter/decl.m4 @@ -32,6 +32,7 @@ m4_divert(-1)m4_dnl # # 101 content of per-inst struct # 102 constructor arguments +# 110 constructor attributes # 103 constructor body # 104 dump line item content # (there may be nothing in dump-line content and @@ -45,6 +46,7 @@ m4_divert(-1)m4_dnl # Here are macros to allow you to _divert to the right directions. m4_define(FID_STRUCT_IN, `m4_divert(101)') m4_define(FID_NEW_ARGS, `m4_divert(102)') +m4_define(FID_NEW_ATTRIBUTES, `m4_divert(110)') m4_define(FID_NEW_BODY, `m4_divert(103)') m4_define(FID_DUMP_BODY, `m4_divert(104)m4_define([[FID_DUMP_BODY_EXISTS]])') m4_define(FID_LINEARIZE_BODY, `m4_divert(105)') @@ -106,15 +108,18 @@ FID_STRUCT_IN()m4_dnl struct f_inst * f$1; FID_NEW_ARGS()m4_dnl , struct f_inst * f$1 +FID_NEW_ATTRIBUTES()m4_dnl +NONNULL(m4_eval($1+1)) FID_NEW_BODY()m4_dnl whati->f$1 = f$1; -for (const struct f_inst *child = f$1; child; child = child->next) { - what->size += child->size; +const struct f_inst *child$1 = f$1; +do { + what->size += child$1->size; FID_IFCONST([[ - if (child->fi_code != FI_CONSTANT) + if (child$1->fi_code != FI_CONSTANT) constargs = 0; ]]) -} +} while (child$1 = child$1->next); FID_LINEARIZE_BODY pos = linearize(dest, whati->f$1, pos); FID_INTERPRET_BODY()') @@ -190,6 +195,7 @@ FID_INTERPRET_BODY()') # that was needed in the former implementation. m4_define(LINEX, `FID_INTERPRET_EXEC()LINEX_($1)FID_INTERPRET_NEW()return $1 FID_INTERPRET_BODY()') m4_define(LINEX_, `do { + if (fstk->ecnt + 1 >= fstk->elen) runtime("Filter execution stack overflow"); fstk->estk[fstk->ecnt].pos = 0; fstk->estk[fstk->ecnt].line = $1; fstk->estk[fstk->ecnt].ventry = fstk->vcnt; @@ -227,7 +233,7 @@ FID_INTERPRET_BODY()') # state the result and put it to the right place. m4_define(RESULT, `RESULT_TYPE([[$1]]) RESULT_([[$1]],[[$2]],[[$3]])') m4_define(RESULT_, `RESULT_VAL([[ (struct f_val) { .type = $1, .val.$2 = $3 } ]])') -m4_define(RESULT_VAL, `FID_HIC(, [[do { res = $1; fstk->vcnt++; } while (0)]], +m4_define(RESULT_VAL, `FID_HIC(, [[do { res = $1; f_vcnt_check_overflow(1); fstk->vcnt++; } while (0)]], [[return fi_constant(what, $1)]])') m4_define(RESULT_VOID, `RESULT_VAL([[ (struct f_val) { .type = T_VOID } ]])') @@ -309,7 +315,9 @@ m4_undivert(107)m4_dnl FID_NEW()m4_dnl Constructor and interpreter code together FID_HIC( [[m4_dnl Public declaration of constructor in H file -struct f_inst *f_new_inst_]]INST_NAME()[[(enum f_instruction_code fi_code +struct f_inst * +m4_undivert(110)m4_dnl +f_new_inst_]]INST_NAME()[[(enum f_instruction_code fi_code m4_undivert(102)m4_dnl );]], [[m4_dnl The one case in The Big Switch inside interpreter @@ -321,7 +329,9 @@ m4_undivert(102)m4_dnl break; ]], [[m4_dnl Constructor itself -struct f_inst *f_new_inst_]]INST_NAME()[[(enum f_instruction_code fi_code +struct f_inst * +m4_undivert(110)m4_dnl +f_new_inst_]]INST_NAME()[[(enum f_instruction_code fi_code m4_undivert(102)m4_dnl ) { diff --git a/filter/f-inst.c b/filter/f-inst.c index 901d2939..b4795357 100644 --- a/filter/f-inst.c +++ b/filter/f-inst.c @@ -519,20 +519,21 @@ { STATIC_ATTR; ACCESS_RTE; - struct rta *rta = (*fs->rte)->attrs; + struct rta *rta = fs->rte->attrs; switch (sa.sa_code) { case SA_FROM: RESULT(sa.f_type, ip, rta->from); break; case SA_GW: RESULT(sa.f_type, ip, rta->nh.gw); break; - case SA_NET: RESULT(sa.f_type, net, (*fs->rte)->net->n.addr); break; - case SA_PROTO: RESULT(sa.f_type, s, rta->src->proto->name); break; + case SA_NET: RESULT(sa.f_type, net, fs->rte->net); break; + case SA_PROTO: RESULT(sa.f_type, s, fs->rte->src->owner->name); break; case SA_SOURCE: RESULT(sa.f_type, i, rta->source); break; case SA_SCOPE: RESULT(sa.f_type, i, rta->scope); break; case SA_DEST: RESULT(sa.f_type, i, rta->dest); break; case SA_IFNAME: RESULT(sa.f_type, s, rta->nh.iface ? rta->nh.iface->name : ""); break; case SA_IFINDEX: RESULT(sa.f_type, i, rta->nh.iface ? rta->nh.iface->index : 0); break; case SA_WEIGHT: RESULT(sa.f_type, i, rta->nh.weight + 1); break; + case SA_PREF: RESULT(sa.f_type, i, rta->pref); break; case SA_GW_MPLS: RESULT(sa.f_type, i, rta->nh.labels ? rta->nh.label[0] : MPLS_NULL); break; default: @@ -549,7 +550,7 @@ f_rta_cow(fs); { - struct rta *rta = (*fs->rte)->attrs; + struct rta *rta = fs->rte->attrs; switch (sa.sa_code) { @@ -561,7 +562,8 @@ { ip_addr ip = v1.val.ip; struct iface *ifa = ipa_is_link_local(ip) ? rta->nh.iface : NULL; - neighbor *n = neigh_find(rta->src->proto, ip, ifa, 0); + /* XXX this code supposes that every owner is a protocol XXX */ + neighbor *n = neigh_find(SKIP_BACK(struct proto, sources, fs->rte->src->owner), ip, ifa, 0); if (!n || (n->scope == SCOPE_HOST)) runtime( "Invalid gw address" ); @@ -637,6 +639,10 @@ } break; + case SA_PREF: + rta->pref = v1.val.i; + break; + default: bug("Invalid static attribute access (%u/%u)", sa.f_type, sa.sa_code); } @@ -804,20 +810,6 @@ } } - INST(FI_PREF_GET, 0, 1) { - ACCESS_RTE; - RESULT(T_INT, i, (*fs->rte)->pref); - } - - INST(FI_PREF_SET, 1, 0) { - ACCESS_RTE; - ARG(1,T_INT); - if (v1.val.i > 0xFFFF) - runtime( "Setting preference value out of bounds" ); - f_rte_cow(fs); - (*fs->rte)->pref = v1.val.i; - } - INST(FI_LENGTH, 1, 1) { /* Get length of */ ARG_ANY(1); switch(v1.type) { @@ -1103,6 +1095,7 @@ curline.vbase = curline.ventry; /* Storage for local variables */ + f_vcnt_check_overflow(sym->function->vars); memset(&(fstk->vstk[fstk->vcnt]), 0, sizeof(struct f_val) * sym->function->vars); fstk->vcnt += sym->function->vars; } @@ -1319,10 +1312,10 @@ INST(FI_ROA_CHECK_IMPLICIT, 0, 1) { /* ROA Check */ NEVER_CONSTANT; RTC(1); - struct rtable *table = rtc->table; + rtable *table = rtc->table; ACCESS_RTE; ACCESS_EATTRS; - const net_addr *net = (*fs->rte)->net->n.addr; + const net_addr *net = fs->rte->net; /* We ignore temporary attributes, probably not a problem here */ /* 0x02 is a value of BA_AS_PATH, we don't want to include BGP headers */ @@ -1351,7 +1344,7 @@ ARG(1, T_NET); ARG(2, T_INT); RTC(3); - struct rtable *table = rtc->table; + rtable *table = rtc->table; u32 as = v2.val.i; diff --git a/filter/filter.c b/filter/filter.c index e505d570..625b3ade 100644 --- a/filter/filter.c +++ b/filter/filter.c @@ -50,36 +50,31 @@ enum f_exception { FE_RETURN = 0x1, }; - -struct filter_stack { - /* Value stack for execution */ -#define F_VAL_STACK_MAX 4096 - uint vcnt; /* Current value stack size; 0 for empty */ - uint ecnt; /* Current execute stack size; 0 for empty */ - - struct f_val vstk[F_VAL_STACK_MAX]; /* The stack itself */ - - /* Instruction stack for execution */ -#define F_EXEC_STACK_MAX 4096 - struct { - const struct f_line *line; /* The line that is being executed */ - uint pos; /* Instruction index in the line */ - uint ventry; /* Value stack depth on entry */ - uint vbase; /* Where to index variable positions from */ - enum f_exception emask; /* Exception mask */ - } estk[F_EXEC_STACK_MAX]; +struct filter_exec_stack { + const struct f_line *line; /* The line that is being executed */ + uint pos; /* Instruction index in the line */ + uint ventry; /* Value stack depth on entry */ + uint vbase; /* Where to index variable positions from */ + enum f_exception emask; /* Exception mask */ }; /* Internal filter state, to be allocated on stack when executing filters */ struct filter_state { /* Stacks needed for execution */ - struct filter_stack *stack; + struct filter_stack { + /* Current filter stack depth */ - /* The route we are processing. This may be NULL to indicate no route available. */ - struct rte **rte; + /* Value stack */ + uint vcnt, vlen; + struct f_val *vstk; - /* The old rta to be freed after filters are done. */ - struct rta *old_rta; + /* Instruction stack for execution */ + uint ecnt, elen; + struct filter_exec_stack *estk; + } stack; + + /* The route we are processing. This may be NULL to indicate no route available. */ + struct rte *rte; /* Cached pointer to ea_list */ struct ea_list **eattrs; @@ -95,21 +90,16 @@ struct filter_state { }; _Thread_local static struct filter_state filter_state; -_Thread_local static struct filter_stack filter_stack; void (*bt_assert_hook)(int result, const struct f_line_item *assert); -static inline void f_cache_eattrs(struct filter_state *fs) -{ - fs->eattrs = &((*fs->rte)->attrs->eattrs); -} +#define _f_stack_init(fs, px, def) ((fs).stack.px##stk = alloca(sizeof(*(fs).stack.px##stk) * ((fs).stack.px##len = (config && config->filter_##px##stk) ? config->filter_##px##stk : (def)))) -static inline void f_rte_cow(struct filter_state *fs) -{ - if (!((*fs->rte)->flags & REF_COW)) - return; +#define f_stack_init(fs) ( _f_stack_init(fs, v, 128), _f_stack_init(fs, e, 128) ) - *fs->rte = rte_cow(*fs->rte); +static inline void f_cache_eattrs(struct filter_state *fs) +{ + fs->eattrs = &(fs->rte->attrs->eattrs); } /* @@ -118,22 +108,16 @@ static inline void f_rte_cow(struct filter_state *fs) static void f_rta_cow(struct filter_state *fs) { - if (!rta_is_cached((*fs->rte)->attrs)) + if (!rta_is_cached(fs->rte->attrs)) return; - /* Prepare to modify rte */ - f_rte_cow(fs); - - /* Store old rta to free it later, it stores reference from rte_cow() */ - fs->old_rta = (*fs->rte)->attrs; - /* * Get shallow copy of rta. Fields eattrs and nexthops of rta are shared * with fs->old_rta (they will be copied when the cached rta will be obtained * at the end of f_run()), also the lock of hostentry is inherited (we * suppose hostentry is not changed by filters). */ - (*fs->rte)->attrs = rta_do_cow((*fs->rte)->attrs, fs->pool); + fs->rte->attrs = rta_do_cow(fs->rte->attrs, fs->pool); /* Re-cache the ea_list */ f_cache_eattrs(fs); @@ -163,15 +147,17 @@ interpret(struct filter_state *fs, const struct f_line *line, struct f_val *val) ASSERT(line->args == 0); /* Initialize the filter stack */ - struct filter_stack *fstk = fs->stack; + struct filter_stack *fstk = &fs->stack; fstk->vcnt = line->vars; memset(fstk->vstk, 0, sizeof(struct f_val) * line->vars); /* The same as with the value stack. Not resetting the stack for performance reasons. */ fstk->ecnt = 1; - fstk->estk[0].line = line; - fstk->estk[0].pos = 0; + fstk->estk[0] = (struct filter_exec_stack) { + .line = line, + .pos = 0, + }; #define curline fstk->estk[fstk->ecnt-1] @@ -191,6 +177,8 @@ interpret(struct filter_state *fs, const struct f_line *line, struct f_val *val) #define v2 vv(1) #define v3 vv(2) +#define f_vcnt_check_overflow(n) do { if (fstk->vcnt + n >= fstk->vlen) runtime("Filter execution stack overflow"); } while (0) + #define runtime(fmt, ...) do { \ if (!(fs->flags & FF_SILENT)) \ log_rl(&rl_runtime_err, L_ERR "filters, line %d: " fmt, what->lineno, ##__VA_ARGS__); \ @@ -241,29 +229,15 @@ interpret(struct filter_state *fs, const struct f_line *line, struct f_val *val) /** * f_run - run a filter for a route * @filter: filter to run - * @rte: route being filtered, may be modified + * @rte: route being filtered, must be write-able * @tmp_pool: all filter allocations go from this pool * @flags: flags * - * If filter needs to modify the route, there are several - * posibilities. @rte might be read-only (with REF_COW flag), in that - * case rw copy is obtained by rte_cow() and @rte is replaced. If - * @rte is originally rw, it may be directly modified (and it is never - * copied). - * - * The returned rte may reuse the (possibly cached, cloned) rta, or - * (if rta was modified) contains a modified uncached rta, which - * uses parts allocated from @tmp_pool and parts shared from original - * rta. There is one exception - if @rte is rw but contains a cached - * rta and that is modified, rta in returned rte is also cached. - * - * Ownership of cached rtas is consistent with rte, i.e. - * if a new rte is returned, it has its own clone of cached rta - * (and cached rta of read-only source rte is intact), if rte is - * modified in place, old cached rta is possibly freed. + * If @rte->attrs is cached, the returned rte allocates a new rta on + * tmp_pool, otherwise the filters may modify it. */ enum filter_return -f_run(const struct filter *filter, struct rte **rte, struct linpool *tmp_pool, int flags) +f_run(const struct filter *filter, struct rte *rte, struct linpool *tmp_pool, int flags) { if (filter == FILTER_ACCEPT) return F_ACCEPT; @@ -271,48 +245,22 @@ f_run(const struct filter *filter, struct rte **rte, struct linpool *tmp_pool, i if (filter == FILTER_REJECT) return F_REJECT; - int rte_cow = ((*rte)->flags & REF_COW); DBG( "Running filter `%s'...", filter->name ); /* Initialize the filter state */ filter_state = (struct filter_state) { - .stack = &filter_stack, .rte = rte, .pool = tmp_pool, .flags = flags, }; + f_stack_init(filter_state); + LOG_BUFFER_INIT(filter_state.buf); /* Run the interpreter itself */ enum filter_return fret = interpret(&filter_state, filter->root, NULL); - if (filter_state.old_rta) { - /* - * Cached rta was modified and filter_state->rte contains now an uncached one, - * sharing some part with the cached one. The cached rta should - * be freed (if rte was originally COW, filter_state->old_rta is a clone - * obtained during rte_cow()). - * - * This also implements the exception mentioned in f_run() - * description. The reason for this is that rta reuses parts of - * filter_state->old_rta, and these may be freed during rta_free(filter_state->old_rta). - * This is not the problem if rte was COW, because original rte - * also holds the same rta. - */ - if (!rte_cow) { - /* Cache the new attrs */ - (*filter_state.rte)->attrs = rta_lookup((*filter_state.rte)->attrs); - - /* Drop cached ea_list pointer */ - filter_state.eattrs = NULL; - } - - /* Uncache the old attrs and drop the pointer as it is invalid now. */ - rta_free(filter_state.old_rta); - filter_state.old_rta = NULL; - } - /* Process the filter output, log it and return */ if (fret < F_ACCEPT) { if (!(filter_state.flags & FF_SILENT)) @@ -337,18 +285,18 @@ f_run(const struct filter *filter, struct rte **rte, struct linpool *tmp_pool, i */ enum filter_return -f_eval_rte(const struct f_line *expr, struct rte **rte, struct linpool *tmp_pool) +f_eval_rte(const struct f_line *expr, struct rte *rte, struct linpool *tmp_pool) { filter_state = (struct filter_state) { - .stack = &filter_stack, .rte = rte, .pool = tmp_pool, }; + f_stack_init(filter_state); + LOG_BUFFER_INIT(filter_state.buf); - ASSERT(!((*rte)->flags & REF_COW)); - ASSERT(!rta_is_cached((*rte)->attrs)); + ASSERT(!rta_is_cached(rte->attrs)); return interpret(&filter_state, expr, NULL); } @@ -363,10 +311,11 @@ enum filter_return f_eval(const struct f_line *expr, struct linpool *tmp_pool, struct f_val *pres) { filter_state = (struct filter_state) { - .stack = &filter_stack, .pool = tmp_pool, }; + f_stack_init(filter_state); + LOG_BUFFER_INIT(filter_state.buf); enum filter_return fret = interpret(&filter_state, expr, pres); @@ -383,10 +332,11 @@ f_eval_int(const struct f_line *expr) { /* Called independently in parse-time to eval expressions */ filter_state = (struct filter_state) { - .stack = &filter_stack, .pool = cfg_mem, }; + f_stack_init(filter_state); + struct f_val val; LOG_BUFFER_INIT(filter_state.buf); @@ -475,6 +425,23 @@ filter_commit(struct config *new, struct config *old) } } +void channel_filter_dump(const struct filter *f) +{ + if (f == FILTER_ACCEPT) + debug(" ALL"); + else if (f == FILTER_REJECT) + debug(" NONE"); + else if (f == FILTER_UNDEF) + debug(" UNDEF"); + else if (f->sym) { + ASSERT(f->sym->filter == f); + debug(" named filter %s", f->sym->name); + } else { + debug("\n"); + f_dump_line(f->root, 2); + } +} + void filters_dump_all(void) { struct symbol *sym; @@ -494,19 +461,10 @@ void filters_dump_all(void) struct channel *c; WALK_LIST(c, sym->proto->proto->channels) { debug(" Channel %s (%s) IMPORT", c->name, net_label[c->net_type]); - if (c->in_filter == FILTER_ACCEPT) - debug(" ALL\n"); - else if (c->in_filter == FILTER_REJECT) - debug(" NONE\n"); - else if (c->in_filter == FILTER_UNDEF) - debug(" UNDEF\n"); - else if (c->in_filter->sym) { - ASSERT(c->in_filter->sym->filter == c->in_filter); - debug(" named filter %s\n", c->in_filter->sym->name); - } else { - debug("\n"); - f_dump_line(c->in_filter->root, 2); - } + channel_filter_dump(c->in_filter); + debug(" EXPORT", c->name, net_label[c->net_type]); + channel_filter_dump(c->out_filter); + debug("\n"); } } } diff --git a/filter/filter.h b/filter/filter.h index 26c1037b..9964831c 100644 --- a/filter/filter.h +++ b/filter/filter.h @@ -51,8 +51,8 @@ struct filter { struct rte; -enum filter_return f_run(const struct filter *filter, struct rte **rte, struct linpool *tmp_pool, int flags); -enum filter_return f_eval_rte(const struct f_line *expr, struct rte **rte, struct linpool *tmp_pool); +enum filter_return f_run(const struct filter *filter, struct rte *rte, struct linpool *tmp_pool, int flags); +enum filter_return f_eval_rte(const struct f_line *expr, struct rte *rte, struct linpool *tmp_pool); uint f_eval_int(const struct f_line *expr); enum filter_return f_eval_buf(const struct f_line *expr, struct linpool *tmp_pool, buffer *buf); diff --git a/filter/filter_test.c b/filter/filter_test.c index 7e4af092..2a0b5431 100644 --- a/filter/filter_test.c +++ b/filter/filter_test.c @@ -72,6 +72,7 @@ int main(int argc, char *argv[]) { bt_init(argc, argv); + bt_bird_init(); bt_assert_hook = bt_assert_filter; diff --git a/filter/test.conf b/filter/test.conf index 6a28e4b3..f373bac5 100644 --- a/filter/test.conf +++ b/filter/test.conf @@ -378,7 +378,6 @@ bt_test_suite(t_ip_set, "Testing sets of ip address"); function t_enum() { - bt_assert(format(RTS_DUMMY) = "(enum 30)0"); bt_assert(format(RTS_STATIC) = "(enum 30)1"); bt_assert(format(NET_IP4) = "(enum 36)1"); bt_assert(format(NET_VPN6) = "(enum 36)4"); diff --git a/filter/tree_test.c b/filter/tree_test.c index 6472d17e..883d13bf 100644 --- a/filter/tree_test.c +++ b/filter/tree_test.c @@ -20,7 +20,7 @@ start_conf_env(void) { bt_bird_init(); - pool *p = rp_new(&root_pool, "helper_pool"); + pool *p = rp_new(&root_pool, &main_birdloop, "helper_pool"); linpool *l = lp_new_default(p); cfg_mem = l; } diff --git a/lib/Makefile b/lib/Makefile index 4378a7bd..98c5db3c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -1,4 +1,4 @@ -src := bitmap.c bitops.c blake2s.c blake2b.c checksum.c event.c flowspec.c idm.c ip.c lists.c mac.c md5.c mempool.c net.c patmatch.c printf.c resource.c sha1.c sha256.c sha512.c slab.c slists.c strtoul.c tbf.c timer.c xmalloc.c +src := bitmap.c bitops.c blake2s.c blake2b.c checksum.c event.c flowspec.c idm.c ip.c lists.c mac.c md5.c mempool.c net.c patmatch.c printf.c resource.c rcu.c sha1.c sha256.c sha512.c slab.c slists.c strtoul.c tbf.c timer.c xmalloc.c obj := $(src-o-files) $(all-daemon) diff --git a/lib/birdlib.h b/lib/birdlib.h index 431b7c0d..385bf75c 100644 --- a/lib/birdlib.h +++ b/lib/birdlib.h @@ -9,6 +9,7 @@ #ifndef _BIRD_BIRDLIB_H_ #define _BIRD_BIRDLIB_H_ +#include "sysdep/config.h" #include "lib/alloca.h" /* Ugly structure offset handling macros */ @@ -16,7 +17,7 @@ struct align_probe { char x; long int y; }; #define OFFSETOF(s, i) ((size_t) &((s *)0)->i) -#define SKIP_BACK(s, i, p) ((s *)((char *)p - OFFSETOF(s, i))) +#define SKIP_BACK(s, i, p) ({ s *_ptr = ((s *)((char *)p - OFFSETOF(s, i))); ASSERT_DIE(&_ptr->i == p); _ptr; }) #define BIRD_ALIGN(s, a) (((s)+a-1)&~(a-1)) #define CPU_STRUCT_ALIGN (sizeof(struct align_probe)) @@ -70,6 +71,7 @@ static inline int u64_cmp(u64 i1, u64 i2) /* Macros for gcc attributes */ #define NORET __attribute__((noreturn)) +#define USE_RESULT __atribute__((warn_unused_result)) #define UNUSED __attribute__((unused)) #define PACKED __attribute__((packed)) #define NONNULL(...) __attribute__((nonnull((__VA_ARGS__)))) @@ -77,10 +79,6 @@ static inline int u64_cmp(u64 i1, u64 i2) #define STATIC_ASSERT(EXP) _Static_assert(EXP, #EXP) #define STATIC_ASSERT_MSG(EXP,MSG) _Static_assert(EXP, MSG) -#ifndef HAVE_THREAD_LOCAL -#define _Thread_local -#endif - /* Microsecond time */ typedef s64 btime; diff --git a/lib/coro.h b/lib/coro.h new file mode 100644 index 00000000..b36f1d2c --- /dev/null +++ b/lib/coro.h @@ -0,0 +1,31 @@ +/* + * BIRD Coroutines + * + * (c) 2017 Martin Mares <mj@ucw.cz> + * (c) 2020-2021 Maria Matejka <mq@jmq.cz> + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_CORO_H_ +#define _BIRD_CORO_H_ + +#include "lib/resource.h" + +/* A completely opaque coroutine handle. */ +struct coroutine; + +/* Coroutines are independent threads bound to pools. + * You request a coroutine by calling coro_run(). + * It is forbidden to free a running coroutine from outside. + * The running coroutine must free itself by rfree() before returning. + */ +struct coroutine *coro_run(pool *, void (*entry)(void *), void *data); + +/* Get self. */ +extern _Thread_local struct coroutine *this_coro; + +/* Just wait for a little while. Not intended for general use; use events if possible. */ +void coro_yield(void); + +#endif diff --git a/lib/event.c b/lib/event.c index 273447e0..10f83c28 100644 --- a/lib/event.c +++ b/lib/event.c @@ -19,8 +19,14 @@ * events in them and explicitly ask to run them. */ +#undef LOCAL_DEBUG + #include "nest/bird.h" #include "lib/event.h" +#include "lib/locking.h" +#include "lib/io-loop.h" + +extern _Thread_local struct coroutine *this_coro; event_list global_event_list; event_list global_work_list; @@ -28,11 +34,16 @@ event_list global_work_list; inline void ev_postpone(event *e) { + event_list *el = e->list; + if (!el) + return; + + ASSERT_DIE(birdloop_inside(el->loop)); + + LOCK_DOMAIN(event, el->lock); if (ev_active(e)) - { - rem_node(&e->n); - e->n.next = NULL; - } + rem_node(&e->n); + UNLOCK_DOMAIN(event, el->lock); } static void @@ -95,40 +106,50 @@ ev_run(event *e) * list @l which can be run by calling ev_run_list(). */ inline void -ev_enqueue(event_list *l, event *e) +ev_send(event_list *l, event *e) { - ev_postpone(e); - add_tail(l, &e->n); -} + DBG("ev_send(%p, %p)\n", l, e); + ASSERT_DIE(e->hook); + ASSERT_DIE(!e->list || (e->list == l) || (e->list->loop == l->loop)); -/** - * ev_schedule - schedule an event - * @e: an event - * - * This function schedules an event by enqueueing it to a system-wide - * event list which is run by the platform dependent code whenever - * appropriate. - */ -void -ev_schedule(event *e) -{ - ev_enqueue(&global_event_list, e); -} + e->list = l; -/** - * ev_schedule_work - schedule a work-event. - * @e: an event - * - * This function schedules an event by enqueueing it to a system-wide work-event - * list which is run by the platform dependent code whenever appropriate. This - * is designated for work-events instead of regular events. They are executed - * less often in order to not clog I/O loop. - */ -void -ev_schedule_work(event *e) -{ - if (!ev_active(e)) - add_tail(&global_work_list, &e->n); + struct event_cork *ec = e->cork; + + uint ping = 0; + + if (ec) + { + LOCK_DOMAIN(cork, ec->lock); + LOCK_DOMAIN(event, l->lock); + + if (!enlisted(&e->n)) + if (ec->count) + add_tail(&ec->events, &e->n); + else + { + add_tail(&l->events, &e->n); + ping = 1; + } + + UNLOCK_DOMAIN(event, l->lock); + UNLOCK_DOMAIN(cork, ec->lock); + } + else + { + LOCK_DOMAIN(event, l->lock); + + if (!enlisted(&e->n)) + { + add_tail(&l->events, &e->n); + ping = 1; + } + + UNLOCK_DOMAIN(event, l->lock); + } + + if (ping) + birdloop_ping(l->loop); } void io_log_event(void *hook, void *data); @@ -142,35 +163,65 @@ void io_log_event(void *hook, void *data); int ev_run_list(event_list *l) { + const _Bool legacy = LEGACY_EVENT_LIST(l); + + if (legacy) + ASSERT_THE_BIRD_LOCKED; + node *n; - list tmp_list; + list tmp_list; init_list(&tmp_list); - add_tail_list(&tmp_list, l); - init_list(l); + + /* Move the event list contents to a local list to avoid executing repeatedly added events */ + LOCK_DOMAIN(event, l->lock); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + UNLOCK_DOMAIN(event, l->lock); + WALK_LIST_FIRST(n, tmp_list) { event *e = SKIP_BACK(event, n, n); + ASSERT_DIE(n->next->prev == n); - /* This is ugly hack, we want to log just events executed from the main I/O loop */ - if ((l == &global_event_list) || (l == &global_work_list)) + if (legacy) + { + /* The legacy way of event execution */ io_log_event(e->hook, e->data); - - ev_run(e); + ev_postpone(e); + e->hook(e->data); + } + else + { + // io_log_event(e->hook, e->data); /* TODO: add support for event logging in other io loops */ + ASSERT_DIE(e->list == l); + LOCK_DOMAIN(event, l->lock); + rem_node(&e->n); + UNLOCK_DOMAIN(event, l->lock); + e->hook(e->data); + } } - return !EMPTY_LIST(*l); + LOCK_DOMAIN(event, l->lock); + int repeat = ! EMPTY_LIST(l->events); + UNLOCK_DOMAIN(event, l->lock); + return repeat; } int ev_run_list_limited(event_list *l, uint limit) { + ASSERT_DIE(LEGACY_EVENT_LIST(l)); + ASSERT_THE_BIRD_LOCKED; + node *n; list tmp_list; + LOCK_DOMAIN(event, l->lock); init_list(&tmp_list); - add_tail_list(&tmp_list, l); - init_list(l); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + UNLOCK_DOMAIN(event, l->lock); WALK_LIST_FIRST(n, tmp_list) { @@ -179,21 +230,66 @@ ev_run_list_limited(event_list *l, uint limit) if (!limit) break; - /* This is ugly hack, we want to log just events executed from the main I/O loop */ - if ((l == &global_event_list) || (l == &global_work_list)) - io_log_event(e->hook, e->data); + io_log_event(e->hook, e->data); ev_run(e); limit--; } + LOCK_DOMAIN(event, l->lock); if (!EMPTY_LIST(tmp_list)) { /* Attach new items after the unprocessed old items */ - add_tail_list(&tmp_list, l); - init_list(l); - add_tail_list(l, &tmp_list); + add_tail_list(&tmp_list, &l->events); + init_list(&l->events); + add_tail_list(&l->events, &tmp_list); } - return !EMPTY_LIST(*l); + int repeat = ! EMPTY_LIST(l->events); + UNLOCK_DOMAIN(event, l->lock); + + return repeat; +} + +void ev_cork(struct event_cork *ec) +{ + LOCK_DOMAIN(cork, ec->lock); + ec->count++; + UNLOCK_DOMAIN(cork, ec->lock); +} + +void ev_uncork(struct event_cork *ec) +{ + LOCK_DOMAIN(cork, ec->lock); + + if (--ec->count) + { + UNLOCK_DOMAIN(cork, ec->lock); + return; + } + + node *n; + WALK_LIST_FIRST(n, ec->events) + { + event *e = SKIP_BACK(event, n, n); + event_list *el = e->list; + + rem_node(&e->n); + + LOCK_DOMAIN(event, el->lock); + add_tail(&el->events, &e->n); + UNLOCK_DOMAIN(event, el->lock); + + birdloop_ping(el->loop); + } + + struct birdsock *sk; + WALK_LIST_FIRST2(sk, cork_node, ec->sockets) + { +// log(L_TRACE "Socket %p uncorked", sk); + rem_node(&sk->cork_node); + sk_ping(sk); + } + + UNLOCK_DOMAIN(cork, ec->lock); } diff --git a/lib/event.h b/lib/event.h index 5f3b78d8..cbabfc66 100644 --- a/lib/event.h +++ b/lib/event.h @@ -10,33 +10,95 @@ #define _BIRD_EVENT_H_ #include "lib/resource.h" +#include "lib/locking.h" + +#include <stdatomic.h> + +DEFINE_DOMAIN(event); +DEFINE_DOMAIN(cork); typedef struct event { resource r; void (*hook)(void *); void *data; node n; /* Internal link */ + struct event_list *list; /* List where this event is put in */ + struct event_cork *cork; /* Event execution limiter */ + node cork_node; } event; -typedef list event_list; +typedef struct event_list { + list events; + pool *pool; + struct birdloop *loop; + DOMAIN(event) lock; +} event_list; + +struct event_cork { + DOMAIN(cork) lock; + u32 count; + list events; + list sockets; +}; extern event_list global_event_list; extern event_list global_work_list; event *ev_new(pool *); void ev_run(event *); -#define ev_init_list(el) init_list(el) -void ev_enqueue(event_list *, event *); -void ev_schedule(event *); -void ev_schedule_work(event *); + +static inline void ev_init_list(event_list *el, struct birdloop *loop, const char *name) +{ + init_list(&el->events); + el->loop = loop; + el->lock = DOMAIN_NEW(event, name); +} + +static inline void ev_init_cork(struct event_cork *ec, const char *name) +{ + init_list(&ec->events); + init_list(&ec->sockets); + ec->lock = DOMAIN_NEW(cork, name); + ec->count = 0; +}; + +void ev_send(event_list *, event *); +#define ev_send_self(e) ({ ASSERT_DIE((e)->list); ev_send((e)->list, (e)); }) +#define ev_send_loop(l, e) ev_send(birdloop_event_list((l)), (e)) + +#define ev_schedule(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_event_list, (e)); }) +#define ev_schedule_work(e) ({ ASSERT_THE_BIRD_LOCKED; if (!ev_active((e))) ev_send(&global_work_list, (e)); }) + void ev_postpone(event *); int ev_run_list(event_list *); int ev_run_list_limited(event_list *, uint); +#define LEGACY_EVENT_LIST(l) (((l) == &global_event_list) || ((l) == &global_work_list)) + +void ev_cork(struct event_cork *); +void ev_uncork(struct event_cork *); + +static inline u32 ev_corked(struct event_cork *ec) +{ + if (!ec) + return 0; + + LOCK_DOMAIN(cork, ec->lock); + u32 out = ec->count; + UNLOCK_DOMAIN(cork, ec->lock); + return out; +} + +_Bool birdloop_inside(struct birdloop *loop); + static inline int ev_active(event *e) { - return e->n.next != NULL; + if (e->list == NULL) + return 0; + + ASSERT_DIE(birdloop_inside(e->list->loop)); + return enlisted(&e->n); } static inline event* diff --git a/lib/event_test.c b/lib/event_test.c index e1215bba..f8bb303b 100644 --- a/lib/event_test.c +++ b/lib/event_test.c @@ -48,14 +48,17 @@ init_event_check_points(void) event_check_points[i] = 0; } +void resource_sys_init(void); + static int t_ev_run_list(void) { int i; + resource_sys_init(); resource_init(); + birdloop_init(); olock_init(); - timer_init(); io_init(); rt_init(); if_init(); @@ -82,7 +85,9 @@ main(int argc, char *argv[]) { bt_init(argc, argv); + the_bird_lock(); bt_test_suite(t_ev_run_list, "Schedule and run 3 events in right order."); + the_bird_unlock(); return bt_exit_value(); } diff --git a/lib/flowspec_test.c b/lib/flowspec_test.c index ed4afe51..518b8fc7 100644 --- a/lib/flowspec_test.c +++ b/lib/flowspec_test.c @@ -8,6 +8,7 @@ #include "test/birdtest.h" #include "lib/flowspec.h" +#include "lib/io-loop.h" #define NET_ADDR_FLOW4_(what,prefix,pxlen,data_) \ do \ @@ -446,8 +447,6 @@ t_validation6(void) static int t_builder4(void) { - resource_init(); - struct flow_builder *fb = flow_builder_init(&root_pool); linpool *lp = lp_new_default(&root_pool); @@ -529,7 +528,6 @@ t_builder6(void) { net_addr_ip6 ip; - resource_init(); linpool *lp = lp_new_default(&root_pool); struct flow_builder *fb = flow_builder_init(&root_pool); fb->ipv6 = 1; @@ -666,10 +664,16 @@ t_formatting6(void) return 1; } +void resource_sys_init(void); + int main(int argc, char *argv[]) { bt_init(argc, argv); + resource_sys_init(); + resource_init(); + the_bird_lock(); + birdloop_init(); bt_test_suite(t_read_length, "Testing get NLRI length"); bt_test_suite(t_write_length, "Testing set NLRI length"); @@ -685,5 +689,6 @@ main(int argc, char *argv[]) bt_test_suite(t_formatting4, "Formatting Flow Specification (IPv4) into text representation"); bt_test_suite(t_formatting6, "Formatting Flow Specification (IPv6) into text representation"); + the_bird_unlock(); return bt_exit_value(); } @@ -215,6 +215,12 @@ mem_hash_mix(u64 *h, const void *p, uint s) *h = *h * multiplier + pp[i]; } +static inline void +mem_hash_mix_num(u64 *h, u64 val) +{ + mem_hash_mix(h, &val, sizeof(val)); +} + static inline uint mem_hash_value(u64 *h) { diff --git a/lib/hash_test.c b/lib/hash_test.c index 59beb7c0..7ef54662 100644 --- a/lib/hash_test.c +++ b/lib/hash_test.c @@ -9,7 +9,9 @@ #undef LOCAL_DEBUG #include "test/birdtest.h" +#include "test/bt-utils.h" +#include "lib/io-loop.h" #include "lib/hash.h" struct test_node { @@ -61,8 +63,7 @@ dump_nodes(void) static void init_hash_(uint order) { - resource_init(); - my_pool = rp_new(&root_pool, "Test pool"); + my_pool = rp_new(&root_pool, &main_birdloop, "Test pool"); HASH_INIT(hash, my_pool, order); @@ -290,6 +291,7 @@ int main(int argc, char *argv[]) { bt_init(argc, argv); + bt_bird_init(); bt_test_suite(t_insert_find, "HASH_INSERT and HASH_FIND"); bt_test_suite(t_insert_find_random, "HASH_INSERT pseudo-random keys and HASH_FIND"); diff --git a/lib/io-loop.h b/lib/io-loop.h new file mode 100644 index 00000000..d60fb1ae --- /dev/null +++ b/lib/io-loop.h @@ -0,0 +1,58 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_IO_LOOP_H_ +#define _BIRD_IO_LOOP_H_ + +#include "nest/bird.h" +#include "lib/lists.h" +#include "lib/locking.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/socket.h" + +void sk_start(sock *s); +void sk_stop(sock *s); +void sk_reloop(sock *s, struct birdloop *loop); +void sk_ping(sock *s); + +extern struct birdloop main_birdloop; + +/* Start a new birdloop owned by given pool and domain. + * The loop allocates its internal pool for local allocations + * which is freed when the loop itself is stopped. */ +struct birdloop *birdloop_new(pool *p, uint order, const char *name); + +/* Stop the loop. At the end, the @stopped callback is called with locked + * parent to finish cleanup. The loop then frees itself together with its pool. */ +void birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data); +void birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data); + +/* Get birdloop's event list */ +event_list *birdloop_event_list(struct birdloop *loop); + +/* Get birdloop's time heap */ +struct timeloop *birdloop_time_loop(struct birdloop *loop); + +/* Get birdloop's resource pool */ +pool *birdloop_pool(struct birdloop *loop); + +/* Enter and exit the birdloop */ +void birdloop_enter(struct birdloop *loop); +void birdloop_leave(struct birdloop *loop); + +_Bool birdloop_inside(struct birdloop *loop); + +void birdloop_mask_wakeups(struct birdloop *loop); +void birdloop_unmask_wakeups(struct birdloop *loop); + +void birdloop_link(struct birdloop *loop); +void birdloop_unlink(struct birdloop *loop); + +void birdloop_ping(struct birdloop *loop); + +void birdloop_init(void); +#endif /* _BIRD_IO_LOOP_H_ */ diff --git a/lib/lists.c b/lib/lists.c index 200576cf..2f92defe 100644 --- a/lib/lists.c +++ b/lib/lists.c @@ -26,7 +26,7 @@ #define _BIRD_LISTS_C_ -#include "nest/bird.h" +#include "lib/birdlib.h" #include "lib/lists.h" LIST_INLINE int @@ -35,11 +35,12 @@ check_list(list *l, node *n) if (!l) { ASSERT_DIE(n); - ASSERT_DIE(n->prev); - do { n = n->prev; } while (n->prev); + node *nn = n; + while (nn->prev) + nn = nn->prev; - l = SKIP_BACK(list, head_node, n); + l = SKIP_BACK(list, head_node, nn); } int seen = 0; @@ -60,7 +61,7 @@ check_list(list *l, node *n) } ASSERT_DIE(cur == &(l->tail_node)); - ASSERT_DIE(!n || (seen == 1)); + ASSERT_DIE(!n || (seen == 1) || (n == &l->head_node) || (n == &l->tail_node)); return 1; } @@ -109,6 +110,15 @@ add_head(list *l, node *n) l->head = n; } +LIST_INLINE void +self_link(node *n) +{ + ASSUME(n->prev == NULL); + ASSUME(n->next == NULL); + + n->prev = n->next = n; +} + /** * insert_node - insert a node to a list * @n: a new list node @@ -120,7 +130,7 @@ add_head(list *l, node *n) LIST_INLINE void insert_node(node *n, node *after) { - EXPENSIVE_CHECK(check_list(l, after)); + EXPENSIVE_CHECK(check_list(NULL, after)); ASSUME(n->prev == NULL); ASSUME(n->next == NULL); @@ -141,7 +151,7 @@ insert_node(node *n, node *after) LIST_INLINE void rem_node(node *n) { - EXPENSIVE_CHECK(check_list(NULL, n)); + EXPENSIVE_CHECK((n == n->prev) && (n == n->next) || check_list(NULL, n)); node *z = n->prev; node *x = n->next; @@ -198,6 +208,9 @@ add_tail_list(list *to, list *l) EXPENSIVE_CHECK(check_list(to, NULL)); EXPENSIVE_CHECK(check_list(l, NULL)); + ASSERT_DIE(l->head->prev == &l->head_node); + ASSERT_DIE(l->tail->next == &l->tail_node); + node *p = to->tail; node *q = l->head; diff --git a/lib/lists.h b/lib/lists.h index 479f4ed1..f36c051c 100644 --- a/lib/lists.h +++ b/lib/lists.h @@ -42,6 +42,7 @@ typedef union list { /* In fact two overlayed nodes */ }; } list; +#define STATIC_LIST_INIT(name) name = { .head = &name.tail_node, .tail = &name.head_node, .null = NULL } #define NODE (node *) #define HEAD(list) ((void *)((list).head)) @@ -68,6 +69,18 @@ typedef union list { /* In fact two overlayed nodes */ #define EMPTY_LIST(list) (!(list).head->next) +static inline _Bool +enlisted(node *n) +{ + switch ((!!n->next) + (!!n->prev)) + { + case 0: return 0; + case 2: return 1; + case 1: bug("Garbled event list node"); + } + + bug("Maths is broken. And you should see a new heaven and a new earth: for the first heaven and the first earth had been passed away."); +} #ifndef _BIRD_LISTS_C_ #define LIST_INLINE static inline @@ -78,6 +91,7 @@ typedef union list { /* In fact two overlayed nodes */ #define LIST_INLINE void add_tail(list *, node *); void add_head(list *, node *); +void self_link(node *); void rem_node(node *); void add_tail_list(list *, list *); void init_list(list *); diff --git a/lib/locking.h b/lib/locking.h new file mode 100644 index 00000000..1a8bdcd4 --- /dev/null +++ b/lib/locking.h @@ -0,0 +1,66 @@ +/* + * BIRD Library -- Locking + * + * (c) 2020--2021 Maria Matejka <mq@jmq.cz> + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_LOCKING_H_ +#define _BIRD_LOCKING_H_ + +struct domain_generic; + +/* Here define the global lock order; first to last. */ +struct lock_order { + struct domain_generic *the_bird; + struct domain_generic *proto; + struct domain_generic *rtable; + struct domain_generic *attrs; + struct domain_generic *cork; + struct domain_generic *event; + struct domain_generic *resource; +}; + +extern _Thread_local struct lock_order locking_stack; +extern _Thread_local struct domain_generic **last_locked; + +#define DOMAIN(type) struct domain__##type +#define DEFINE_DOMAIN(type) DOMAIN(type) { struct domain_generic *type; } +#define DOMAIN_ORDER(type) OFFSETOF(struct lock_order, type) + +#define DOMAIN_NEW(type, name) (DOMAIN(type)) { .type = domain_new(name, DOMAIN_ORDER(type)) } +struct domain_generic *domain_new(const char *name, uint order); + +#define DOMAIN_FREE(type, d) domain_free((d).type) +void domain_free(struct domain_generic *); + +#define DOMAIN_NULL(type) (DOMAIN(type)) {} + +#define LOCK_DOMAIN(type, d) do_lock(((d).type), &(locking_stack.type)) +#define UNLOCK_DOMAIN(type, d) do_unlock(((d).type), &(locking_stack.type)) + +#define DOMAIN_IS_LOCKED(type, d) (((d).type) == (locking_stack.type)) +#define DG_IS_LOCKED(d) ((d) == *(DG_LSP(d))) + +/* Internal for locking */ +void do_lock(struct domain_generic *dg, struct domain_generic **lsp); +void do_unlock(struct domain_generic *dg, struct domain_generic **lsp); + +uint dg_order(struct domain_generic *dg); + +#define DG_LSP(d) ((struct domain_generic **) (((void *) &locking_stack) + dg_order(d))) +#define DG_LOCK(d) do_lock(d, DG_LSP(d)) +#define DG_UNLOCK(d) do_unlock(d, DG_LSP(d)) + +/* Use with care. To be removed in near future. */ +DEFINE_DOMAIN(the_bird); +extern DOMAIN(the_bird) the_bird_domain; + +#define the_bird_lock() LOCK_DOMAIN(the_bird, the_bird_domain) +#define the_bird_unlock() UNLOCK_DOMAIN(the_bird, the_bird_domain) +#define the_bird_locked() DOMAIN_IS_LOCKED(the_bird, the_bird_domain) + +#define ASSERT_THE_BIRD_LOCKED ({ if (!the_bird_locked()) bug("The BIRD lock must be locked here: %s:%d", __FILE__, __LINE__); }) + +#endif diff --git a/lib/mempool.c b/lib/mempool.c index 90d7c774..6d157e5c 100644 --- a/lib/mempool.c +++ b/lib/mempool.c @@ -37,9 +37,10 @@ const int lp_chunk_size = sizeof(struct lp_chunk); struct linpool { resource r; byte *ptr, *end; + pool *p; struct lp_chunk *first, *current; /* Normal (reusable) chunks */ struct lp_chunk *first_large; /* Large chunks */ - uint chunk_size, threshold, total, total_large; + uint chunk_size, threshold, total:31, use_pages:1, total_large; }; static void lp_free(resource *); @@ -69,6 +70,13 @@ linpool *lp_new(pool *p, uint blk) { linpool *m = ralloc(p, &lp_class); + m->p = p; + if (!blk) + { + m->use_pages = 1; + blk = page_size - lp_chunk_size; + } + m->chunk_size = blk; m->threshold = 3*blk/4; return m; @@ -121,7 +129,11 @@ lp_alloc(linpool *m, uint size) else { /* Need to allocate a new chunk */ - c = xmalloc(sizeof(struct lp_chunk) + m->chunk_size); + if (m->use_pages) + c = alloc_page(); + else + c = xmalloc(sizeof(struct lp_chunk) + m->chunk_size); + m->total += m->chunk_size; c->next = NULL; c->size = m->chunk_size; @@ -258,7 +270,10 @@ lp_free(resource *r) for(d=m->first; d; d = c) { c = d->next; - xfree(d); + if (m->use_pages) + free_page(d); + else + xfree(d); } for(d=m->first_large; d; d = c) { diff --git a/lib/rcu.c b/lib/rcu.c new file mode 100644 index 00000000..69f3442f --- /dev/null +++ b/lib/rcu.c @@ -0,0 +1,79 @@ +/* + * BIRD Library -- Read-Copy-Update Basic Operations + * + * (c) 2021 Maria Matejka <mq@jmq.cz> + * (c) 2021 CZ.NIC z.s.p.o. + * + * Can be freely distributed and used under the terms of the GNU GPL. + * Note: all the relevant patents shall be expired. + * + * Using the Supplementary Material for User-Level Implementations of Read-Copy-Update + * by Matthieu Desnoyers, Paul E. McKenney, Alan S. Stern, Michel R. Dagenais and Jonathan Walpole + * obtained from https://www.efficios.com/pub/rcu/urcu-supp-accepted.pdf + */ + +#include "lib/rcu.h" +#include "lib/coro.h" +#include "lib/locking.h" + +_Atomic uint rcu_gp_ctl = RCU_NEST_CNT; +_Thread_local struct rcu_coro *this_rcu_coro = NULL; + +static list rcu_coro_list; + +static struct rcu_coro main_rcu_coro; + +DEFINE_DOMAIN(resource); +static DOMAIN(resource) rcu_domain; + +static int +rcu_gp_ongoing(_Atomic uint *ctl) +{ + uint val = atomic_load(ctl); + return (val & RCU_NEST_CNT) && ((val ^ rcu_gp_ctl) & RCU_GP_PHASE); +} + +static void +update_counter_and_wait(void) +{ + atomic_fetch_xor(&rcu_gp_ctl, RCU_GP_PHASE); + struct rcu_coro *rc; + WALK_LIST(rc, rcu_coro_list) + while (rcu_gp_ongoing(&rc->ctl)) + coro_yield(); +} + +void +synchronize_rcu(void) +{ + LOCK_DOMAIN(resource, rcu_domain); + update_counter_and_wait(); + update_counter_and_wait(); + UNLOCK_DOMAIN(resource, rcu_domain); +} + +void +rcu_coro_start(struct rcu_coro *rc) +{ + LOCK_DOMAIN(resource, rcu_domain); + add_tail(&rcu_coro_list, &rc->n); + this_rcu_coro = rc; + UNLOCK_DOMAIN(resource, rcu_domain); +} + +void +rcu_coro_stop(struct rcu_coro *rc) +{ + LOCK_DOMAIN(resource, rcu_domain); + this_rcu_coro = NULL; + rem_node(&rc->n); + UNLOCK_DOMAIN(resource, rcu_domain); +} + +void +rcu_init(void) +{ + rcu_domain = DOMAIN_NEW(resource, "Read-Copy-Update"); + init_list(&rcu_coro_list); + rcu_coro_start(&main_rcu_coro); +} diff --git a/lib/rcu.h b/lib/rcu.h new file mode 100644 index 00000000..ac8fc9ce --- /dev/null +++ b/lib/rcu.h @@ -0,0 +1,55 @@ +/* + * BIRD Library -- Read-Copy-Update Basic Operations + * + * (c) 2021 Maria Matejka <mq@jmq.cz> + * (c) 2021 CZ.NIC z.s.p.o. + * + * Can be freely distributed and used under the terms of the GNU GPL. + * Note: all the relevant patents shall be expired. + */ + +#ifndef _BIRD_RCU_H_ +#define _BIRD_RCU_H_ + +#include "lib/birdlib.h" +#include "lib/lists.h" +#include <stdatomic.h> + +#define RCU_GP_PHASE 0x100000 +#define RCU_NEST_MASK 0x0fffff +#define RCU_NEST_CNT 0x000001 + +extern _Atomic uint rcu_gp_ctl; + +struct rcu_coro { + node n; + _Atomic uint ctl; +}; + +extern _Thread_local struct rcu_coro *this_rcu_coro; + +static inline void rcu_read_lock(void) +{ + uint cmp = atomic_load_explicit(&this_rcu_coro->ctl, memory_order_acquire); + + if (cmp & RCU_NEST_MASK) + atomic_store_explicit(&this_rcu_coro->ctl, cmp + RCU_NEST_CNT, memory_order_relaxed); + else + atomic_store(&this_rcu_coro->ctl, atomic_load_explicit(&rcu_gp_ctl, memory_order_acquire)); +} + +static inline void rcu_read_unlock(void) +{ + atomic_fetch_sub(&this_rcu_coro->ctl, RCU_NEST_CNT); +} + +void synchronize_rcu(void); + +/* Registering and unregistering a coroutine. To be called from coroutine implementation */ +void rcu_coro_start(struct rcu_coro *); +void rcu_coro_stop(struct rcu_coro *); + +/* Run this from resource init */ +void rcu_init(void); + +#endif diff --git a/lib/resource.c b/lib/resource.c index 5d4c7780..9a4c7717 100644 --- a/lib/resource.c +++ b/lib/resource.c @@ -14,6 +14,8 @@ #include "nest/bird.h" #include "lib/resource.h" #include "lib/string.h" +#include "lib/rcu.h" +#include "lib/io-loop.h" /** * DOC: Resource pools @@ -29,12 +31,6 @@ * is freed upon shutdown of the module. */ -struct pool { - resource r; - list inside; - const char *name; -}; - static void pool_dump(resource *); static void pool_free(resource *); static resource *pool_lookup(resource *, unsigned long); @@ -56,26 +52,39 @@ static int indent; /** * rp_new - create a resource pool * @p: parent pool + * @l: loop to assign * @name: pool name (to be included in debugging dumps) * * rp_new() creates a new resource pool inside the specified * parent pool. */ pool * -rp_new(pool *p, const char *name) +rp_new(pool *p, struct birdloop *loop, const char *name) { + ASSERT_DIE(birdloop_inside(p->loop)); + ASSERT_DIE(birdloop_inside(loop)); + pool *z = ralloc(p, &pool_class); + z->loop = loop; z->name = name; init_list(&z->inside); return z; } +_Thread_local static pool *pool_parent = NULL; + static void pool_free(resource *P) { + ASSERT_DIE(pool_parent); + pool *p = (pool *) P; - resource *r, *rr; + ASSERT_DIE(birdloop_inside(p->loop)); + + pool *parent = pool_parent; + pool_parent = p; + resource *r, *rr; r = HEAD(p->inside); while (rr = (resource *) r->n.next) { @@ -83,14 +92,25 @@ pool_free(resource *P) xfree(r); r = rr; } + + pool_parent = parent; +} + +void +rp_free(pool *p, pool *parent) +{ + ASSERT_DIE(pool_parent == NULL); + pool_parent = parent; + rfree(p); + ASSERT_DIE(pool_parent == parent); + pool_parent = NULL; } static void -pool_dump(resource *P) +pool_dump_locked(pool *p) { - pool *p = (pool *) P; resource *r; - + debug("%s\n", p->name); indent += 3; WALK_LIST(r, p->inside) @@ -98,10 +118,47 @@ pool_dump(resource *P) indent -= 3; } -static struct resmem -pool_memsize(resource *P) +static void +pool_dump(resource *P) { pool *p = (pool *) P; + + if (p->loop != pool_parent->loop) + birdloop_enter(p->loop); + + pool *parent = pool_parent; + pool_parent = p; + + pool_dump_locked(p); + + pool_parent = parent; + + if (p->loop != pool_parent->loop) + birdloop_leave(p->loop); +} + +void +rp_dump(pool *p) +{ + int inside = birdloop_inside(p->loop); + if (!inside) + birdloop_enter(p->loop); + + ASSERT_DIE(pool_parent == NULL); + pool_parent = p; + + pool_dump_locked(p); + + ASSERT_DIE(pool_parent == p); + pool_parent = NULL; + + if (!inside) + birdloop_leave(p->loop); +} + +static struct resmem +pool_memsize_locked(pool *p) +{ resource *r; struct resmem sum = { .effective = 0, @@ -118,6 +175,46 @@ pool_memsize(resource *P) return sum; } +static struct resmem +pool_memsize(resource *P) +{ + pool *p = (pool *) P; + + pool *parent = pool_parent; + pool_parent = p; + + if (p->loop != parent->loop) + birdloop_enter(p->loop); + + struct resmem sum = pool_memsize_locked(p); + + if (p->loop != parent->loop) + birdloop_leave(p->loop); + + pool_parent = parent; + + return sum; +} + +struct resmem +rp_memsize(pool *p) +{ + int inside = birdloop_inside(p->loop); + if (!inside) + birdloop_enter(p->loop); + + ASSERT_DIE(pool_parent == NULL); + pool_parent = p; + struct resmem sum = pool_memsize_locked(p); + ASSERT_DIE(pool_parent == p); + pool_parent = NULL; + + if (!inside) + birdloop_leave(p->loop); + + return sum; +} + static resource * pool_lookup(resource *P, unsigned long a) { @@ -228,12 +325,15 @@ rmemsize(void *res) void * ralloc(pool *p, struct resclass *c) { + ASSERT_DIE(p); + ASSERT_DIE(birdloop_inside(p->loop)); + resource *r = xmalloc(c->size); bzero(r, c->size); r->class = c; - if (p) - add_tail(&p->inside, &r->n); + add_tail(&p->inside, &r->n); + return r; } @@ -270,6 +370,8 @@ rlookup(unsigned long a) void resource_init(void) { + rcu_init(); + root_pool.r.class = &pool_class; root_pool.name = "Root"; init_list(&root_pool.inside); @@ -440,7 +542,6 @@ mb_free(void *m) } - #define STEP_UP(x) ((x) + (x)/2 + 4) void diff --git a/lib/resource.h b/lib/resource.h index 9ec41ed8..3e589e32 100644 --- a/lib/resource.h +++ b/lib/resource.h @@ -40,10 +40,15 @@ struct resclass { /* Generic resource manipulation */ -typedef struct pool pool; +typedef struct pool { + resource r; + list inside; + struct birdloop *loop; + const char *name; +} pool; void resource_init(void); -pool *rp_new(pool *, const char *); /* Create new pool */ + void rfree(void *); /* Free single resource */ void rdump(void *); /* Dump to debug output */ struct resmem rmemsize(void *res); /* Return size of memory used by the resource */ @@ -52,6 +57,11 @@ void rmove(void *, pool *); /* Move to a different pool */ void *ralloc(pool *, struct resclass *); +pool *rp_new(pool *, struct birdloop *loop, const char *); /* Create new pool */ +void rp_free(pool *p, pool *parent); /* Free parent pool */ +struct resmem rp_memsize(pool *p); /* Return size of memory used by the pool */ +void rp_dump(pool *p); /* Dump pool to debug output */ + extern pool root_pool; /* Normal memory blocks */ @@ -82,7 +92,7 @@ void lp_restore(linpool *m, lp_state *p); /* Restore state */ extern const int lp_chunk_size; #define LP_GAS 1024 #define LP_GOOD_SIZE(x) (((x + LP_GAS - 1) & (~(LP_GAS - 1))) - lp_chunk_size) -#define lp_new_default(p) lp_new(p, LP_GOOD_SIZE(LP_GAS*4)) +#define lp_new_default(p) lp_new(p, 0) /* Slabs */ @@ -100,11 +110,12 @@ void sl_free(slab *, void *); void buffer_realloc(void **buf, unsigned *size, unsigned need, unsigned item_size); +extern long page_size; + /* Allocator of whole pages; for use in slabs and other high-level allocators. */ -u64 get_page_size(void); void *alloc_page(void); void free_page(void *); -extern uint pages_kept; +#define PAGE_HEAD(x) ((void *) (((intptr_t) (x)) & ~(page_size-1))) #ifdef HAVE_LIBDMALLOC /* @@ -155,6 +155,7 @@ slab_memsize(resource *r) struct slab { resource r; + pool *p; uint obj_size, head_size, head_bitfield_len; uint objs_per_slab, num_empty_heads, data_size; list empty_heads, partial_heads, full_heads; @@ -180,7 +181,7 @@ struct sl_alignment { /* Magic structure for testing of alignment */ int x[0]; }; -#define SL_GET_HEAD(x) ((struct sl_head *) (((uintptr_t) (x)) & ~(get_page_size()-1))) +#define SL_GET_HEAD(x) ((struct sl_head *) PAGE_HEAD(x)) /** * sl_new - create a new Slab @@ -194,6 +195,7 @@ slab * sl_new(pool *p, uint size) { slab *s = ralloc(p, &sl_class); + s->p = p; uint align = sizeof(struct sl_alignment); if (align < sizeof(int)) align = sizeof(int); @@ -202,7 +204,6 @@ sl_new(pool *p, uint size) s->obj_size = size; s->head_size = sizeof(struct sl_head); - u64 page_size = get_page_size(); do { s->objs_per_slab = (page_size - s->head_size) / size; @@ -272,6 +273,9 @@ no_partial: goto okay; } h = alloc_page(); +#ifdef POISON + memset(h, 0xba, page_size); +#endif ASSERT_DIE(SL_GET_HEAD(h) == h); memset(h, 0, s->head_size); add_head(&s->partial_heads, &h->n); @@ -327,7 +331,12 @@ sl_free(slab *s, void *oo) { rem_node(&h->n); if (s->num_empty_heads >= MAX_EMPTY_HEADS) + { +#ifdef POISON + memset(h, 0xde, page_size); +#endif free_page(h); + } else { add_head(&s->empty_heads, &h->n); @@ -391,7 +400,7 @@ slab_memsize(resource *r) return (struct resmem) { .effective = eff, - .overhead = ALLOC_OVERHEAD + sizeof(struct slab) + heads * get_page_size() - eff, + .overhead = ALLOC_OVERHEAD + sizeof(struct slab) + heads * page_size - eff, }; } @@ -402,10 +411,10 @@ slab_lookup(resource *r, unsigned long a) struct sl_head *h; WALK_LIST(h, s->partial_heads) - if ((unsigned long) h < a && (unsigned long) h + get_page_size() < a) + if ((unsigned long) h < a && (unsigned long) h + page_size < a) return r; WALK_LIST(h, s->full_heads) - if ((unsigned long) h < a && (unsigned long) h + get_page_size() < a) + if ((unsigned long) h < a && (unsigned long) h + page_size < a) return r; return NULL; } diff --git a/lib/socket.h b/lib/socket.h index 96fedeeb..89398edf 100644 --- a/lib/socket.h +++ b/lib/socket.h @@ -12,6 +12,7 @@ #include <errno.h> #include "lib/resource.h" +#include "lib/event.h" #ifdef HAVE_LIBSSH #define LIBSSH_LEGACY_0_4 #include <libssh/libssh.h> @@ -39,6 +40,7 @@ struct ssh_sock { typedef struct birdsock { resource r; pool *pool; /* Pool where incoming connections should be allocated (for SK_xxx_PASSIVE) */ + struct birdloop *loop; /* The birdloop where this socket belongs to */ int type; /* Socket type */ int subtype; /* Socket subtype */ void *data; /* User data */ @@ -56,6 +58,8 @@ typedef struct birdsock { uint fast_rx; /* RX has higher priority in event loop */ uint rbsize; int (*rx_hook)(struct birdsock *, uint size); /* NULL=receiving turned off, returns 1 to clear rx buffer */ + struct event_cork *cork; /* Cork to temporarily stop receiving data */ + node cork_node; /* Node in cork list */ byte *tbuf, *tpos; /* NULL=allocate automatically */ byte *ttx; /* Internal */ @@ -79,6 +83,7 @@ typedef struct birdsock { const char *password; /* Password for MD5 authentication */ const char *err; /* Error message */ struct ssh_sock *ssh; /* Used in SK_SSH */ + struct event reloop; /* Reloop event */ } sock; sock *sock_new(pool *); /* Allocate new socket */ @@ -128,6 +133,7 @@ extern int sk_priority_control; /* Suggested priority for control traffic, shou #define SKF_TRUNCATED 0x200 /* Received packet was truncated, set by IO layer */ #define SKF_HDRINCL 0x400 /* Used internally */ #define SKF_PKTINFO 0x800 /* Used internally */ +#define SKF_PASSIVE_THREAD 0x1000 /* Child sockets used in thread, do not add to main loop */ /* * Socket types SA SP DA DP IF TTL SendTo (?=may, -=must not, *=must) diff --git a/lib/timer.c b/lib/timer.c index 381163d0..eb7ea690 100644 --- a/lib/timer.c +++ b/lib/timer.c @@ -32,61 +32,18 @@ #include "nest/bird.h" +#include "lib/coro.h" #include "lib/heap.h" #include "lib/resource.h" #include "lib/timer.h" - -struct timeloop main_timeloop; - - -#ifdef USE_PTHREADS - #include <pthread.h> -/* Data accessed and modified from proto/bfd/io.c */ -pthread_key_t current_time_key; - -static inline struct timeloop * -timeloop_current(void) -{ - return pthread_getspecific(current_time_key); -} - -static inline void -timeloop_init_current(void) -{ - pthread_key_create(¤t_time_key, NULL); - pthread_setspecific(current_time_key, &main_timeloop); -} +_Atomic btime last_time; +_Atomic btime real_time; void wakeup_kick_current(void); -#else - -/* Just use main timelooop */ -static inline struct timeloop * timeloop_current(void) { return &main_timeloop; } -static inline void timeloop_init_current(void) { } - -#endif - -btime -current_time(void) -{ - return timeloop_current()->last_time; -} - -btime -current_real_time(void) -{ - struct timeloop *loop = timeloop_current(); - - if (!loop->real_time) - times_update_real_time(loop); - - return loop->real_time; -} - #define TIMER_LESS(a,b) ((a)->expires < (b)->expires) #define TIMER_SWAP(heap,a,b,t) (t = heap[a], heap[a] = heap[b], heap[b] = t, \ @@ -112,7 +69,7 @@ tm_dump(resource *r) if (t->recurrent) debug("recur %d, ", t->recurrent); if (t->expires) - debug("expires in %d ms)\n", (t->expires - current_time()) TO_MS); + debug("in loop %p expires in %d ms)\n", t->loop, (t->expires - current_time()) TO_MS); else debug("inactive)\n"); } @@ -135,41 +92,40 @@ tm_new(pool *p) return t; } -void -tm_set(timer *t, btime when) +static void +tm_set_in_tl(timer *t, btime when, struct timeloop *local_timeloop) { - struct timeloop *loop = timeloop_current(); - uint tc = timers_count(loop); + uint tc = timers_count(local_timeloop); if (!t->expires) { t->index = ++tc; t->expires = when; - BUFFER_PUSH(loop->timers) = t; - HEAP_INSERT(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP); + BUFFER_PUSH(local_timeloop->timers) = t; + HEAP_INSERT(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP); } else if (t->expires < when) { t->expires = when; - HEAP_INCREASE(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + HEAP_INCREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); } else if (t->expires > when) { t->expires = when; - HEAP_DECREASE(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + HEAP_DECREASE(local_timeloop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); } -#ifdef CONFIG_BFD - /* Hack to notify BFD loops */ - if ((loop != &main_timeloop) && (t->index == 1)) - wakeup_kick_current(); -#endif + t->loop = local_timeloop; + + if ((t->index == 1) && (local_timeloop->coro != this_coro)) + birdloop_ping(local_timeloop->loop); } void -tm_start(timer *t, btime after) +tm_set_in(timer *t, btime when, struct birdloop *loop) { - tm_set(t, current_time() + MAX(after, 0)); + ASSERT_DIE(birdloop_inside(loop)); + tm_set_in_tl(t, when, birdloop_time_loop(loop)); } void @@ -178,20 +134,22 @@ tm_stop(timer *t) if (!t->expires) return; - struct timeloop *loop = timeloop_current(); - uint tc = timers_count(loop); + TLOCK_TIMER_ASSERT(t->loop); - HEAP_DELETE(loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); - BUFFER_POP(loop->timers); + uint tc = timers_count(t->loop); + + HEAP_DELETE(t->loop->timers.data, tc, timer *, TIMER_LESS, TIMER_SWAP, t->index); + BUFFER_POP(t->loop->timers); t->index = -1; t->expires = 0; + t->loop = NULL; } void timers_init(struct timeloop *loop, pool *p) { - times_init(loop); + TLOCK_TIMER_ASSERT(loop); BUFFER_INIT(loop->timers, p, 4); BUFFER_PUSH(loop->timers) = NULL; @@ -200,13 +158,15 @@ timers_init(struct timeloop *loop, pool *p) void io_log_event(void *hook, void *data); void -timers_fire(struct timeloop *loop) +timers_fire(struct timeloop *loop, int io_log) { + TLOCK_TIMER_ASSERT(loop); + btime base_time; timer *t; - times_update(loop); - base_time = loop->last_time; + times_update(); + base_time = current_time(); while (t = timers_first(loop)) { @@ -217,32 +177,25 @@ timers_fire(struct timeloop *loop) { btime when = t->expires + t->recurrent; - if (when <= loop->last_time) - when = loop->last_time + t->recurrent; + if (when <= base_time) + when = base_time + t->recurrent; if (t->randomize) when += random() % (t->randomize + 1); - tm_set(t, when); + tm_set_in_tl(t, when, loop); } else tm_stop(t); /* This is ugly hack, we want to log just timers executed from the main I/O loop */ - if (loop == &main_timeloop) + if (io_log) io_log_event(t->hook, t->data); t->hook(t); } } -void -timer_init(void) -{ - timers_init(&main_timeloop, &root_pool); - timeloop_init_current(); -} - /** * tm_parse_time - parse a date and time diff --git a/lib/timer.h b/lib/timer.h index c5ea430c..161e39d3 100644 --- a/lib/timer.h +++ b/lib/timer.h @@ -12,8 +12,14 @@ #include "nest/bird.h" #include "lib/buffer.h" +#include "lib/io-loop.h" +#include "lib/locking.h" #include "lib/resource.h" +#include <stdatomic.h> + +extern _Atomic btime last_time; +extern _Atomic btime real_time; typedef struct timer { @@ -25,36 +31,44 @@ typedef struct timer uint randomize; /* Amount of randomization */ uint recurrent; /* Timer recurrence */ + struct timeloop *loop; /* Loop where the timer is active */ + int index; } timer; struct timeloop { BUFFER_(timer *) timers; - btime last_time; - btime real_time; + struct domain_generic *domain; + struct birdloop *loop; + struct coroutine *coro; }; +#define TLOCK_TIMER_ASSERT(loop) ASSERT_DIE((loop)->domain && DG_IS_LOCKED((loop)->domain)) +#define TLOCK_LOCAL_ASSERT(loop) ASSERT_DIE(!(loop)->domain || DG_IS_LOCKED((loop)->domain)) + static inline uint timers_count(struct timeloop *loop) -{ return loop->timers.used - 1; } +{ TLOCK_TIMER_ASSERT(loop); return loop->timers.used - 1; } static inline timer *timers_first(struct timeloop *loop) -{ return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } +{ TLOCK_TIMER_ASSERT(loop); return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } -extern struct timeloop main_timeloop; - -btime current_time(void); -btime current_real_time(void); +#define current_time() atomic_load_explicit(&last_time, memory_order_acquire) +#define current_real_time() atomic_load_explicit(&real_time, memory_order_acquire) +#define current_time_update() ({ times_update(); current_time(); }) //#define now (current_time() TO_S) //#define now_real (current_real_time() TO_S) extern btime boot_time; timer *tm_new(pool *p); -void tm_set(timer *t, btime when); -void tm_start(timer *t, btime after); +#define tm_set(t, when) tm_set_in((t), (when), &main_birdloop) +#define tm_start(t, after) tm_start_in((t), (after), &main_birdloop) void tm_stop(timer *t); +void tm_set_in(timer *t, btime when, struct birdloop *loop); +#define tm_start_in(t, after, loop) tm_set_in((t), (current_time() + MAX_((after), 0)), loop) + static inline int tm_active(timer *t) { @@ -94,15 +108,11 @@ tm_start_max(timer *t, btime after) } /* In sysdep code */ -void times_init(struct timeloop *loop); -void times_update(struct timeloop *loop); -void times_update_real_time(struct timeloop *loop); +void times_update(void); /* For I/O loop */ void timers_init(struct timeloop *loop, pool *p); -void timers_fire(struct timeloop *loop); - -void timer_init(void); +void timers_fire(struct timeloop *loop, int io_log); struct timeformat { diff --git a/nest/a-path_test.c b/nest/a-path_test.c index 9ed0a786..2533dbae 100644 --- a/nest/a-path_test.c +++ b/nest/a-path_test.c @@ -12,6 +12,7 @@ #include "nest/route.h" #include "nest/attrs.h" #include "lib/resource.h" +#include "lib/io-loop.h" #define TESTS_NUM 30 #define AS_PATH_LENGTH 1000 @@ -23,8 +24,6 @@ static int t_as_path_match(void) { - resource_init(); - int round; for (round = 0; round < TESTS_NUM; round++) { @@ -70,8 +69,6 @@ t_as_path_match(void) static int t_path_format(void) { - resource_init(); - struct adata empty_as_path = {}; struct adata *as_path = &empty_as_path; struct linpool *lp = lp_new_default(&root_pool); @@ -116,8 +113,6 @@ count_asn_in_array(const u32 *array, u32 asn) static int t_path_include(void) { - resource_init(); - struct adata empty_as_path = {}; struct adata *as_path = &empty_as_path; struct linpool *lp = lp_new_default(&root_pool); @@ -161,8 +156,6 @@ t_path_include(void) static int t_as_path_converting(void) { - resource_init(); - struct adata empty_as_path = {}; struct adata *as_path = &empty_as_path; struct linpool *lp = lp_new_default(&root_pool); @@ -204,10 +197,18 @@ t_as_path_converting(void) } #endif +void resource_sys_init(void); +void io_init(void); + int main(int argc, char *argv[]) { bt_init(argc, argv); + resource_sys_init(); + resource_init(); + the_bird_lock(); + birdloop_init(); + io_init(); bt_test_suite(t_as_path_match, "Testing AS path matching and some a-path utilities."); bt_test_suite(t_path_format, "Testing formating as path into byte buffer"); diff --git a/nest/a-set_test.c b/nest/a-set_test.c index 96b6a727..f8f6e781 100644 --- a/nest/a-set_test.c +++ b/nest/a-set_test.c @@ -13,6 +13,7 @@ #include "nest/route.h" #include "nest/attrs.h" #include "lib/resource.h" +#include "lib/io-loop.h" #define SET_SIZE 10 static const struct adata *set_sequence; /* <0; SET_SIZE) */ @@ -71,7 +72,6 @@ t_set_int_contains(void) { int i; - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE); bt_assert(int_set_get_size(set_sequence) == SET_SIZE); @@ -92,7 +92,6 @@ t_set_int_contains(void) static int t_set_int_union(void) { - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE); const struct adata *set_union; @@ -111,7 +110,6 @@ t_set_int_union(void) static int t_set_int_format(void) { - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE_FOR_FORMAT_OUTPUT); bt_assert(int_set_format(set_sequence, 0, 0, buf, BUFFER_SIZE) == 0); @@ -132,7 +130,6 @@ t_set_int_format(void) static int t_set_int_delete(void) { - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE); const struct adata *deleting_sequence = set_sequence; @@ -160,7 +157,6 @@ t_set_ec_contains(void) { u32 i; - resource_init(); generate_set_sequence(SET_TYPE_EC, SET_SIZE); bt_assert(ec_set_get_size(set_sequence) == SET_SIZE); @@ -181,7 +177,6 @@ t_set_ec_contains(void) static int t_set_ec_union(void) { - resource_init(); generate_set_sequence(SET_TYPE_EC, SET_SIZE); const struct adata *set_union; @@ -200,8 +195,6 @@ t_set_ec_union(void) static int t_set_ec_format(void) { - resource_init(); - const struct adata empty_as_path = {}; set_sequence = set_sequence_same = set_sequence_higher = set_random = &empty_as_path; lp = lp_new_default(&root_pool); @@ -222,7 +215,6 @@ t_set_ec_format(void) static int t_set_ec_delete(void) { - resource_init(); generate_set_sequence(SET_TYPE_EC, SET_SIZE); const struct adata *deleting_sequence = set_sequence; @@ -240,10 +232,17 @@ t_set_ec_delete(void) return 1; } + +void resource_sys_init(void); + int main(int argc, char *argv[]) { bt_init(argc, argv); + resource_sys_init(); + resource_init(); + the_bird_lock(); + birdloop_init(); bt_test_suite(t_set_int_contains, "Testing sets of integers: contains, get_data"); bt_test_suite(t_set_int_format, "Testing sets of integers: format"); @@ -23,6 +23,11 @@ struct bfd_options { u8 mode; }; +struct bfd_session_state { + u8 state; + u8 diag; +}; + struct bfd_request { resource r; node n; @@ -35,12 +40,12 @@ struct bfd_request { void (*hook)(struct bfd_request *); void *data; + event event; struct bfd_session *session; - + struct bfd_session_state old_state; u8 state; u8 diag; - u8 old_state; u8 down; }; @@ -51,13 +56,12 @@ struct bfd_request { #define BFD_STATE_INIT 2 #define BFD_STATE_UP 3 - static inline struct bfd_options * bfd_new_options(void) { return cfg_allocz(sizeof(struct bfd_options)); } #ifdef CONFIG_BFD -struct bfd_request * bfd_request_session(pool *p, ip_addr addr, ip_addr local, struct iface *iface, struct iface *vrf, void (*hook)(struct bfd_request *), void *data, const struct bfd_options *opts); +struct bfd_request * bfd_request_session(pool *p, ip_addr addr, ip_addr local, struct iface *iface, struct iface *vrf, void (*hook)(struct bfd_request *), void *data, struct event_list *list, const struct bfd_options *opts); void bfd_update_request(struct bfd_request *req, const struct bfd_options *opts); static inline void cf_check_bfd(int use UNUSED) { } @@ -262,7 +262,7 @@ cli_command(struct cli *c) log(L_TRACE "CLI: %s", c->rx_buf); bzero(&f, sizeof(f)); f.mem = c->parser_pool; - f.pool = rp_new(c->pool, "Config"); + f.pool = rp_new(c->pool, &main_birdloop, "Config"); init_list(&f.symbols); cf_read_hook = cli_cmd_read_hook; cli_rh_pos = c->rx_buf; @@ -308,7 +308,7 @@ cli_event(void *data) cli * cli_new(void *priv) { - pool *p = rp_new(cli_pool, "CLI"); + pool *p = rp_new(cli_pool, &main_birdloop, "CLI"); cli *c = mb_alloc(p, sizeof(cli)); bzero(c, sizeof(cli)); @@ -413,7 +413,7 @@ cli_free(cli *c) c->cleanup(c); if (c == cmd_reconfig_stored_cli) cmd_reconfig_stored_cli = NULL; - rfree(c->pool); + rp_free(c->pool, &root_pool); } /** @@ -425,7 +425,7 @@ cli_free(cli *c) void cli_init(void) { - cli_pool = rp_new(&root_pool, "CLI"); + cli_pool = rp_new(&root_pool, &main_birdloop, "CLI"); init_list(&cli_log_hooks); cli_log_inited = 1; } diff --git a/nest/cmds.c b/nest/cmds.c index 1a16f9c7..77c92077 100644 --- a/nest/cmds.c +++ b/nest/cmds.c @@ -114,15 +114,10 @@ cmd_show_memory(void) { cli_msg(-1018, "BIRD memory usage"); cli_msg(-1018, "%-17s Effective Overhead", ""); - print_size("Routing tables:", rmemsize(rt_table_pool)); - print_size("Route attributes:", rmemsize(rta_pool)); - print_size("Protocols:", rmemsize(proto_pool)); - struct resmem total = rmemsize(&root_pool); -#ifdef HAVE_MMAP - print_size("Standby memory:", (struct resmem) { .overhead = get_page_size() * pages_kept }); - total.overhead += get_page_size() * pages_kept; -#endif - print_size("Total:", total); + print_size("Routing tables:", rp_memsize(rt_table_pool)); + print_size("Route attributes:", rp_memsize(rta_pool)); + print_size("Protocols:", rp_memsize(proto_pool)); + print_size("Total:", rp_memsize(&root_pool)); cli_msg(0, ""); } diff --git a/nest/config.Y b/nest/config.Y index 7ead8589..f9ed0e69 100644 --- a/nest/config.Y +++ b/nest/config.Y @@ -111,7 +111,7 @@ proto_postconfig(void) CF_DECLS -CF_KEYWORDS(ROUTER, ID, HOSTNAME, PROTOCOL, TEMPLATE, PREFERENCE, DISABLED, DEBUG, ALL, OFF, DIRECT) +CF_KEYWORDS(ROUTER, ID, HOSTNAME, PROTOCOL, TEMPLATE, PREFERENCE, DISABLED, DEBUG, ALL, OFF, DIRECT, PIPE) CF_KEYWORDS(INTERFACE, IMPORT, EXPORT, FILTER, NONE, VRF, DEFAULT, TABLE, STATES, ROUTES, FILTERS) CF_KEYWORDS(IPV4, IPV6, VPN4, VPN6, ROA4, ROA6, FLOW4, FLOW6, SADR, MPLS) CF_KEYWORDS(RECEIVE, LIMIT, ACTION, WARN, BLOCK, RESTART, DISABLE, KEEP, FILTERED, RPKI) @@ -128,7 +128,7 @@ CF_KEYWORDS(CHECK, LINK) /* For r_args_channel */ CF_KEYWORDS(IPV4, IPV4_MC, IPV4_MPLS, IPV6, IPV6_MC, IPV6_MPLS, IPV6_SADR, VPN4, VPN4_MC, VPN4_MPLS, VPN6, VPN6_MC, VPN6_MPLS, ROA4, ROA6, FLOW4, FLOW6, MPLS, PRI, SEC) -CF_ENUM(T_ENUM_RTS, RTS_, DUMMY, STATIC, INHERIT, DEVICE, STATIC_DEVICE, REDIRECT, +CF_ENUM(T_ENUM_RTS, RTS_, STATIC, INHERIT, DEVICE, STATIC_DEVICE, REDIRECT, RIP, OSPF, OSPF_IA, OSPF_EXT1, OSPF_EXT2, BGP, PIPE, BABEL) CF_ENUM(T_ENUM_SCOPE, SCOPE_, HOST, LINK, SITE, ORGANIZATION, UNIVERSE, UNDEFINED) CF_ENUM(T_ENUM_RTD, RTD_, UNICAST, BLACKHOLE, UNREACHABLE, PROHIBIT) @@ -266,8 +266,8 @@ proto_item: | MRTDUMP mrtdump_mask { this_proto->mrtdump = $2; } | ROUTER ID idval { this_proto->router_id = $3; } | DESCRIPTION text { this_proto->dsc = $2; } - | VRF text { this_proto->vrf = if_get_by_name($2); this_proto->vrf_set = 1; } - | VRF DEFAULT { this_proto->vrf = NULL; this_proto->vrf_set = 1; } + | VRF text { this_proto->vrf = if_get_by_name($2); } + | VRF DEFAULT { this_proto->vrf = &default_vrf; } ; @@ -348,6 +348,7 @@ debug_default: DEBUG PROTOCOLS debug_mask { new_config->proto_default_debug = $3; } | DEBUG CHANNELS debug_mask { new_config->channel_default_debug = $3; } | DEBUG COMMANDS expr { new_config->cli_debug = $3; } + | DEBUG TABLES bool { new_config->table_debug = $3; } ; /* MRTDUMP PROTOCOLS is in systep/unix/config.Y */ @@ -433,6 +434,7 @@ proto: dev_proto '}' ; dev_proto_start: proto_start DIRECT { this_proto = proto_config_new(&proto_device, $1); init_list(&DIRECT_CFG->iface_list); + this_proto->late_if_feed = 1; } ; @@ -643,12 +645,12 @@ r_args: } | r_args IMPORT TABLE channel_arg { if (!$4->in_table) cf_error("No import table in channel %s.%s", $4->proto->name, $4->name); - rt_show_add_table($$, $4->in_table); + rt_show_add_table($$, $4->in_table->tab); $$->tables_defined_by = RSD_TDB_DIRECT; } | r_args EXPORT TABLE channel_arg { if (!$4->out_table) cf_error("No export table in channel %s.%s", $4->proto->name, $4->name); - rt_show_add_table($$, $4->out_table); + rt_show_add_table($$, $4->out_table->tab); $$->tables_defined_by = RSD_TDB_DIRECT; } | r_args FILTER filter { @@ -808,7 +810,7 @@ sym_args: CF_CLI_HELP(DUMP, ..., [[Dump debugging information]]) CF_CLI(DUMP RESOURCES,,, [[Dump all allocated resource]]) -{ rdump(&root_pool); cli_msg(0, ""); } ; +{ rp_dump(&root_pool); cli_msg(0, ""); } ; CF_CLI(DUMP SOCKETS,,, [[Dump open sockets]]) { sk_dump_all(); cli_msg(0, ""); } ; CF_CLI(DUMP EVENTS,,, [[Dump event log]]) @@ -819,8 +821,10 @@ CF_CLI(DUMP NEIGHBORS,,, [[Dump neighbor cache]]) { neigh_dump_all(); cli_msg(0, ""); } ; CF_CLI(DUMP ATTRIBUTES,,, [[Dump attribute cache]]) { rta_dump_all(); cli_msg(0, ""); } ; -CF_CLI(DUMP ROUTES,,, [[Dump routing table]]) +CF_CLI(DUMP ROUTES,,, [[Dump routes]]) { rt_dump_all(); cli_msg(0, ""); } ; +CF_CLI(DUMP TABLES,,, [[Dump table connections]]) +{ rt_dump_hooks_all(); cli_msg(0, ""); } ; CF_CLI(DUMP PROTOCOLS,,, [[Dump protocol information]]) { protos_dump_all(); cli_msg(0, ""); } ; CF_CLI(DUMP FILTER ALL,,, [[Dump all filters in linearized form]]) diff --git a/nest/iface.c b/nest/iface.c index 682340c5..78f4eb0f 100644 --- a/nest/iface.c +++ b/nest/iface.c @@ -36,9 +36,12 @@ static pool *if_pool; -list iface_list; +DOMAIN(attrs) iface_domain; +list global_iface_list; +struct iface default_vrf; static void if_recalc_preferred(struct iface *i); +static void ifa_delete_locked(struct ifa *a); /** * ifa_dump - dump interface address @@ -49,6 +52,7 @@ static void if_recalc_preferred(struct iface *i); void ifa_dump(struct ifa *a) { + IFACE_LEGACY_ACCESS; debug("\t%I, net %N bc %I -> %I%s%s%s%s\n", a->ip, &a->prefix, a->brd, a->opposite, (a->flags & IA_PRIMARY) ? " PRIMARY" : "", (a->flags & IA_SECONDARY) ? " SEC" : "", @@ -68,6 +72,7 @@ if_dump(struct iface *i) { struct ifa *a; + IFACE_LEGACY_ACCESS; debug("IF%d: %s", i->index, i->name); if (i->flags & IF_SHUTDOWN) debug(" SHUTDOWN"); @@ -109,8 +114,9 @@ if_dump_all(void) { struct iface *i; + IFACE_LEGACY_ACCESS; debug("Known network interfaces:\n"); - WALK_LIST(i, iface_list) + WALK_LIST(i, global_iface_list) if_dump(i); debug("Router ID: %08x\n", config->router_id); } @@ -147,7 +153,7 @@ ifa_send_notify(struct proto *p, unsigned c, struct ifa *a) { if (p->ifa_notify && (p->proto_state != PS_DOWN) && - (!p->vrf_set || p->vrf == a->iface->master)) + (!p->vrf || p->vrf == a->iface->master)) { if (p->debug & D_IFACES) log(L_TRACE "%s < address %N on interface %s %s", @@ -165,7 +171,8 @@ ifa_notify_change_(unsigned c, struct ifa *a) DBG("IFA change notification (%x) for %s:%I\n", c, a->iface->name, a->ip); WALK_LIST(p, proto_list) - ifa_send_notify(p, c, a); + PROTO_LOCKED_FROM_MAIN(p) + ifa_send_notify(p, c, a); } static inline void @@ -185,7 +192,7 @@ if_send_notify(struct proto *p, unsigned c, struct iface *i) { if (p->if_notify && (p->proto_state != PS_DOWN) && - (!p->vrf_set || p->vrf == i->master)) + (!p->vrf || p->vrf == i->master)) { if (p->debug & D_IFACES) log(L_TRACE "%s < interface %s %s", p->name, i->name, @@ -225,7 +232,8 @@ if_notify_change(unsigned c, struct iface *i) ifa_notify_change_(IF_CHANGE_DOWN, a); WALK_LIST(p, proto_list) - if_send_notify(p, c, i); + PROTO_LOCKED_FROM_MAIN(p) + if_send_notify(p, c, i); if (c & IF_CHANGE_UP) WALK_LIST(a, i->addrs) @@ -243,7 +251,7 @@ if_recalc_flags(struct iface *i UNUSED, uint flags) { if ((flags & IF_ADMIN_UP) && !(flags & (IF_SHUTDOWN | IF_TMP_DOWN)) && - !(i->master_index && !i->master)) + !(i->master_index && i->master == &default_vrf)) flags |= IF_UP; else flags &= ~IF_UP; @@ -301,7 +309,13 @@ if_update(struct iface *new) struct iface *i; unsigned c; - WALK_LIST(i, iface_list) + if (!new->master) + new->master = &default_vrf; + + IFACE_LEGACY_ACCESS; + IFACE_LOCK; + + WALK_LIST(i, global_iface_list) if (!strcmp(new->name, i->name)) { new->flags = if_recalc_flags(new, new->flags); @@ -322,6 +336,8 @@ if_update(struct iface *new) } if_copy(i, new); + IFACE_UNLOCK; + if (c) if_notify_change(c, i); @@ -334,7 +350,9 @@ if_update(struct iface *new) newif: init_list(&i->neighbors); i->flags |= IF_UPDATED | IF_TMP_DOWN; /* Tmp down as we don't have addresses yet */ - add_tail(&iface_list, &i->n); + add_tail(&global_iface_list, &i->n); + IFACE_UNLOCK; + return i; } @@ -344,7 +362,8 @@ if_start_update(void) struct iface *i; struct ifa *a; - WALK_LIST(i, iface_list) + IFACE_LEGACY_ACCESS; + WALK_LIST(i, global_iface_list) { i->flags &= ~IF_UPDATED; WALK_LIST(a, i->addrs) @@ -355,6 +374,8 @@ if_start_update(void) void if_end_partial_update(struct iface *i) { + IFACE_LEGACY_ACCESS; + if (i->flags & IF_NEEDS_RECALC) if_recalc_preferred(i); @@ -368,7 +389,8 @@ if_end_update(void) struct iface *i; struct ifa *a, *b; - WALK_LIST(i, iface_list) + IFACE_LEGACY_ACCESS; + WALK_LIST(i, global_iface_list) { if (!(i->flags & IF_UPDATED)) if_change_flags(i, (i->flags & ~IF_ADMIN_UP) | IF_SHUTDOWN); @@ -376,7 +398,11 @@ if_end_update(void) { WALK_LIST_DELSAFE(a, b, i->addrs) if (!(a->flags & IA_UPDATED)) - ifa_delete(a); + { + IFACE_LOCK; + ifa_delete_locked(a); + IFACE_UNLOCK; + } if_end_partial_update(i); } } @@ -385,6 +411,7 @@ if_end_update(void) void if_flush_ifaces(struct proto *p) { + IFACE_LEGACY_ACCESS; if (p->debug & D_EVENTS) log(L_TRACE "%s: Flushing interfaces", p->name); if_start_update(); @@ -404,10 +431,12 @@ if_feed_baby(struct proto *p) struct iface *i; struct ifa *a; + IFACE_LEGACY_ACCESS; + if (!p->if_notify && !p->ifa_notify) /* shortcut */ return; DBG("Announcing interfaces to new protocol %s\n", p->name); - WALK_LIST(i, iface_list) + WALK_LIST(i, global_iface_list) { if_send_notify(p, IF_CHANGE_CREATE | ((i->flags & IF_UP) ? IF_CHANGE_UP : 0), i); if (i->flags & IF_UP) @@ -429,9 +458,15 @@ if_find_by_index(unsigned idx) { struct iface *i; - WALK_LIST(i, iface_list) + IFACE_LOCK; + WALK_LIST(i, global_iface_list) if (i->index == idx && !(i->flags & IF_SHUTDOWN)) + { + IFACE_UNLOCK; return i; + } + + IFACE_UNLOCK; return NULL; } @@ -448,9 +483,15 @@ if_find_by_name(const char *name) { struct iface *i; - WALK_LIST(i, iface_list) + IFACE_LOCK; + WALK_LIST(i, global_iface_list) if (!strcmp(i->name, name) && !(i->flags & IF_SHUTDOWN)) + { + IFACE_UNLOCK; return i; + } + + IFACE_UNLOCK; return NULL; } @@ -459,17 +500,21 @@ if_get_by_name(const char *name) { struct iface *i; - WALK_LIST(i, iface_list) + IFACE_LEGACY_ACCESS; + + WALK_LIST(i, global_iface_list) if (!strcmp(i->name, name)) return i; /* No active iface, create a dummy */ + IFACE_LOCK; i = mb_allocz(if_pool, sizeof(struct iface)); strncpy(i->name, name, sizeof(i->name)-1); i->flags = IF_SHUTDOWN; init_list(&i->addrs); init_list(&i->neighbors); - add_tail(&iface_list, &i->n); + add_tail(&global_iface_list, &i->n); + IFACE_UNLOCK; return i; } @@ -555,7 +600,9 @@ if_recalc_all_preferred_addresses(void) { struct iface *i; - WALK_LIST(i, iface_list) + IFACE_LEGACY_ACCESS; + + WALK_LIST(i, global_iface_list) { if_recalc_preferred(i); @@ -585,6 +632,8 @@ ifa_update(struct ifa *a) struct iface *i = a->iface; struct ifa *b; + IFACE_LEGACY_ACCESS; + WALK_LIST(b, i->addrs) if (ifa_same(b, a)) { @@ -603,10 +652,12 @@ ifa_update(struct ifa *a) if ((a->prefix.type == NET_IP4) && (i->flags & IF_BROADCAST) && ipa_zero(a->brd)) log(L_WARN "Missing broadcast address for interface %s", i->name); + IFACE_LOCK; b = mb_alloc(if_pool, sizeof(struct ifa)); memcpy(b, a, sizeof(struct ifa)); add_tail(&i->addrs, &b->n); b->flags |= IA_UPDATED; + IFACE_UNLOCK; i->flags |= IF_NEEDS_RECALC; if (i->flags & IF_UP) @@ -631,6 +682,17 @@ ifa_delete(struct ifa *a) WALK_LIST(b, i->addrs) if (ifa_same(b, a)) { + IFACE_LOCK; + ifa_delete_locked(b); + IFACE_UNLOCK; + return; + } +} + +static void +ifa_delete_locked(struct ifa *b) +{ + struct iface *i = b->iface; rem_node(&b->n); if (b->flags & IA_PRIMARY) @@ -653,7 +715,6 @@ ifa_delete(struct ifa *a) mb_free(b); return; - } } u32 @@ -662,8 +723,10 @@ if_choose_router_id(struct iface_patt *mask, u32 old_id) struct iface *i; struct ifa *a, *b; + IFACE_LEGACY_ACCESS; + b = NULL; - WALK_LIST(i, iface_list) + WALK_LIST(i, global_iface_list) { if (!(i->flags & IF_ADMIN_UP) || (i->flags & IF_SHUTDOWN)) @@ -709,8 +772,10 @@ if_choose_router_id(struct iface_patt *mask, u32 old_id) void if_init(void) { - if_pool = rp_new(&root_pool, "Interfaces"); - init_list(&iface_list); + iface_domain = DOMAIN_NEW(attrs, "Interfaces"); + if_pool = rp_new(&root_pool, &main_birdloop, "Interfaces"); + init_list(&global_iface_list); + strcpy(default_vrf.name, "default"); neigh_init(if_pool); } @@ -837,13 +902,15 @@ if_show(void) struct ifa *a; char *type; - WALK_LIST(i, iface_list) + IFACE_LEGACY_ACCESS; + + WALK_LIST(i, global_iface_list) { if (i->flags & IF_SHUTDOWN) continue; char mbuf[16 + sizeof(i->name)] = {}; - if (i->master) + if (i->master != &default_vrf) bsprintf(mbuf, " master=%s", i->master->name); else if (i->master_index) bsprintf(mbuf, " master=#%u", i->master_index); @@ -879,8 +946,10 @@ if_show_summary(void) { struct iface *i; + IFACE_LEGACY_ACCESS; + cli_msg(-2005, "%-10s %-6s %-18s %s", "Interface", "State", "IPv4 address", "IPv6 address"); - WALK_LIST(i, iface_list) + WALK_LIST(i, global_iface_list) { byte a4[IPA_MAX_TEXT_LENGTH + 17]; byte a6[IPA_MAX_TEXT_LENGTH + 17]; diff --git a/nest/iface.h b/nest/iface.h index 1189cdd4..87ad86cf 100644 --- a/nest/iface.h +++ b/nest/iface.h @@ -9,10 +9,20 @@ #ifndef _BIRD_IFACE_H_ #define _BIRD_IFACE_H_ +#include "lib/event.h" #include "lib/lists.h" #include "lib/ip.h" +#include "lib/locking.h" -extern list iface_list; +DEFINE_DOMAIN(attrs); +extern list global_iface_list; +extern DOMAIN(attrs) iface_domain; + +#define IFACE_LEGACY_ACCESS ASSERT_DIE(birdloop_inside(&main_birdloop)) + +#define IFACE_LOCK LOCK_DOMAIN(attrs, iface_domain) +#define IFACE_UNLOCK UNLOCK_DOMAIN(attrs, iface_domain) +#define ASSERT_IFACE_LOCKED ASSERT_DIE(DOMAIN_IS_LOCKED(attrs, iface_domain)) struct proto; struct pool; @@ -28,6 +38,8 @@ struct ifa { /* Interface address */ unsigned flags; /* Analogous to iface->flags */ }; +extern struct iface default_vrf; + struct iface { node n; char name[16]; @@ -129,6 +141,7 @@ typedef struct neighbor { struct ifa *ifa; /* Ifa on related iface */ struct iface *iface; /* Interface it's connected to */ struct iface *ifreq; /* Requested iface, NULL for any */ + struct event event; /* Notification event */ struct proto *proto; /* Protocol this belongs to */ void *data; /* Protocol-specific data */ uint aux; /* Protocol-specific data */ @@ -140,13 +153,14 @@ typedef struct neighbor { #define NEF_STICKY 1 #define NEF_ONLINK 2 #define NEF_IFACE 4 /* Entry for whole iface */ +#define NEF_NOTIFY_MAIN 0x100 /* Notify from main_birdloop context */ neighbor *neigh_find(struct proto *p, ip_addr a, struct iface *ifa, uint flags); void neigh_dump(neighbor *); void neigh_dump_all(void); -void neigh_prune(void); +void neigh_prune(struct proto *p); void neigh_if_up(struct iface *); void neigh_if_down(struct iface *); void neigh_if_link(struct iface *); diff --git a/nest/limit.h b/nest/limit.h new file mode 100644 index 00000000..5838ad3b --- /dev/null +++ b/nest/limit.h @@ -0,0 +1,49 @@ +/* + * BIRD Internet Routing Daemon -- Limits + * + * (c) 1998--2000 Martin Mares <mj@ucw.cz> + * (c) 2021 Maria Matejka <mq@jmq.cz> + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_LIMIT_H_ +#define _BIRD_LIMIT_H_ + +struct limit { + u32 max; + u32 count; + int (*action)(struct limit *, void *data); +}; + +static inline int limit_do_action(struct limit *l, void *data) +{ + return l->action ? l->action(l, data) : 1; +} + +static inline int limit_push(struct limit *l, void *data) +{ + if ((l->count >= l->max) && limit_do_action(l, data)) + return 1; + + l->count++; + return 0; +} + +static inline void limit_pop(struct limit *l) +{ + --l->count; +} + +static inline void limit_reset(struct limit *l) +{ + l->count = 0; +} + +static inline void limit_update(struct limit *l, void *data, u32 max) +{ + if (l->count > (l->max = max)) + limit_do_action(l, data); +} + +#endif diff --git a/nest/neighbor.c b/nest/neighbor.c index 1a31fb79..7b951366 100644 --- a/nest/neighbor.c +++ b/nest/neighbor.c @@ -59,6 +59,9 @@ static slab *neigh_slab; static list neigh_hash_table[NEIGH_HASH_SIZE], sticky_neigh_list; +static void neigh_do_notify(void *); +static void neigh_do_notify_main(void *); +static void neigh_free(neighbor *n); static inline uint neigh_hash(struct proto *p, ip_addr a, struct iface *i) @@ -142,7 +145,7 @@ if_connected(ip_addr a, struct iface *i, struct ifa **ap, uint flags) } static inline int -if_connected_any(ip_addr a, struct iface *vrf, uint vrf_set, struct iface **iface, struct ifa **addr, uint flags) +if_connected_any(ip_addr a, struct iface *vrf, struct iface **iface, struct ifa **addr, uint flags) { struct iface *i; struct ifa *b; @@ -152,8 +155,8 @@ if_connected_any(ip_addr a, struct iface *vrf, uint vrf_set, struct iface **ifac *addr = NULL; /* Prefer SCOPE_HOST or longer prefix */ - WALK_LIST(i, iface_list) - if ((!vrf_set || vrf == i->master) && ((s = if_connected(a, i, &b, flags)) >= 0)) + WALK_LIST(i, global_iface_list) + if ((!vrf || vrf == i->master) && ((s = if_connected(a, i, &b, flags)) >= 0)) if (scope_better(s, scope) || (scope_remote(s, scope) && ifa_better(b, *addr))) { *iface = i; @@ -216,28 +219,34 @@ neigh_find(struct proto *p, ip_addr a, struct iface *iface, uint flags) struct iface *ifreq = iface; struct ifa *addr = NULL; + IFACE_LOCK; WALK_LIST(n, neigh_hash_table[h]) /* Search the cache */ if ((n->proto == p) && ipa_equal(n->addr, a) && (n->ifreq == iface)) + { + IFACE_UNLOCK; return n; + } + +#define NOT_FOUND goto not_found if (flags & NEF_IFACE) { if (ipa_nonzero(a) || !iface) - return NULL; + NOT_FOUND; } else { class = ipa_classify(a); if (class < 0) /* Invalid address */ - return NULL; + NOT_FOUND; if (((class & IADDR_SCOPE_MASK) == SCOPE_HOST) || (((class & IADDR_SCOPE_MASK) == SCOPE_LINK) && !iface) || !(class & IADDR_HOST)) - return NULL; /* Bad scope or a somecast */ + NOT_FOUND; /* Bad scope or a somecast */ } if ((flags & NEF_ONLINK) && !iface) - return NULL; + NOT_FOUND; if (iface) { @@ -245,13 +254,13 @@ neigh_find(struct proto *p, ip_addr a, struct iface *iface, uint flags) iface = (scope < 0) ? NULL : iface; } else - scope = if_connected_any(a, p->vrf, p->vrf_set, &iface, &addr, flags); + scope = if_connected_any(a, p->vrf, &iface, &addr, flags); /* scope < 0 means i don't know neighbor */ /* scope >= 0 <=> iface != NULL */ if ((scope < 0) && !(flags & NEF_STICKY)) - return NULL; + NOT_FOUND; n = sl_allocz(neigh_slab); add_tail(&neigh_hash_table[h], &n->n); @@ -264,7 +273,36 @@ neigh_find(struct proto *p, ip_addr a, struct iface *iface, uint flags) n->flags = flags; n->scope = scope; + ASSERT_DIE(birdloop_inside(p->loop)); + + if (flags & NEF_NOTIFY_MAIN) + n->event = (event) { + .hook = neigh_do_notify_main, + .data = n, + .list = &global_event_list, + }; + else if (p->loop == &main_birdloop) + n->event = (event) { + .hook = neigh_do_notify, + .data = n, + .list = &global_event_list, + }; + else + { + birdloop_link(p->loop); + n->event = (event) { + .hook = neigh_do_notify, + .data = n, + .list = birdloop_event_list(p->loop), + }; + } + + IFACE_UNLOCK; return n; + +not_found: + IFACE_UNLOCK; + return NULL; } /** @@ -298,18 +336,46 @@ neigh_dump_all(void) neighbor *n; int i; + IFACE_LOCK; + debug("Known neighbors:\n"); for(i=0; i<NEIGH_HASH_SIZE; i++) WALK_LIST(n, neigh_hash_table[i]) neigh_dump(n); debug("\n"); + + IFACE_UNLOCK; } static inline void neigh_notify(neighbor *n) { - if (n->proto->neigh_notify && (n->proto->proto_state != PS_STOP)) + if (!n->proto->neigh_notify) + return; + + ev_send(n->event.list, &n->event); +} + +static void +neigh_do_notify_main(void *data) +{ + neighbor *n = data; + PROTO_LOCKED_FROM_MAIN(n->proto) + neigh_do_notify(data); +} + +static void +neigh_do_notify(void *data) +{ + neighbor *n = data; + + ASSERT_DIE(birdloop_inside(n->proto->loop)); + + if (n->proto->proto_state != PS_STOP) n->proto->neigh_notify(n); + + if ((n->scope < 0) && !(n->flags & NEF_STICKY)) + neigh_free(n); } static void @@ -340,11 +406,21 @@ neigh_down(neighbor *n) neigh_notify(n); } -static inline void +static void neigh_free(neighbor *n) { + ASSERT_DIE(birdloop_inside(n->proto->loop)); + + if (n->flags & NEF_NOTIFY_MAIN) + ASSERT_DIE(birdloop_inside(&main_birdloop)); + rem_node(&n->n); rem_node(&n->if_n); + + if (n->event.list != &global_event_list) + birdloop_unlink(n->proto->loop); + + ev_postpone(&n->event); sl_free(neigh_slab, n); } @@ -360,6 +436,8 @@ neigh_free(neighbor *n) void neigh_update(neighbor *n, struct iface *iface) { + ASSERT_IFACE_LOCKED; + struct proto *p = n->proto; struct ifa *ifa = NULL; int scope = -1; @@ -369,7 +447,7 @@ neigh_update(neighbor *n, struct iface *iface) return; /* VRF-bound neighbors ignore changes in other VRFs */ - if (p->vrf_set && (p->vrf != iface->master)) + if (p->vrf && (p->vrf != iface->master)) return; scope = if_connected(n->addr, iface, &ifa, n->flags); @@ -379,7 +457,7 @@ neigh_update(neighbor *n, struct iface *iface) { /* When neighbor is going down, try to respawn it on other ifaces */ if ((scope < 0) && (n->scope >= 0) && !n->ifreq && (n->flags & NEF_STICKY)) - scope = if_connected_any(n->addr, p->vrf, p->vrf_set, &iface, &ifa, n->flags); + scope = if_connected_any(n->addr, p->vrf, &iface, &ifa, n->flags); } else { @@ -406,12 +484,6 @@ neigh_update(neighbor *n, struct iface *iface) if (n->scope >= 0) neigh_down(n); - if ((n->scope < 0) && !(n->flags & NEF_STICKY)) - { - neigh_free(n); - return; - } - if (scope >= 0) neigh_up(n, iface, ifa, scope); } @@ -433,14 +505,18 @@ neigh_if_up(struct iface *i) neighbor *n; node *x, *y; + IFACE_LOCK; + /* Update neighbors that might be better off with the new iface */ - WALK_LIST(ii, iface_list) + WALK_LIST(ii, global_iface_list) if (!EMPTY_LIST(ii->neighbors) && (ii != i) && if_intersect(i, ii)) WALK_LIST2_DELSAFE(n, x, y, ii->neighbors, if_n) neigh_update(n, i); WALK_LIST2_DELSAFE(n, x, y, sticky_neigh_list, if_n) neigh_update(n, i); + + IFACE_UNLOCK; } /** @@ -457,8 +533,12 @@ neigh_if_down(struct iface *i) neighbor *n; node *x, *y; + IFACE_LOCK; + WALK_LIST2_DELSAFE(n, x, y, i->neighbors, if_n) neigh_update(n, i); + + IFACE_UNLOCK; } /** @@ -474,8 +554,12 @@ neigh_if_link(struct iface *i) neighbor *n; node *x, *y; + IFACE_LOCK; + WALK_LIST2_DELSAFE(n, x, y, i->neighbors, if_n) neigh_notify(n); + + IFACE_UNLOCK; } /** @@ -495,8 +579,10 @@ neigh_ifa_up(struct ifa *a) neighbor *n; node *x, *y; + IFACE_LOCK; + /* Update neighbors that might be better off with the new ifa */ - WALK_LIST(ii, iface_list) + WALK_LIST(ii, global_iface_list) if (!EMPTY_LIST(ii->neighbors) && ifa_intersect(a, ii)) WALK_LIST2_DELSAFE(n, x, y, ii->neighbors, if_n) neigh_update(n, i); @@ -504,6 +590,8 @@ neigh_ifa_up(struct ifa *a) /* Wake up all sticky neighbors that are reachable now */ WALK_LIST2_DELSAFE(n, x, y, sticky_neigh_list, if_n) neigh_update(n, i); + + IFACE_UNLOCK; } void @@ -513,10 +601,14 @@ neigh_ifa_down(struct ifa *a) neighbor *n; node *x, *y; + IFACE_LOCK; + /* Update all neighbors whose scope has changed */ WALK_LIST2_DELSAFE(n, x, y, i->neighbors, if_n) if (n->ifa == a) neigh_update(n, i); + + IFACE_UNLOCK; } static inline void @@ -536,16 +628,21 @@ neigh_prune_one(neighbor *n) * is shut down to get rid of all its heritage. */ void -neigh_prune(void) +neigh_prune(struct proto *p) { neighbor *n; node *m; int i; + IFACE_LOCK; + DBG("Pruning neighbors\n"); for(i=0; i<NEIGH_HASH_SIZE; i++) WALK_LIST_DELSAFE(n, m, neigh_hash_table[i]) - neigh_prune_one(n); + if (n->proto == p) + neigh_prune_one(n); + + IFACE_UNLOCK; } /** diff --git a/nest/proto-hooks.c b/nest/proto-hooks.c index bc88b4b4..716ce86c 100644 --- a/nest/proto-hooks.c +++ b/nest/proto-hooks.c @@ -76,16 +76,6 @@ void dump(struct proto *p) { DUMMY; } /** - * dump_attrs - dump protocol-dependent attributes - * @e: a route entry - * - * This hook dumps all attributes in the &rte which belong to this - * protocol to the debug output. - */ -void dump_attrs(rte *e) -{ DUMMY; } - -/** * start - request instance startup * @p: protocol instance * @@ -228,36 +218,6 @@ void neigh_notify(neighbor *neigh) { DUMMY; } /** - * make_tmp_attrs - convert embedded attributes to temporary ones - * @e: route entry - * @pool: linear pool to allocate attribute memory in - * - * This hook is called by the routing table functions if they need - * to convert the protocol attributes embedded directly in the &rte - * to temporary extended attributes in order to distribute them - * to other protocols or to filters. make_tmp_attrs() creates - * an &ea_list in the linear pool @pool, fills it with values of the - * temporary attributes and returns a pointer to it. - */ -ea_list *make_tmp_attrs(rte *e, struct linpool *pool) -{ DUMMY; } - -/** - * store_tmp_attrs - convert temporary attributes to embedded ones - * @e: route entry - * @attrs: temporary attributes to be converted - * - * This hook is an exact opposite of make_tmp_attrs() -- it takes - * a list of extended attributes and converts them to attributes - * embedded in the &rte corresponding to this protocol. - * - * You must be prepared for any of the attributes being missing - * from the list and use default values instead. - */ -void store_tmp_attrs(rte *e, ea_list *attrs) -{ DUMMY; } - -/** * preexport - pre-filtering decisions before route export * @p: protocol instance the route is going to be exported to * @e: the route in question diff --git a/nest/proto.c b/nest/proto.c index 31ee1fa1..2546e812 100644 --- a/nest/proto.c +++ b/nest/proto.c @@ -15,6 +15,7 @@ #include "lib/event.h" #include "lib/timer.h" #include "lib/string.h" +#include "lib/coro.h" #include "conf/conf.h" #include "nest/route.h" #include "nest/iface.h" @@ -31,7 +32,6 @@ struct protocol *class_to_protocol[PROTOCOL__MAX]; #define CD(c, msg, args...) ({ if (c->debug & D_STATES) log(L_TRACE "%s.%s: " msg, c->proto->name, c->name ?: "?", ## args); }) #define PD(p, msg, args...) ({ if (p->debug & D_STATES) log(L_TRACE "%s: " msg, p->name, ## args); }) -static timer *proto_shutdown_timer; static timer *gr_wait_timer; #define GRS_NONE 0 @@ -43,24 +43,34 @@ static int graceful_restart_state; static u32 graceful_restart_locks; static char *p_states[] = { "DOWN", "START", "UP", "STOP" }; -static char *c_states[] = { "DOWN", "START", "UP", "FLUSHING" }; -static char *e_states[] = { "DOWN", "FEEDING", "READY" }; +static char *c_states[] = { "DOWN", "START", "UP", "STOP", "RESTART" }; extern struct protocol proto_unix_iface; -static void channel_request_reload(struct channel *c); -static void proto_shutdown_loop(timer *); +static void channel_aux_request_refeed(struct channel_aux_table *cat); static void proto_rethink_goal(struct proto *p); static char *proto_state_name(struct proto *p); -static void channel_verify_limits(struct channel *c); -static inline void channel_reset_limit(struct channel_limit *l); - +static void channel_init_limit(struct channel *c, struct limit *l, int dir, struct channel_limit *cf); +static void channel_update_limit(struct channel *c, struct limit *l, int dir, struct channel_limit *cf); +static void channel_reset_limit(struct channel *c, struct limit *l, int dir); +static void channel_feed_end(struct channel *c); +static void channel_export_stopped(struct rt_export_request *req); static inline int proto_is_done(struct proto *p) -{ return (p->proto_state == PS_DOWN) && (p->active_channels == 0); } +{ return (p->proto_state == PS_DOWN) && proto_is_inactive(p); } + +static inline event_list *proto_event_list(struct proto *p) +{ return p->loop == &main_birdloop ? &global_event_list : birdloop_event_list(p->loop); } + +static inline event_list *proto_work_list(struct proto *p) +{ return p->loop == &main_birdloop ? &global_work_list : birdloop_event_list(p->loop); } + +static inline void proto_send_event(struct proto *p) +{ ev_send(proto_event_list(p), p->event); } + static inline int channel_is_active(struct channel *c) -{ return (c->channel_state == CS_START) || (c->channel_state == CS_UP); } +{ return (c->channel_state != CS_DOWN); } static inline int channel_reloadable(struct channel *c) { return c->proto->reload_routes && c->reloadable; } @@ -68,10 +78,48 @@ static inline int channel_reloadable(struct channel *c) static inline void channel_log_state_change(struct channel *c) { - if (c->export_state) - CD(c, "State changed to %s/%s", c_states[c->channel_state], e_states[c->export_state]); - else - CD(c, "State changed to %s", c_states[c->channel_state]); + CD(c, "State changed to %s", c_states[c->channel_state]); +} + +void +channel_import_log_state_change(struct rt_import_request *req, u8 state) +{ + struct channel *c = SKIP_BACK(struct channel, in_req, req); + CD(c, "Channel import state changed to %s", rt_import_state_name(state)); +} + +void +channel_export_log_state_change(struct rt_export_request *req, u8 state) +{ + struct channel *c = SKIP_BACK(struct channel, out_req, req); + CD(c, "Channel export state changed to %s", rt_export_state_name(state)); + + switch (state) + { + case TES_FEEDING: + if (c->out_table) + rt_refresh_begin(&c->out_table->push); + else if (c->proto->feed_begin) + c->proto->feed_begin(c, !c->refeeding); + break; + case TES_READY: + channel_feed_end(c); + break; + } +} + +static void +channel_dump_import_req(struct rt_import_request *req) +{ + struct channel *c = SKIP_BACK(struct channel, in_req, req); + debug(" Channel %s.%s import request %p\n", c->proto->name, c->name, req); +} + +static void +channel_dump_export_req(struct rt_export_request *req) +{ + struct channel *c = SKIP_BACK(struct channel, out_req, req); + debug(" Channel %s.%s export request %p\n", c->proto->name, c->name, req); } static void @@ -111,7 +159,7 @@ proto_cf_find_channel(struct proto_config *pc, uint net_type) * Returns pointer to channel or NULL */ struct channel * -proto_find_channel_by_table(struct proto *p, struct rtable *t) +proto_find_channel_by_table(struct proto *p, rtable *t) { struct channel *c; @@ -141,6 +189,16 @@ proto_find_channel_by_name(struct proto *p, const char *n) return NULL; } +rte * channel_preimport(struct rt_import_request *req, rte *new, rte *old); +rte * channel_in_preimport(struct rt_import_request *req, rte *new, rte *old); + +void rt_notify_optimal(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe); +void rt_notify_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe); +void rt_feed_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe, rte **feed, uint count); +void rt_notify_accepted(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe, rte **feed, uint count); +void rt_notify_merged(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe, rte **feed, uint count); + + /** * proto_add_channel - connect protocol to a routing table * @p: protocol instance @@ -166,11 +224,17 @@ proto_add_channel(struct proto *p, struct channel_config *cf) c->proto = p; c->table = cf->table->table; + RT_LOCKED(c->table, t) + rt_lock_table(t); + c->in_filter = cf->in_filter; c->out_filter = cf->out_filter; - c->rx_limit = cf->rx_limit; - c->in_limit = cf->in_limit; - c->out_limit = cf->out_limit; + + channel_init_limit(c, &c->rx_limit, PLD_RX, &cf->rx_limit); + channel_init_limit(c, &c->in_limit, PLD_IN, &cf->in_limit); + channel_init_limit(c, &c->out_limit, PLD_OUT, &cf->out_limit); + + c->rte_update_pool = lp_new_default(proto_pool); c->net_type = cf->net_type; c->ra_mode = cf->ra_mode; @@ -181,7 +245,6 @@ proto_add_channel(struct proto *p, struct channel_config *cf) c->rpki_reload = cf->rpki_reload; c->channel_state = CS_DOWN; - c->export_state = ES_DOWN; c->last_state_change = current_time(); c->reloadable = 1; @@ -203,6 +266,9 @@ proto_remove_channel(struct proto *p UNUSED, struct channel *c) CD(c, "Removed", c->name); + RT_LOCKED(c->table, t) + rt_unlock_table(t); + rem_node(&c->n); mb_free(c); } @@ -223,7 +289,7 @@ proto_pause_channels(struct proto *p) struct channel *c; WALK_LIST(c, p->channels) if (!c->disabled && channel_is_active(c)) - channel_set_state(c, CS_START); + channel_set_state(c, CS_PAUSE); } static void @@ -232,7 +298,7 @@ proto_stop_channels(struct proto *p) struct channel *c; WALK_LIST(c, p->channels) if (!c->disabled && channel_is_active(c)) - channel_set_state(c, CS_FLUSHING); + channel_set_state(c, CS_STOP); } static void @@ -244,96 +310,25 @@ proto_remove_channels(struct proto *p) } static void -channel_schedule_feed(struct channel *c, int initial) -{ - // DBG("%s: Scheduling meal\n", p->name); - ASSERT(c->channel_state == CS_UP); - - c->export_state = ES_FEEDING; - c->refeeding = !initial; - - ev_schedule_work(c->feed_event); -} - -static void -channel_feed_loop(void *ptr) -{ - struct channel *c = ptr; - - if (c->export_state != ES_FEEDING) - return; - - /* Start feeding */ - if (!c->feed_active) - { - if (c->proto->feed_begin) - c->proto->feed_begin(c, !c->refeeding); - - c->refeed_pending = 0; - } - - // DBG("Feeding protocol %s continued\n", p->name); - if (!rt_feed_channel(c)) - { - ev_schedule_work(c->feed_event); - return; - } - - /* Reset export limit if the feed ended with acceptable number of exported routes */ - struct channel_limit *l = &c->out_limit; - if (c->refeeding && - (l->state == PLS_BLOCKED) && - (c->refeed_count <= l->limit) && - (c->stats.exp_routes <= l->limit)) - { - log(L_INFO "Protocol %s resets route export limit (%u)", c->proto->name, l->limit); - channel_reset_limit(&c->out_limit); - - /* Continue in feed - it will process routing table again from beginning */ - c->refeed_count = 0; - ev_schedule_work(c->feed_event); - return; - } - - // DBG("Feeding protocol %s finished\n", p->name); - c->export_state = ES_READY; - channel_log_state_change(c); - - if (c->proto->feed_end) - c->proto->feed_end(c); - - /* Restart feeding */ - if (c->refeed_pending) - channel_request_feeding(c); -} - - -static void -channel_roa_in_changed(struct rt_subscription *s) +channel_roa_in_changed(void *_data) { - struct channel *c = s->data; - int active = c->reload_event && ev_active(c->reload_event); + struct channel *c = _data; - CD(c, "Reload triggered by RPKI change%s", active ? " - already active" : ""); + CD(c, "Reload triggered by RPKI change"); - if (!active) - channel_request_reload(c); - else - c->reload_pending = 1; + channel_request_reload(c); } static void -channel_roa_out_changed(struct rt_subscription *s) +channel_roa_out_changed(void *_data) { - struct channel *c = s->data; - int active = (c->export_state == ES_FEEDING); + struct channel *c = _data; + CD(c, "Feeding triggered by RPKI change"); - CD(c, "Feeding triggered by RPKI change%s", active ? " - already active" : ""); + c->refeed_pending = 1; - if (!active) - channel_request_feeding(c); - else - c->refeed_pending = 1; + if (c->out_req.hook) + rt_stop_export(&c->out_req, channel_export_stopped); } /* Temporary code, subscriptions should be changed to resources */ @@ -345,14 +340,14 @@ struct roa_subscription { static int channel_roa_is_subscribed(struct channel *c, rtable *tab, int dir) { - void (*hook)(struct rt_subscription *) = + void (*hook)(void *) = dir ? channel_roa_in_changed : channel_roa_out_changed; struct roa_subscription *s; node *n; WALK_LIST2(s, n, c->roa_subscriptions, roa_node) - if ((s->s.tab == tab) && (s->s.hook == hook)) + if ((s->s.tab == tab) && (s->s.event->hook == hook)) return 1; return 0; @@ -366,9 +361,9 @@ channel_roa_subscribe(struct channel *c, rtable *tab, int dir) return; struct roa_subscription *s = mb_allocz(c->proto->pool, sizeof(struct roa_subscription)); + s->s.event = ev_new_init(c->proto->pool, dir ? channel_roa_in_changed : channel_roa_out_changed, c); + s->s.event->list = proto_work_list(c->proto); - s->s.hook = dir ? channel_roa_in_changed : channel_roa_out_changed; - s->s.data = c; rt_subscribe(tab, &s->s); add_tail(&c->roa_subscriptions, &s->roa_node); @@ -379,6 +374,7 @@ channel_roa_unsubscribe(struct roa_subscription *s) { rt_unsubscribe(&s->s); rem_node(&s->roa_node); + rfree(s->s.event); mb_free(s); } @@ -386,7 +382,7 @@ static void channel_roa_subscribe_filter(struct channel *c, int dir) { const struct filter *f = dir ? c->in_filter : c->out_filter; - struct rtable *tab; + rtable *tab; int valid = 1, found = 0; if ((f == FILTER_ACCEPT) || (f == FILTER_REJECT)) @@ -445,119 +441,554 @@ channel_roa_unsubscribe_all(struct channel *c) } static void -channel_start_export(struct channel *c) +channel_start_import(struct channel *c) { + if (c->in_req.hook) + { + log(L_WARN "%s.%s: Attempted to start channel's already started import", c->proto->name, c->name); + return; + } + + int nlen = strlen(c->name) + strlen(c->proto->name) + 2; + char *rn = mb_allocz(c->proto->pool, nlen); + bsprintf(rn, "%s.%s", c->proto->name, c->name); + + c->in_req = (struct rt_import_request) { + .name = rn, + .list = proto_work_list(c->proto), + .trace_routes = c->debug | c->proto->debug, + .dump_req = channel_dump_import_req, + .log_state_change = channel_import_log_state_change, + .preimport = channel_preimport, + }; + ASSERT(c->channel_state == CS_UP); - ASSERT(c->export_state == ES_DOWN); - channel_schedule_feed(c, 1); /* Sets ES_FEEDING */ + channel_reset_limit(c, &c->rx_limit, PLD_RX); + channel_reset_limit(c, &c->in_limit, PLD_IN); + + memset(&c->import_stats, 0, sizeof(struct channel_import_stats)); + + DBG("%s.%s: Channel start import req=%p\n", c->proto->name, c->name, &c->in_req); + rt_request_import(c->table, &c->in_req); } static void -channel_stop_export(struct channel *c) +channel_start_export(struct channel *c) { - /* Need to abort feeding */ - if (c->export_state == ES_FEEDING) - rt_feed_channel_abort(c); + if (c->out_req.hook) + { + c->restart_export = 1; + log(L_WARN "%s.%s: Fast channel export restart", c->proto->name, c->name); + return; + } + + ASSERT(c->channel_state == CS_UP); + int nlen = strlen(c->name) + strlen(c->proto->name) + 2; + char *rn = mb_allocz(c->proto->pool, nlen); + bsprintf(rn, "%s.%s", c->proto->name, c->name); + + c->out_req = (struct rt_export_request) { + .name = rn, + .list = proto_work_list(c->proto), + .trace_routes = c->debug | c->proto->debug, + .dump_req = channel_dump_export_req, + .log_state_change = channel_export_log_state_change, + }; + + bmap_init(&c->export_map, c->proto->pool, 1024); + bmap_init(&c->export_reject_map, c->proto->pool, 1024); + + channel_reset_limit(c, &c->out_limit, PLD_OUT); - c->export_state = ES_DOWN; - c->stats.exp_routes = 0; - bmap_reset(&c->export_map, 1024); + memset(&c->export_stats, 0, sizeof(struct channel_export_stats)); + + switch (c->ra_mode) { + case RA_OPTIMAL: + c->out_req.export_one = rt_notify_optimal; + break; + case RA_ANY: + c->out_req.export_one = rt_notify_any; + c->out_req.export_bulk = rt_feed_any; + break; + case RA_ACCEPTED: + c->out_req.export_bulk = rt_notify_accepted; + break; + case RA_MERGED: + c->out_req.export_bulk = rt_notify_merged; + break; + default: + bug("Unknown route announcement mode"); + } + + DBG("%s.%s: Channel start export req=%p\n", c->proto->name, c->name, &c->out_req); + rt_request_export(c->table, &c->out_req); } +static void +channel_check_stopped(struct channel *c) +{ + switch (c->channel_state) + { + case CS_STOP: + if (c->out_req.hook || c->in_req.hook || c->out_table || c->in_table) + return; + + channel_set_state(c, CS_DOWN); + proto_send_event(c->proto); + + break; + case CS_PAUSE: + if (c->out_req.hook) + return; + + channel_set_state(c, CS_START); + break; + default: + bug("Stopped channel in a bad state: %d", c->channel_state); + } + + DBG("%s.%s: Channel requests/hooks stopped (in state %s)\n", c->proto->name, c->name, c_states[c->channel_state]); +} -/* Called by protocol for reload from in_table */ void -channel_schedule_reload(struct channel *c) +channel_import_stopped(void *_c) { - ASSERT(c->channel_state == CS_UP); + struct channel *c = _c; + + c->in_req.hook = NULL; - rt_reload_channel_abort(c); - ev_schedule_work(c->reload_event); + mb_free(c->in_req.name); + c->in_req.name = NULL; + + channel_check_stopped(c); } static void -channel_reload_loop(void *ptr) +channel_export_stopped(struct rt_export_request *req) { - struct channel *c = ptr; + struct channel *c = SKIP_BACK(struct channel, out_req, req); - /* Start reload */ - if (!c->reload_active) - c->reload_pending = 0; + /* The hook has already stopped */ + req->hook = NULL; - if (!rt_reload_channel(c)) + if (c->refeed_pending) { - ev_schedule_work(c->reload_event); + c->refeeding = 1; + c->refeed_pending = 0; + + bmap_reset(&c->export_map, 1024); + bmap_reset(&c->export_reject_map, 1024); + + rt_request_export(c->table, req); return; } - /* Restart reload */ - if (c->reload_pending) - channel_request_reload(c); + mb_free(c->out_req.name); + c->out_req.name = NULL; + + bmap_free(&c->export_map); + bmap_free(&c->export_reject_map); + + if (c->restart_export) + { + c->restart_export = 0; + channel_start_export(c); + } + else + channel_check_stopped(c); +} + +static void +channel_feed_end(struct channel *c) +{ + struct rt_export_request *req = &c->out_req; + + /* Reset export limit if the feed ended with acceptable number of exported routes */ + struct limit *l = &c->out_limit; + if (c->refeeding && + (c->limit_active & (1 << PLD_OUT)) && + (c->refeed_count <= l->max) && + (l->count <= l->max)) + { + log(L_INFO "Protocol %s resets route export limit (%u)", c->proto->name, l->max); + channel_reset_limit(c, &c->out_limit, PLD_OUT); + + c->refeed_pending = 1; + rt_stop_export(req, channel_export_stopped); + return; + } + + if (c->out_table) + rt_refresh_end(&c->out_table->push); + else if (c->proto->feed_end) + c->proto->feed_end(c); + + if (c->refeed_pending) + rt_stop_export(req, channel_export_stopped); +} + +#define CHANNEL_AUX_TABLE_DUMP_REQ(inout, imex, pgimex, pushget) static void \ + channel_##inout##_##pushget##_dump_req(struct rt_##pgimex##_request *req) { \ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, pushget, req); \ + debug(" Channel %s.%s " #imex " table " #pushget " request %p\n", cat->c->proto->name, cat->c->name, req); } + +CHANNEL_AUX_TABLE_DUMP_REQ(in, import, import, push) +CHANNEL_AUX_TABLE_DUMP_REQ(in, import, export, get) +CHANNEL_AUX_TABLE_DUMP_REQ(out, export, import, push) +CHANNEL_AUX_TABLE_DUMP_REQ(out, export, export, get) + +#undef CHANNEL_AUX_TABLE_DUMP_REQ + +static uint channel_aux_imex(struct channel_aux_table *cat) +{ + if (cat->c->in_table == cat) + return 0; + else if (cat->c->out_table == cat) + return 1; + else + bug("Channel aux table must be in_table or out_table"); } static void -channel_reset_import(struct channel *c) +channel_aux_stopped(void *data) { - /* Need to abort feeding */ - ev_postpone(c->reload_event); - rt_reload_channel_abort(c); + struct channel_aux_table *cat; + + RT_LOCKED((rtable *) data, t) + cat = t->config->owner; - rt_prune_sync(c->in_table, 1); + ASSERT_DIE(cat->push.hook == NULL); + ASSERT_DIE(cat->get.hook == NULL); + ASSERT_DIE(cat->stop_pending); + + struct channel *c = cat->c; + + if (channel_aux_imex(cat)) + c->out_table = NULL; + else + c->in_table = NULL; + + mb_free(cat); + channel_check_stopped(c); } static void -channel_reset_export(struct channel *c) +channel_aux_import_stopped(void *_cat) { - /* Just free the routes */ - rt_prune_sync(c->out_table, 1); + struct channel_aux_table *cat = _cat; + + cat->push.hook = NULL; + + if (!cat->get.hook) + RT_LOCKED(cat->tab, t) + { + t->delete = channel_aux_stopped; + rt_unlock_table(t); + } } -/* Called by protocol to activate in_table */ -void -channel_setup_in_table(struct channel *c) +static void +channel_aux_export_stopped(struct rt_export_request *req) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + req->hook = NULL; + + if (cat->refeed_pending && !cat->stop_pending) + { + cat->refeed_pending = 0; + rt_request_export(cat->tab, req); + + return; + } + + if (!cat->push.hook) + RT_LOCKED(cat->tab, t) + { + t->delete = channel_aux_stopped; + rt_unlock_table(t); + } +} + +static void +channel_aux_stop(struct channel_aux_table *cat) +{ + ASSERT_DIE(!cat->stop_pending); + + cat->stop_pending = 1; + + RT_LOCKED(cat->tab, t) + rt_lock_table(t); + + cat->push_stopped = (event) { + .hook = channel_aux_import_stopped, + .data = cat, + .list = proto_event_list(cat->c->proto), + }; + + rt_stop_import(&cat->push, &cat->push_stopped); + rt_stop_export(&cat->get, channel_aux_export_stopped); +} + +static void +channel_push_log_state_change(struct rt_import_request *req, u8 state) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, push, req); + const char *imex = channel_aux_imex(cat) ? "export" : "import"; + CD(cat->c, "Channel %s table import state changed to %s", imex, rt_import_state_name(state)); +} + +static void +channel_get_log_state_change(struct rt_export_request *req, u8 state) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + const char *imex = channel_aux_imex(cat) ? "export" : "import"; + CD(cat->c, "Channel %s table export state changed to %s", imex, rt_export_state_name(state)); + + switch (state) + { + case TES_FEEDING: + if (imex && cat->c->proto->feed_begin) + cat->c->proto->feed_begin(cat->c, !cat->c->refeeding); + else if (!imex) + rt_refresh_begin(&cat->c->in_req); + break; + + case TES_READY: + if (imex && cat->c->proto->feed_end) + cat->c->proto->feed_end(cat->c); + else if (!imex) + rt_refresh_end(&cat->c->in_req); + + if (cat->refeed_pending) + rt_stop_export(&cat->get, channel_aux_export_stopped); + + break; + } +} + +void rte_update_direct(struct channel *c, const net_addr *n, rte *new, struct rte_src *src); + +static int +channel_aux_export_one_any(struct rt_export_request *req, struct rt_pending_export *rpe, rte **new, rte **old) +{ + struct rte_src *src = rpe->new ? rpe->new->rte.src : rpe->old->rte.src; + *old = RTES_OR_NULL(rpe->old); + struct rte_storage *new_stored; + + while (rpe) + { + new_stored = rpe->new; + rpe_mark_seen(req->hook, rpe); + rpe = rpe_next(rpe, src); + } + + *new = RTES_CLONE(new_stored, *new); + + return (*new || *old) && (&new_stored->rte != *old); +} + +static int +channel_aux_export_one_best(struct rt_export_request *req, struct rt_pending_export *rpe, rte **new, rte **old) +{ + *old = RTES_OR_NULL(rpe->old_best); + struct rte_storage *new_stored; + + while (rpe) + { + new_stored = rpe->new_best; + rpe_mark_seen(req->hook, rpe); + rpe = rpe_next(rpe, NULL); + } + + *new = RTES_CLONE(new_stored, *new); + + return (*new || *old) && (&new_stored->rte != *old); +} + +static void +channel_in_export_one_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + + rte n0, *new = &n0, *old; + if (channel_aux_export_one_any(req, rpe, &new, &old)) + rte_update_direct(cat->c, net, new, old ? old->src : new->src); +} + +static void +channel_in_export_one_best(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + + rte n0, *new = &n0, *old; + if (channel_aux_export_one_best(req, rpe, &new, &old)) + rte_update_direct(cat->c, net, new, old ? old->src : new->src); +} + +static void +channel_in_export_bulk_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe UNUSED, rte **feed, uint count) { - struct rtable_config *cf = mb_allocz(c->proto->pool, sizeof(struct rtable_config)); + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + for (uint i=0; i<count; i++) + { + rte n0 = *feed[i]; + rte_update_direct(cat->c, net, &n0, n0.src); + } +} - cf->name = "import"; - cf->addr_type = c->net_type; - cf->internal = 1; +static void +channel_in_export_bulk_best(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe UNUSED, rte **feed, uint count) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + if (!count) + return; + + rte n0 = *feed[0]; + rte_update_direct(cat->c, net, &n0, n0.src); +} + +void do_rt_notify_direct(struct channel *c, const net_addr *net, rte *new, const rte *old); + +static void +channel_out_export_one_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + rte n0, *new = &n0, *old; + if (channel_aux_export_one_any(req, rpe, &new, &old)) + do_rt_notify_direct(cat->c, net, new, old); +} + +static void +channel_out_export_one_best(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + rte n0, *new = &n0, *old; + if (channel_aux_export_one_best(req, rpe, &new, &old)) + do_rt_notify_direct(cat->c, net, new, old); +} + +static void +channel_out_export_bulk(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe UNUSED, rte **feed, uint count) +{ + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, get, req); + if (cat->c->ra_mode != RA_ANY) + ASSERT_DIE(count <= 1); + + for (uint i=0; i<count; i++) + { + rte n0 = *feed[i]; + do_rt_notify_direct(cat->c, net, &n0, NULL); + } +} + +/* Called by protocol to activate in_table */ +void +channel_setup_in_table(struct channel *c, int best) +{ + int nlen = sizeof("import") + strlen(c->name) + strlen(c->proto->name) + 3; + + struct { + struct channel_aux_table cat; + struct rtable_config tab_cf; + char name[0]; + } *cat = mb_allocz(c->proto->pool, sizeof(*cat) + nlen); + + bsprintf(cat->name, "%s.%s.import", c->proto->name, c->name); + + cat->tab_cf.owner = cat; + cat->tab_cf.name = cat->name; + cat->tab_cf.addr_type = c->net_type; + cat->tab_cf.cork_limit = 4 * page_size / sizeof(struct rt_pending_export); + + c->in_table = &cat->cat; + c->in_table->push = (struct rt_import_request) { + .name = cat->name, + .list = proto_work_list(c->proto), + .trace_routes = c->debug | c->proto->debug, + .dump_req = channel_in_push_dump_req, + .log_state_change = channel_push_log_state_change, + .preimport = channel_in_preimport, + }; + c->in_table->get = (struct rt_export_request) { + .name = cat->name, + .list = proto_work_list(c->proto), + .trace_routes = c->debug | c->proto->debug, + .dump_req = channel_in_get_dump_req, + .log_state_change = channel_get_log_state_change, + .export_one = best ? channel_in_export_one_best : channel_in_export_one_any, + .export_bulk = best ? channel_in_export_bulk_best : channel_in_export_bulk_any, + }; - c->in_table = rt_setup(c->proto->pool, cf); + c->in_table->c = c; + c->in_table->tab = rt_setup(c->proto->pool, &cat->tab_cf); - c->reload_event = ev_new_init(c->proto->pool, channel_reload_loop, c); + rt_request_import(c->in_table->tab, &c->in_table->push); + rt_request_export(c->in_table->tab, &c->in_table->get); } /* Called by protocol to activate out_table */ void channel_setup_out_table(struct channel *c) { - struct rtable_config *cf = mb_allocz(c->proto->pool, sizeof(struct rtable_config)); - cf->name = "export"; - cf->addr_type = c->net_type; - cf->internal = 1; + int nlen = sizeof("export") + strlen(c->name) + strlen(c->proto->name) + 3; + + struct { + struct channel_aux_table cat; + struct rtable_config tab_cf; + char name[0]; + } *cat = mb_allocz(c->proto->pool, sizeof(*cat) + nlen); - c->out_table = rt_setup(c->proto->pool, cf); + bsprintf(cat->name, "%s.%s.export", c->proto->name, c->name); + + cat->tab_cf.owner = cat; + cat->tab_cf.name = cat->name; + cat->tab_cf.addr_type = c->net_type; + cat->tab_cf.cork_limit = 4 * page_size / sizeof(struct rt_pending_export); + + c->out_table = &cat->cat; + c->out_table->push = (struct rt_import_request) { + .name = cat->name, + .list = proto_work_list(c->proto), + .trace_routes = c->debug | c->proto->debug, + .dump_req = channel_out_push_dump_req, + .log_state_change = channel_push_log_state_change, + }; + c->out_table->get = (struct rt_export_request) { + .name = cat->name, + .list = proto_work_list(c->proto), + .trace_routes = c->debug | c->proto->debug, + .dump_req = channel_out_get_dump_req, + .log_state_change = channel_get_log_state_change, + .export_one = (c->ra_mode == RA_ANY) ? channel_out_export_one_any : channel_out_export_one_best, + .export_bulk = channel_out_export_bulk, + }; + + c->out_table->c = c; + c->out_table->tab = rt_setup(c->proto->pool, &cat->tab_cf); + + rt_request_import(c->out_table->tab, &c->out_table->push); + rt_request_export(c->out_table->tab, &c->out_table->get); } +static void +channel_aux_request_refeed(struct channel_aux_table *cat) +{ + if (cat->stop_pending) + return; + + cat->refeed_pending = 1; + rt_stop_export(&cat->get, channel_aux_export_stopped); +} static void channel_do_start(struct channel *c) { - rt_lock_table(c->table); - add_tail(&c->table->channels, &c->table_node); c->proto->active_channels++; - c->feed_event = ev_new_init(c->proto->pool, channel_feed_loop, c); - - bmap_init(&c->export_map, c->proto->pool, 1024); - memset(&c->stats, 0, sizeof(struct proto_stats)); - - channel_reset_limit(&c->rx_limit); - channel_reset_limit(&c->in_limit); - channel_reset_limit(&c->out_limit); - CALL(c->channel->start, c); + + channel_start_import(c); } static void @@ -572,9 +1003,38 @@ channel_do_up(struct channel *c) } static void -channel_do_flush(struct channel *c) +channel_do_pause(struct channel *c) +{ + /* Stop export */ + if (c->out_req.hook) + { + rt_stop_export(&c->out_req, channel_export_stopped); + c->refeeding = 0; + } + + channel_roa_unsubscribe_all(c); +} + +static void +channel_do_stop(struct channel *c) { - rt_schedule_prune(c->table); + /* Drop auxiliary tables */ + if (c->in_table) + channel_aux_stop(c->in_table); + + if (c->out_table) + channel_aux_stop(c->out_table); + + /* Stop import */ + if (c->in_req.hook) + { + c->in_stopped = (event) { + .hook = channel_import_stopped, + .data = c, + .list = proto_event_list(c->proto), + }; + rt_stop_import(&c->in_req, &c->in_stopped); + } c->gr_wait = 0; if (c->gr_lock) @@ -582,48 +1042,30 @@ channel_do_flush(struct channel *c) CALL(c->channel->shutdown, c); - /* This have to be done in here, as channel pool is freed before channel_do_down() */ - bmap_free(&c->export_map); - c->in_table = NULL; - c->reload_event = NULL; - c->out_table = NULL; - channel_roa_unsubscribe_all(c); } static void channel_do_down(struct channel *c) { - ASSERT(!c->feed_active && !c->reload_active); + ASSERT(!c->out_req.hook && !c->in_req.hook && !c->out_table && !c->in_table); - rem_node(&c->table_node); - rt_unlock_table(c->table); c->proto->active_channels--; - if ((c->stats.imp_routes + c->stats.filt_routes) != 0) - log(L_ERR "%s: Channel %s is down but still has some routes", c->proto->name, c->name); - - // bmap_free(&c->export_map); - memset(&c->stats, 0, sizeof(struct proto_stats)); - - c->in_table = NULL; - c->reload_event = NULL; - c->out_table = NULL; - - /* The in_table and out_table are going to be freed by freeing their resource pools. */ + memset(&c->import_stats, 0, sizeof(struct channel_import_stats)); + memset(&c->export_stats, 0, sizeof(struct channel_export_stats)); CALL(c->channel->cleanup, c); /* Schedule protocol shutddown */ if (proto_is_done(c->proto)) - ev_schedule(c->proto->event); + proto_send_event(c->proto); } void channel_set_state(struct channel *c, uint state) { uint cs = c->channel_state; - uint es = c->export_state; DBG("%s reporting channel %s state transition %s -> %s\n", c->proto->name, c->name, c_states[cs], c_states[state]); if (state == cs) @@ -635,24 +1077,15 @@ channel_set_state(struct channel *c, uint state) switch (state) { case CS_START: - ASSERT(cs == CS_DOWN || cs == CS_UP); + ASSERT(cs == CS_DOWN || cs == CS_PAUSE); if (cs == CS_DOWN) channel_do_start(c); - if (es != ES_DOWN) - channel_stop_export(c); - - if (c->in_table && (cs == CS_UP)) - channel_reset_import(c); - - if (c->out_table && (cs == CS_UP)) - channel_reset_export(c); - break; case CS_UP: - ASSERT(cs == CS_DOWN || cs == CS_START); + ASSERT(cs == CS_DOWN || cs == CS_START || cs == CS_PAUSE); if (cs == CS_DOWN) channel_do_start(c); @@ -663,23 +1096,24 @@ channel_set_state(struct channel *c, uint state) channel_do_up(c); break; - case CS_FLUSHING: - ASSERT(cs == CS_START || cs == CS_UP); + case CS_PAUSE: + ASSERT(cs == CS_UP); - if (es != ES_DOWN) - channel_stop_export(c); + if (cs == CS_UP) + channel_do_pause(c); + break; - if (c->in_table && (cs == CS_UP)) - channel_reset_import(c); + case CS_STOP: + ASSERT(cs == CS_UP || cs == CS_START || cs == CS_PAUSE); - if (c->out_table && (cs == CS_UP)) - channel_reset_export(c); + if (cs == CS_UP) + channel_do_pause(c); - channel_do_flush(c); + channel_do_stop(c); break; case CS_DOWN: - ASSERT(cs == CS_FLUSHING); + ASSERT(cs == CS_STOP); channel_do_down(c); break; @@ -701,50 +1135,62 @@ channel_set_state(struct channel *c, uint state) * completed, it will switch back to ES_READY. This function can be called * even when feeding is already running, in that case it is restarted. */ -void -channel_request_feeding(struct channel *c) +static void +channel_request_table_feeding(struct channel *c) { - ASSERT(c->channel_state == CS_UP); + ASSERT(c->out_req.hook); - CD(c, "Feeding requested"); + c->refeed_pending = 1; + rt_stop_export(&c->out_req, channel_export_stopped); +} - /* Do nothing if we are still waiting for feeding */ - if (c->export_state == ES_DOWN) +void +channel_request_feeding(struct channel *c) +{ + if (c->gr_wait || !c->proto->rt_notify) return; - /* If we are already feeding, we want to restart it */ - if (c->export_state == ES_FEEDING) - { - /* Unless feeding is in initial state */ - if (!c->feed_active) - return; - - rt_feed_channel_abort(c); - } + CD(c, "Refeed requested"); - /* Track number of exported routes during refeed */ - c->refeed_count = 0; + ASSERT_DIE(c->out_req.hook); - channel_schedule_feed(c, 0); /* Sets ES_FEEDING */ - channel_log_state_change(c); + if (c->out_table) + channel_aux_request_refeed(c->out_table); + else + channel_request_table_feeding(c); } -static void +void channel_request_reload(struct channel *c) { - ASSERT(c->channel_state == CS_UP); + ASSERT(c->in_req.hook); ASSERT(channel_reloadable(c)); CD(c, "Reload requested"); - c->proto->reload_routes(c); + if (c->in_table) + channel_aux_request_refeed(c->in_table); + else + c->proto->reload_routes(c); +} - /* - * Should this be done before reload_routes() hook? - * Perhaps, but routes are updated asynchronously. - */ - channel_reset_limit(&c->rx_limit); - channel_reset_limit(&c->in_limit); +void +channel_refresh_begin(struct channel *c) +{ + CD(c, "Channel route refresh begin"); + if (c->in_table) + rt_refresh_begin(&c->in_table->push); + else + rt_refresh_begin(&c->in_req); +} + +void +channel_refresh_end(struct channel *c) +{ + if (c->in_table) + rt_refresh_end(&c->in_table->push); + else + rt_refresh_end(&c->in_req); } const struct channel_class channel_basic = { @@ -847,19 +1293,19 @@ channel_reconfigure(struct channel *c, struct channel_config *cf) /* Reconfigure channel fields */ c->in_filter = cf->in_filter; c->out_filter = cf->out_filter; - c->rx_limit = cf->rx_limit; - c->in_limit = cf->in_limit; - c->out_limit = cf->out_limit; + + channel_update_limit(c, &c->rx_limit, PLD_RX, &cf->rx_limit); + channel_update_limit(c, &c->in_limit, PLD_IN, &cf->in_limit); + channel_update_limit(c, &c->out_limit, PLD_OUT, &cf->out_limit); // c->ra_mode = cf->ra_mode; c->merge_limit = cf->merge_limit; c->preference = cf->preference; c->debug = cf->debug; + c->in_req.trace_routes = c->out_req.trace_routes = c->debug | c->proto->debug; c->in_keep_filtered = cf->in_keep_filtered; c->rpki_reload = cf->rpki_reload; - channel_verify_limits(c); - /* Execute channel-specific reconfigure hook */ if (c->channel->reconfigure && !c->channel->reconfigure(c, cf, &import_changed, &export_changed)) return 0; @@ -902,7 +1348,7 @@ channel_reconfigure(struct channel *c, struct channel_config *cf) channel_request_reload(c); if (export_changed) - channel_request_feeding(c); + channel_request_table_feeding(c); done: CD(c, "Reconfigured"); @@ -950,34 +1396,50 @@ proto_configure_channel(struct proto *p, struct channel **pc, struct channel_con return 1; } +static void +proto_cleanup(struct proto *p) +{ + p->active = 0; + proto_log_state_change(p); + proto_rethink_goal(p); +} static void -proto_event(void *ptr) +proto_loop_stopped(void *ptr) { struct proto *p = ptr; - if (p->do_start) - { - if_feed_baby(p); - p->do_start = 0; - } + ASSERT_DIE(birdloop_inside(&main_birdloop)); + + p->loop = &main_birdloop; + p->pool = NULL; + p->event->list = NULL; + + proto_cleanup(p); +} + +static void +proto_event(void *ptr) +{ + struct proto *p = ptr; if (p->do_stop) { if (p->proto == &proto_unix_iface) if_flush_ifaces(p); + p->do_stop = 0; } if (proto_is_done(p)) - { - if (p->proto->cleanup) - p->proto->cleanup(p); - - p->active = 0; - proto_log_state_change(p); - proto_rethink_goal(p); - } + if (p->loop != &main_birdloop) + birdloop_stop_self(p->loop, proto_loop_stopped, p); + else + { + rp_free(p->pool, proto_pool); + p->pool = NULL; + proto_cleanup(p); + } } @@ -1018,10 +1480,10 @@ proto_init(struct proto_config *c, node *n) struct protocol *pr = c->protocol; struct proto *p = pr->init(c); + p->loop = &main_birdloop; p->proto_state = PS_DOWN; p->last_state_change = current_time(); p->vrf = c->vrf; - p->vrf_set = c->vrf_set; insert_node(&p->n, n); p->event = ev_new_init(proto_pool, proto_event, p); @@ -1034,11 +1496,30 @@ proto_init(struct proto_config *c, node *n) static void proto_start(struct proto *p) { - /* Here we cannot use p->cf->name since it won't survive reconfiguration */ - p->pool = rp_new(proto_pool, p->proto->name); + DBG("Kicking %s up\n", p->name); + PD(p, "Starting"); + + int ns = strlen("Protocol ") + strlen(p->cf->name) + 1; + void *nb = mb_alloc(proto_pool, ns); + ASSERT_DIE(ns - 1 == bsnprintf(nb, ns, "Protocol %s", p->cf->name)); if (graceful_restart_state == GRS_INIT) p->gr_recovery = 1; + + if (p->cf->loop_order == DOMAIN_ORDER(the_bird)) + p->pool = rp_new(proto_pool, &main_birdloop, nb); + else + { + p->loop = birdloop_new(proto_pool, p->cf->loop_order, nb); + p->pool = birdloop_pool(p->loop); + } + + p->event->list = proto_event_list(p); + + mb_move(nb, p->pool); + + PROTO_LOCKED_FROM_MAIN(p) + proto_notify_state(p, (p->proto->start ? p->proto->start(p) : PS_UP)); } @@ -1074,6 +1555,7 @@ proto_config_new(struct protocol *pr, int class) cf->class = class; cf->debug = new_config->proto_default_debug; cf->mrtdump = new_config->proto_default_mrtdump; + cf->loop_order = DOMAIN_ORDER(the_bird); init_list(&cf->channels); @@ -1189,8 +1671,7 @@ proto_reconfigure(struct proto *p, struct proto_config *oc, struct proto_config if ((nc->protocol != oc->protocol) || (nc->net_type != oc->net_type) || (nc->disabled != p->disabled) || - (nc->vrf != oc->vrf) || - (nc->vrf_set != oc->vrf_set)) + (nc->vrf != oc->vrf)) return 0; p->name = nc->name; @@ -1279,8 +1760,14 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty nc->proto = p; /* We will try to reconfigure protocol p */ - if (! force_reconfig && proto_reconfigure(p, oc, nc, type)) - continue; + if (!force_reconfig) + { + int ok; + PROTO_LOCKED_FROM_MAIN(p) + ok = proto_reconfigure(p, oc, nc, type); + if (ok) + continue; + } if (nc->parent) { @@ -1363,11 +1850,20 @@ protos_commit(struct config *new, struct config *old, int force_reconfig, int ty } static void -proto_rethink_goal(struct proto *p) +proto_shutdown(struct proto *p) { - struct protocol *q; - byte goal; + if (p->proto_state == PS_START || p->proto_state == PS_UP) + { + /* Going down */ + DBG("Kicking %s down\n", p->name); + PD(p, "Shutting down"); + proto_notify_state(p, (p->proto->shutdown ? p->proto->shutdown(p) : PS_DOWN)); + } +} +static void +proto_rethink_goal(struct proto *p) +{ if (p->reconfiguring && !p->active) { struct proto_config *nc = p->cf_new; @@ -1387,32 +1883,12 @@ proto_rethink_goal(struct proto *p) /* Determine what state we want to reach */ if (p->disabled || p->reconfiguring) - goal = PS_DOWN; - else - goal = PS_UP; - - q = p->proto; - if (goal == PS_UP) - { - if (!p->active) - { - /* Going up */ - DBG("Kicking %s up\n", p->name); - PD(p, "Starting"); - proto_start(p); - proto_notify_state(p, (q->start ? q->start(p) : PS_UP)); - } - } - else { - if (p->proto_state == PS_START || p->proto_state == PS_UP) - { - /* Going down */ - DBG("Kicking %s down\n", p->name); - PD(p, "Shutting down"); - proto_notify_state(p, (q->shutdown ? q->shutdown(p) : PS_DOWN)); - } + PROTO_LOCKED_FROM_MAIN(p) + proto_shutdown(p); } + else if (!p->active) + proto_start(p); } struct proto * @@ -1524,7 +2000,7 @@ graceful_restart_done(timer *t UNUSED) WALK_LIST(c, p->channels) { /* Resume postponed export of routes */ - if ((c->channel_state == CS_UP) && c->gr_wait && c->proto->rt_notify) + if ((c->channel_state == CS_UP) && c->gr_wait && p->rt_notify) channel_start_export(c); /* Cleanup */ @@ -1614,7 +2090,11 @@ protos_dump_all(void) struct proto *p; WALK_LIST(p, proto_list) { - debug(" protocol %s state %s\n", p->name, p_states[p->proto_state]); +#define DPF(x) (p->x ? " " #x : "") + debug(" protocol %s (%p) state %s with %d active channels flags: %s%s%s%s\n", + p->name, p, p_states[p->proto_state], p->active_channels, + DPF(disabled), DPF(active), DPF(do_stop), DPF(reconfiguring)); +#undef DPF struct channel *c; WALK_LIST(c, p->channels) @@ -1624,6 +2104,23 @@ protos_dump_all(void) debug("\tInput filter: %s\n", filter_name(c->in_filter)); if (c->out_filter) debug("\tOutput filter: %s\n", filter_name(c->out_filter)); + debug("\tChannel state: %s/%s/%s\n", c_states[c->channel_state], + c->in_req.hook ? rt_import_state_name(rt_import_get_state(c->in_req.hook)) : "-", + c->out_req.hook ? rt_export_state_name(rt_export_get_state(c->out_req.hook)) : "-"); + if (c->in_table) + { + debug("\tInput aux table:\n"); + rt_dump_hooks(c->in_table->tab); + rt_dump(c->in_table->tab); + debug("\tEnd of input aux table.\n"); + } + if (c->out_table) + { + debug("\tOutput aux table:\n"); + rt_dump_hooks(c->in_table->tab); + rt_dump(c->in_table->tab); + debug("\tEnd of output aux table.\n"); + } } if (p->proto->dump && (p->proto_state != PS_DOWN)) @@ -1702,9 +2199,7 @@ protos_build(void) proto_build(&proto_perf); #endif - proto_pool = rp_new(&root_pool, "Protocols"); - proto_shutdown_timer = tm_new(proto_pool); - proto_shutdown_timer->hook = proto_shutdown_loop; + proto_pool = rp_new(&root_pool, &main_birdloop, "Protocols"); } @@ -1712,7 +2207,7 @@ protos_build(void) int proto_restart; static void -proto_shutdown_loop(timer *t UNUSED) +proto_shutdown_loop(void *data UNUSED) { struct proto *p, *p_next; @@ -1731,6 +2226,11 @@ proto_shutdown_loop(timer *t UNUSED) } } +static event proto_schedule_down_event = { + .hook = proto_shutdown_loop, + .list = &global_event_list, +}; + static inline void proto_schedule_down(struct proto *p, byte restart, byte code) { @@ -1743,7 +2243,8 @@ proto_schedule_down(struct proto *p, byte restart, byte code) p->down_sched = restart ? PDS_RESTART : PDS_DISABLE; p->down_code = code; - tm_start_max(proto_shutdown_timer, restart ? 250 MS : 0); + + ev_send_self(&proto_schedule_down_event); } /** @@ -1780,108 +2281,136 @@ proto_set_message(struct proto *p, char *msg, int len) } -static const char * -channel_limit_name(struct channel_limit *l) -{ - const char *actions[] = { - [PLA_WARN] = "warn", - [PLA_BLOCK] = "block", - [PLA_RESTART] = "restart", - [PLA_DISABLE] = "disable", - }; +static const char * channel_limit_name[] = { + [PLA_WARN] = "warn", + [PLA_BLOCK] = "block", + [PLA_RESTART] = "restart", + [PLA_DISABLE] = "disable", +}; - return actions[l->action]; -} -/** - * channel_notify_limit: notify about limit hit and take appropriate action - * @c: channel - * @l: limit being hit - * @dir: limit direction (PLD_*) - * @rt_count: the number of routes - * - * The function is called by the route processing core when limit @l - * is breached. It activates the limit and tooks appropriate action - * according to @l->action. - */ -void -channel_notify_limit(struct channel *c, struct channel_limit *l, int dir, u32 rt_count) +static void +channel_log_limit(struct channel *c, struct limit *l, int dir) { const char *dir_name[PLD_MAX] = { "receive", "import" , "export" }; - const byte dir_down[PLD_MAX] = { PDC_RX_LIMIT_HIT, PDC_IN_LIMIT_HIT, PDC_OUT_LIMIT_HIT }; - struct proto *p = c->proto; + log(L_WARN "Channel %s.%s hits route %s limit (%d), action: %s", + c->proto->name, c->name, dir_name[dir], l->max, channel_limit_name[c->limit_actions[dir]]); +} - if (l->state == PLS_BLOCKED) +static void +channel_activate_limit(struct channel *c, struct limit *l, int dir) +{ + if (c->limit_active & (1 << dir)) return; - /* For warning action, we want the log message every time we hit the limit */ - if (!l->state || ((l->action == PLA_WARN) && (rt_count == l->limit))) - log(L_WARN "Protocol %s hits route %s limit (%d), action: %s", - p->name, dir_name[dir], l->limit, channel_limit_name(l)); + c->limit_active |= (1 << dir); + channel_log_limit(c, l, dir); +} + +static int +channel_limit_warn(struct limit *l, void *data) +{ + struct channel_limit_data *cld = data; + struct channel *c = cld->c; + int dir = cld->dir; - switch (l->action) - { - case PLA_WARN: - l->state = PLS_ACTIVE; - break; + channel_log_limit(c, l, dir); - case PLA_BLOCK: - l->state = PLS_BLOCKED; - break; + return 0; +} - case PLA_RESTART: - case PLA_DISABLE: - l->state = PLS_BLOCKED; - if (p->proto_state == PS_UP) - proto_schedule_down(p, l->action == PLA_RESTART, dir_down[dir]); - break; - } +static int +channel_limit_block(struct limit *l, void *data) +{ + struct channel_limit_data *cld = data; + struct channel *c = cld->c; + int dir = cld->dir; + + channel_activate_limit(c, l, dir); + + return 1; } -static void -channel_verify_limits(struct channel *c) +static const byte chl_dir_down[PLD_MAX] = { PDC_RX_LIMIT_HIT, PDC_IN_LIMIT_HIT, PDC_OUT_LIMIT_HIT }; + +static int +channel_limit_down(struct limit *l, void *data) { - struct channel_limit *l; - u32 all_routes = c->stats.imp_routes + c->stats.filt_routes; + struct channel_limit_data *cld = data; + struct channel *c = cld->c; + struct proto *p = c->proto; + int dir = cld->dir; - l = &c->rx_limit; - if (l->action && (all_routes > l->limit)) - channel_notify_limit(c, l, PLD_RX, all_routes); + channel_activate_limit(c, l, dir); - l = &c->in_limit; - if (l->action && (c->stats.imp_routes > l->limit)) - channel_notify_limit(c, l, PLD_IN, c->stats.imp_routes); + if (p->proto_state == PS_UP) + proto_schedule_down(p, c->limit_actions[dir] == PLA_RESTART, chl_dir_down[dir]); - l = &c->out_limit; - if (l->action && (c->stats.exp_routes > l->limit)) - channel_notify_limit(c, l, PLD_OUT, c->stats.exp_routes); + return 1; } -static inline void -channel_reset_limit(struct channel_limit *l) +static int (*channel_limit_action[])(struct limit *, void *) = { + [PLA_NONE] = NULL, + [PLA_WARN] = channel_limit_warn, + [PLA_BLOCK] = channel_limit_block, + [PLA_RESTART] = channel_limit_down, + [PLA_DISABLE] = channel_limit_down, +}; + +static void +channel_update_limit(struct channel *c, struct limit *l, int dir, struct channel_limit *cf) +{ + l->action = channel_limit_action[cf->action]; + c->limit_actions[dir] = cf->action; + + struct channel_limit_data cld = { .c = c, .dir = dir }; + limit_update(l, &cld, cf->action ? cf->limit : ~((u32) 0)); +} + +static void +channel_init_limit(struct channel *c, struct limit *l, int dir, struct channel_limit *cf) { - if (l->action) - l->state = PLS_INITIAL; + channel_reset_limit(c, l, dir); + channel_update_limit(c, l, dir, cf); } +static void +channel_reset_limit(struct channel *c, struct limit *l, int dir) +{ + limit_reset(l); + c->limit_active &= ~(1 << dir); +} + +static struct rte_owner_class default_rte_owner_class; + static inline void proto_do_start(struct proto *p) { + ASSERT_DIE(birdloop_inside(p->loop)); + p->active = 1; - p->do_start = 1; - ev_schedule(p->event); + + rt_init_sources(&p->sources, p->name, proto_work_list(p)); + if (!p->sources.class) + p->sources.class = &default_rte_owner_class; + + if (!p->cf->late_if_feed) + if_feed_baby(p); } static void proto_do_up(struct proto *p) { + ASSERT_DIE(birdloop_inside(p->loop)); + if (!p->main_source) - { p->main_source = rt_get_source(p, 0); - rt_lock_source(p->main_source); - } + // Locked automaticaly proto_start_channels(p); + + if (p->cf->late_if_feed) + if_feed_baby(p); } static inline void @@ -1896,9 +2425,6 @@ proto_do_stop(struct proto *p) p->down_sched = 0; p->gr_recovery = 0; - p->do_stop = 1; - ev_schedule(p->event); - if (p->main_source) { rt_unlock_source(p->main_source); @@ -1906,19 +2432,21 @@ proto_do_stop(struct proto *p) } proto_stop_channels(p); + rt_destroy_sources(&p->sources, p->event); + + p->do_stop = 1; + proto_send_event(p); } static void proto_do_down(struct proto *p) { p->down_code = 0; - neigh_prune(); - rfree(p->pool); - p->pool = NULL; + neigh_prune(p); /* Shutdown is finished in the protocol event */ if (proto_is_done(p)) - ev_schedule(p->event); + proto_send_event(p); } @@ -2009,38 +2537,58 @@ proto_state_name(struct proto *p) static void channel_show_stats(struct channel *c) { - struct proto_stats *s = &c->stats; + struct channel_import_stats *ch_is = &c->import_stats; + struct channel_export_stats *ch_es = &c->export_stats; + struct rt_import_stats *rt_is = c->in_req.hook ? &c->in_req.hook->stats : NULL; + struct rt_export_stats *rt_es = c->out_req.hook ? &c->out_req.hook->stats : NULL; + +#define SON(ie, item) ((ie) ? (ie)->item : 0) +#define SCI(item) SON(ch_is, item) +#define SCE(item) SON(ch_es, item) +#define SRI(item) SON(rt_is, item) +#define SRE(item) SON(rt_es, item) + + u32 rx_routes = c->rx_limit.count; + u32 in_routes = c->in_limit.count; + u32 out_routes = c->out_limit.count; if (c->in_keep_filtered) cli_msg(-1006, " Routes: %u imported, %u filtered, %u exported, %u preferred", - s->imp_routes, s->filt_routes, s->exp_routes, s->pref_routes); + in_routes, (rx_routes - in_routes), out_routes, SRI(pref)); else cli_msg(-1006, " Routes: %u imported, %u exported, %u preferred", - s->imp_routes, s->exp_routes, s->pref_routes); - - cli_msg(-1006, " Route change stats: received rejected filtered ignored accepted"); - cli_msg(-1006, " Import updates: %10u %10u %10u %10u %10u", - s->imp_updates_received, s->imp_updates_invalid, - s->imp_updates_filtered, s->imp_updates_ignored, - s->imp_updates_accepted); - cli_msg(-1006, " Import withdraws: %10u %10u --- %10u %10u", - s->imp_withdraws_received, s->imp_withdraws_invalid, - s->imp_withdraws_ignored, s->imp_withdraws_accepted); - cli_msg(-1006, " Export updates: %10u %10u %10u --- %10u", - s->exp_updates_received, s->exp_updates_rejected, - s->exp_updates_filtered, s->exp_updates_accepted); - cli_msg(-1006, " Export withdraws: %10u --- --- --- %10u", - s->exp_withdraws_received, s->exp_withdraws_accepted); + in_routes, out_routes, SRI(pref)); + + cli_msg(-1006, " Route change stats: received rejected filtered ignored limited accepted"); + cli_msg(-1006, " Import updates: %10u %10u %10u %10u %10u %10u", + SCI(updates_received), SCI(updates_invalid), + SCI(updates_filtered), SRI(updates_ignored), + SCI(updates_limited_rx) + SCI(updates_limited_in), + SRI(updates_accepted)); + cli_msg(-1006, " Import withdraws: %10u %10u --- %10u --- %10u", + SCI(withdraws_received), SCI(withdraws_invalid), + SRI(withdraws_ignored), SRI(withdraws_accepted)); + cli_msg(-1006, " Export updates: %10u %10u %10u --- %10u %10u", + SRE(updates_received), SCE(updates_rejected), + SCE(updates_filtered), SCE(updates_limited), SCE(updates_accepted)); + cli_msg(-1006, " Export withdraws: %10u --- --- --- ---%10u", + SRE(withdraws_received), SCE(withdraws_accepted)); + +#undef SRI +#undef SRE +#undef SCI +#undef SCE +#undef SON } void -channel_show_limit(struct channel_limit *l, const char *dsc) +channel_show_limit(struct limit *l, const char *dsc, int active, int action) { if (!l->action) return; - cli_msg(-1006, " %-16s%d%s", dsc, l->limit, l->state ? " [HIT]" : ""); - cli_msg(-1006, " Action: %s", channel_limit_name(l)); + cli_msg(-1006, " %-16s%d%s", dsc, l->max, active ? " [HIT]" : ""); + cli_msg(-1006, " Action: %s", channel_limit_name[action]); } void @@ -2048,6 +2596,8 @@ channel_show_info(struct channel *c) { cli_msg(-1006, " Channel %s", c->name); cli_msg(-1006, " State: %s", c_states[c->channel_state]); + cli_msg(-1006, " Import state: %s", rt_import_state_name(rt_import_get_state(c->in_req.hook))); + cli_msg(-1006, " Export state: %s", rt_export_state_name(rt_export_get_state(c->out_req.hook))); cli_msg(-1006, " Table: %s", c->table->name); cli_msg(-1006, " Preference: %d", c->preference); cli_msg(-1006, " Input filter: %s", filter_name(c->in_filter)); @@ -2058,9 +2608,9 @@ channel_show_info(struct channel *c) c->gr_lock ? " pending" : "", c->gr_wait ? " waiting" : ""); - channel_show_limit(&c->rx_limit, "Receive limit:"); - channel_show_limit(&c->in_limit, "Import limit:"); - channel_show_limit(&c->out_limit, "Export limit:"); + channel_show_limit(&c->rx_limit, "Receive limit:", c->limit_active & (1 << PLD_RX), c->limit_actions[PLD_RX]); + channel_show_limit(&c->in_limit, "Import limit:", c->limit_active & (1 << PLD_IN), c->limit_actions[PLD_IN]); + channel_show_limit(&c->out_limit, "Export limit:", c->limit_active & (1 << PLD_OUT), c->limit_actions[PLD_OUT]); if (c->channel_state != CS_DOWN) channel_show_stats(c); @@ -2106,8 +2656,8 @@ proto_cmd_show(struct proto *p, uintptr_t verbose, int cnt) cli_msg(-1006, " Message: %s", p->message); if (p->cf->router_id) cli_msg(-1006, " Router ID: %R", p->cf->router_id); - if (p->vrf_set) - cli_msg(-1006, " VRF: %s", p->vrf ? p->vrf->name : "default"); + if (p->vrf) + cli_msg(-1006, " VRF: %s", p->vrf->name); if (p->proto->show_proto_info) p->proto->show_proto_info(p); @@ -2135,7 +2685,7 @@ proto_cmd_disable(struct proto *p, uintptr_t arg, int cnt UNUSED) p->disabled = 1; p->down_code = PDC_CMD_DISABLE; proto_set_message(p, (char *) arg, -1); - proto_rethink_goal(p); + proto_shutdown(p); cli_msg(-9, "%s: disabled", p->name); } @@ -2168,9 +2718,9 @@ proto_cmd_restart(struct proto *p, uintptr_t arg, int cnt UNUSED) p->disabled = 1; p->down_code = PDC_CMD_RESTART; proto_set_message(p, (char *) arg, -1); - proto_rethink_goal(p); + proto_shutdown(p); p->disabled = 0; - proto_rethink_goal(p); + /* After the protocol shuts down, proto_rethink_goal() is run from proto_event. */ cli_msg(-12, "%s: restarted", p->name); } @@ -2243,8 +2793,15 @@ proto_apply_cmd_symbol(const struct symbol *s, void (* cmd)(struct proto *, uint return; } - cmd(s->proto->proto, arg, 0); - cli_msg(0, ""); + if (s->proto->proto) + { + struct proto *p = s->proto->proto; + PROTO_LOCKED_FROM_MAIN(p) + cmd(p, arg, 0); + cli_msg(0, ""); + } + else + cli_msg(9002, "%s does not exist", s->name); } static void @@ -2255,7 +2812,8 @@ proto_apply_cmd_patt(const char *patt, void (* cmd)(struct proto *, uintptr_t, i WALK_LIST(p, proto_list) if (!patt || patmatch(patt, p->name)) - cmd(p, arg, cnt++); + PROTO_LOCKED_FROM_MAIN(p) + cmd(p, arg, cnt++); if (!cnt) cli_msg(8003, "No protocols match"); diff --git a/nest/protocol.h b/nest/protocol.h index abcc505d..a4b6152b 100644 --- a/nest/protocol.h +++ b/nest/protocol.h @@ -13,11 +13,11 @@ #include "lib/resource.h" #include "lib/event.h" #include "nest/route.h" +#include "nest/limit.h" #include "conf/conf.h" struct iface; struct ifa; -struct rtable; struct rte; struct neighbor; struct rta; @@ -74,12 +74,9 @@ struct protocol { struct proto * (*init)(struct proto_config *); /* Create new instance */ int (*reconfigure)(struct proto *, struct proto_config *); /* Try to reconfigure instance, returns success */ void (*dump)(struct proto *); /* Debugging dump */ - void (*dump_attrs)(struct rte *); /* Dump protocol-dependent attributes */ int (*start)(struct proto *); /* Start the instance */ int (*shutdown)(struct proto *); /* Stop the instance */ - void (*cleanup)(struct proto *); /* Called after shutdown when protocol became hungry/down */ void (*get_status)(struct proto *, byte *buf); /* Get instance status (for `show protocols' command) */ - void (*get_route_info)(struct rte *, byte *buf); /* Get route information (for `show route' command) */ int (*get_attr)(const struct eattr *, byte *buf, int buflen); /* ASCIIfy dynamic attribute (returns GA_*) */ void (*show_proto_info)(struct proto *); /* Show protocol info (for `show protocols all' command) */ void (*copy_config)(struct proto_config *, struct proto_config *); /* Copy config from given protocol instance */ @@ -120,9 +117,10 @@ struct proto_config { int class; /* SYM_PROTO or SYM_TEMPLATE */ u8 net_type; /* Protocol network type (NET_*), 0 for undefined */ u8 disabled; /* Protocol enabled/disabled by default */ - u8 vrf_set; /* Related VRF instance (below) is defined */ + u8 late_if_feed; /* Delay interface feed after channels are up */ u32 debug, mrtdump; /* Debugging bitfields, both use D_* constants */ u32 router_id; /* Protocol specific router ID */ + uint loop_order; /* Launch a birdloop on this locking level; use DOMAIN_ORDER(the_bird) for mainloop */ list channels; /* List of channel configs (struct channel_config) */ struct iface *vrf; /* Related VRF instance, NULL if global */ @@ -133,31 +131,6 @@ struct proto_config { }; /* Protocol statistics */ -struct proto_stats { - /* Import - from protocol to core */ - u32 imp_routes; /* Number of routes successfully imported to the (adjacent) routing table */ - u32 filt_routes; /* Number of routes rejected in import filter but kept in the routing table */ - u32 pref_routes; /* Number of routes selected as best in the (adjacent) routing table */ - u32 imp_updates_received; /* Number of route updates received */ - u32 imp_updates_invalid; /* Number of route updates rejected as invalid */ - u32 imp_updates_filtered; /* Number of route updates rejected by filters */ - u32 imp_updates_ignored; /* Number of route updates rejected as already in route table */ - u32 imp_updates_accepted; /* Number of route updates accepted and imported */ - u32 imp_withdraws_received; /* Number of route withdraws received */ - u32 imp_withdraws_invalid; /* Number of route withdraws rejected as invalid */ - u32 imp_withdraws_ignored; /* Number of route withdraws rejected as already not in route table */ - u32 imp_withdraws_accepted; /* Number of route withdraws accepted and processed */ - - /* Export - from core to protocol */ - u32 exp_routes; /* Number of routes successfully exported to the protocol */ - u32 exp_updates_received; /* Number of route updates received */ - u32 exp_updates_rejected; /* Number of route updates rejected by protocol */ - u32 exp_updates_filtered; /* Number of route updates rejected by filters */ - u32 exp_updates_accepted; /* Number of route updates accepted and exported */ - u32 exp_withdraws_received; /* Number of route withdraws received */ - u32 exp_withdraws_accepted; /* Number of route withdraws accepted and processed */ -}; - struct proto { node n; /* Node in global proto_list */ struct protocol *proto; /* Protocol */ @@ -165,22 +138,23 @@ struct proto { struct proto_config *cf_new; /* Configuration we want to switch to after shutdown (NULL=delete) */ pool *pool; /* Pool containing local objects */ event *event; /* Protocol event */ + struct birdloop *loop; /* BIRDloop running this protocol */ list channels; /* List of channels to rtables (struct channel) */ struct channel *main_channel; /* Primary channel */ struct rte_src *main_source; /* Primary route source */ + struct rte_owner sources; /* Route source owner structure */ struct iface *vrf; /* Related VRF instance, NULL if global */ const char *name; /* Name of this instance (== cf->name) */ u32 debug; /* Debugging flags */ u32 mrtdump; /* MRTDump flags */ uint active_channels; /* Number of active channels */ + uint active_coroutines; /* Number of active coroutines */ byte net_type; /* Protocol network type (NET_*), 0 for undefined */ byte disabled; /* Manually disabled */ - byte vrf_set; /* Related VRF instance (above) is defined */ byte proto_state; /* Protocol state machine (PS_*, see below) */ byte active; /* From PS_START to cleanup after PS_STOP */ - byte do_start; /* Start actions are scheduled */ byte do_stop; /* Stop actions are scheduled */ byte reconfiguring; /* We're shutting down due to reconfiguration */ byte gr_recovery; /* Protocol should participate in graceful restart recovery */ @@ -198,12 +172,11 @@ struct proto { * ifa_notify Notify protocol about interface address changes. * rt_notify Notify protocol about routing table updates. * neigh_notify Notify protocol about neighbor cache events. - * make_tmp_attrs Add attributes to rta from from private attrs stored in rte. The route and rta MUST NOT be cached. - * store_tmp_attrs Store private attrs back to rte and undef added attributes. The route and rta MUST NOT be cached. - * preexport Called as the first step of the route exporting process. - * It can construct a new rte, add private attributes and - * decide whether the route shall be exported: 1=yes, -1=no, - * 0=process it through the export filter set by the user. + * preexport Called as the first step of the route exporting process. + * It can decide whether the route shall be exported: + * -1 = reject, + * 0 = continue to export filter + * 1 = accept immediately * reload_routes Request channel to reload all its routes to the core * (using rte_update()). Returns: 0=reload cannot be done, * 1= reload is scheduled and will happen (asynchronously). @@ -213,11 +186,9 @@ struct proto { void (*if_notify)(struct proto *, unsigned flags, struct iface *i); void (*ifa_notify)(struct proto *, unsigned flags, struct ifa *a); - void (*rt_notify)(struct proto *, struct channel *, struct network *net, struct rte *new, struct rte *old); + void (*rt_notify)(struct proto *, struct channel *, const net_addr *net, struct rte *new, const struct rte *old); void (*neigh_notify)(struct neighbor *neigh); - void (*make_tmp_attrs)(struct rte *rt, struct linpool *pool); - void (*store_tmp_attrs)(struct rte *rt, struct linpool *pool); - int (*preexport)(struct proto *, struct rte **rt, struct linpool *pool); + int (*preexport)(struct channel *, struct rte *rt); void (*reload_routes)(struct channel *); void (*feed_begin)(struct channel *, int initial); void (*feed_end)(struct channel *); @@ -233,13 +204,12 @@ struct proto { * rte_remove Called whenever a rte is removed from the routing table. */ - int (*rte_recalculate)(struct rtable *, struct network *, struct rte *, struct rte *, struct rte *); + int (*rte_recalculate)(rtable *, struct network *, struct rte *, struct rte *, struct rte *); int (*rte_better)(struct rte *, struct rte *); - int (*rte_same)(struct rte *, struct rte *); int (*rte_mergable)(struct rte *, struct rte *); - struct rte * (*rte_modify)(struct rte *, struct linpool *); void (*rte_insert)(struct network *, struct rte *); void (*rte_remove)(struct network *, struct rte *); + u32 (*rte_igp_metric)(struct rte *); /* Hic sunt protocol-specific data */ }; @@ -279,7 +249,7 @@ void channel_graceful_restart_unlock(struct channel *c); #define DEFAULT_GR_WAIT 240 -void channel_show_limit(struct channel_limit *l, const char *dsc); +void channel_show_limit(struct limit *l, const char *dsc, int active, int action); void channel_show_info(struct channel *c); void channel_cmd_debug(struct channel *c, uint mask); @@ -298,6 +268,18 @@ struct proto *proto_iterate_named(struct symbol *sym, struct protocol *proto, st #define PROTO_WALK_CMD(sym,pr,p) for(struct proto *p = NULL; p = proto_iterate_named(sym, pr, p); ) +#define PROTO_ENTER_FROM_MAIN(p) ({ \ + ASSERT_DIE(birdloop_inside(&main_birdloop)); \ + struct birdloop *_loop = (p)->loop; \ + if (_loop != &main_birdloop) birdloop_enter(_loop); \ + _loop; \ + }) + +#define PROTO_LEAVE_FROM_MAIN(loop) ({ if (loop != &main_birdloop) birdloop_leave(loop); }) + +#define PROTO_LOCKED_FROM_MAIN(p) for (struct birdloop *_proto_loop = PROTO_ENTER_FROM_MAIN(p); _proto_loop; PROTO_LEAVE_FROM_MAIN(_proto_loop), (_proto_loop = NULL)) + + #define CMD_RELOAD 0 #define CMD_RELOAD_IN 1 #define CMD_RELOAD_OUT 2 @@ -386,6 +368,8 @@ void proto_notify_state(struct proto *p, unsigned state); * as a result of received ROUTE-REFRESH request). */ +static inline int proto_is_inactive(struct proto *p) +{ return (p->active_channels == 0) && (p->active_coroutines == 0) && (p->sources.uc == 0); } /* @@ -434,18 +418,29 @@ extern struct proto_config *cf_dev_proto; #define PLA_RESTART 4 /* Force protocol restart */ #define PLA_DISABLE 5 /* Shutdown and disable protocol */ -#define PLS_INITIAL 0 /* Initial limit state after protocol start */ -#define PLS_ACTIVE 1 /* Limit was hit */ -#define PLS_BLOCKED 2 /* Limit is active and blocking new routes */ - struct channel_limit { u32 limit; /* Maximum number of prefixes */ u8 action; /* Action to take (PLA_*) */ - u8 state; /* State of limit (PLS_*) */ }; -void channel_notify_limit(struct channel *c, struct channel_limit *l, int dir, u32 rt_count); +struct channel_limit_data { + struct channel *c; + int dir; +}; + +#define CLP__RX(_c) (&(_c)->rx_limit) +#define CLP__IN(_c) (&(_c)->in_limit) +#define CLP__OUT(_c) (&(_c)->out_limit) + + +#if 0 +#define CHANNEL_LIMIT_LOG(_c, _dir, _op) log(L_TRACE "%s.%s: %s limit %s %u", (_c)->proto->name, (_c)->name, #_dir, _op, (CLP__##_dir(_c))->count) +#else +#define CHANNEL_LIMIT_LOG(_c, _dir, _op) +#endif +#define CHANNEL_LIMIT_PUSH(_c, _dir) ({ CHANNEL_LIMIT_LOG(_c, _dir, "push from"); struct channel_limit_data cld = { .c = (_c), .dir = PLD_##_dir }; limit_push(CLP__##_dir(_c), &cld); }) +#define CHANNEL_LIMIT_POP(_c, _dir) ({ limit_pop(CLP__##_dir(_c)); CHANNEL_LIMIT_LOG(_c, _dir, "pop to"); }) /* * Channels @@ -469,7 +464,6 @@ struct channel_class { void (*dump)(struct proto *); /* Debugging dump */ - void (*dump_attrs)(struct rte *); /* Dump protocol-dependent attributes */ void (*get_status)(struct proto *, byte *buf); /* Get instance status (for `show protocols' command) */ void (*get_route_info)(struct rte *, byte *buf); /* Get route information (for `show route' command) */ @@ -489,6 +483,7 @@ struct channel_config { struct proto_config *parent; /* Where channel is defined (proto or template) */ struct rtable_config *table; /* Table we're attached to */ const struct filter *in_filter, *out_filter; /* Attached filters */ + struct channel_limit rx_limit; /* Limit for receiving routes from protocol (relevant when in_keep_filtered is active) */ struct channel_limit in_limit; /* Limit for importing routes from protocol */ @@ -505,23 +500,50 @@ struct channel_config { struct channel { node n; /* Node in proto->channels */ - node table_node; /* Node in table->channels */ const char *name; /* Channel name (may be NULL) */ const struct channel_class *channel; struct proto *proto; - struct rtable *table; + rtable *table; const struct filter *in_filter; /* Input filter */ const struct filter *out_filter; /* Output filter */ - struct bmap export_map; /* Keeps track which routes passed export filter */ - struct channel_limit rx_limit; /* Receive limit (for in_keep_filtered) */ - struct channel_limit in_limit; /* Input limit */ - struct channel_limit out_limit; /* Output limit */ - - struct event *feed_event; /* Event responsible for feeding */ - struct fib_iterator feed_fit; /* Routing table iterator used during feeding */ - struct proto_stats stats; /* Per-channel protocol statistics */ + struct bmap export_map; /* Keeps track which routes were really exported */ + struct bmap export_reject_map; /* Keeps track which routes were rejected by export filter */ + + struct limit rx_limit; /* Receive limit (for in_keep_filtered) */ + struct limit in_limit; /* Input limit */ + struct limit out_limit; /* Output limit */ + + u8 limit_actions[PLD_MAX]; /* Limit actions enum */ + u8 limit_active; /* Flags for active limits */ + + linpool *rte_update_pool; + uint rte_update_nest_cnt; + + struct channel_import_stats { + /* Import - from protocol to core */ + u32 updates_received; /* Number of route updates received */ + u32 updates_invalid; /* Number of route updates rejected as invalid */ + u32 updates_filtered; /* Number of route updates rejected by filters */ + u32 updates_limited_rx; /* Number of route updates exceeding the rx_limit */ + u32 updates_limited_in; /* Number of route updates exceeding the in_limit */ + u32 withdraws_received; /* Number of route withdraws received */ + u32 withdraws_invalid; /* Number of route withdraws rejected as invalid */ + } import_stats; + + struct channel_export_stats { + /* Export - from core to protocol */ + u32 updates_rejected; /* Number of route updates rejected by protocol */ + u32 updates_filtered; /* Number of route updates rejected by filters */ + u32 updates_accepted; /* Number of route updates accepted and exported */ + u32 updates_limited; /* Number of route updates exceeding the out_limit */ + u32 withdraws_accepted; /* Number of route withdraws accepted and processed */ + } export_stats; + + struct rt_import_request in_req; /* Table import connection */ + struct rt_export_request out_req; /* Table export connection */ + u32 refeed_count; /* Number of routes exported during refeed regardless of out_limit */ u8 net_type; /* Routing table network type (NET_*), 0 for undefined */ @@ -534,31 +556,36 @@ struct channel { u8 stale; /* Used in reconfiguration */ u8 channel_state; - u8 export_state; /* Route export state (ES_*, see below) */ - u8 feed_active; - u8 flush_active; - u8 refeeding; /* We are refeeding (valid only if export_state == ES_FEEDING) */ + u8 refeeding; /* Refeeding the channel. */ u8 reloadable; /* Hook reload_routes() is allowed on the channel */ u8 gr_lock; /* Graceful restart mechanism should wait for this channel */ u8 gr_wait; /* Route export to channel is postponed until graceful restart */ + u8 restart_export; /* Route export should restart as soon as it stops */ btime last_state_change; /* Time of last state transition */ - struct rtable *in_table; /* Internal table for received routes */ - struct event *reload_event; /* Event responsible for reloading from in_table */ - struct fib_iterator reload_fit; /* FIB iterator in in_table used during reloading */ - struct rte *reload_next_rte; /* Route iterator in in_table used during reloading */ - u8 reload_active; /* Iterator reload_fit is linked */ + struct channel_aux_table *in_table; /* Internal table for received routes */ + struct event in_stopped; /* Import stop callback */ u8 reload_pending; /* Reloading and another reload is scheduled */ u8 refeed_pending; /* Refeeding and another refeed is scheduled */ u8 rpki_reload; /* RPKI changes trigger channel reload */ - struct rtable *out_table; /* Internal table for exported routes */ + struct channel_aux_table *out_table; /* Internal table for exported routes */ list roa_subscriptions; /* List of active ROA table subscriptions based on filters roa_check() */ }; +struct channel_aux_table { + struct channel *c; + struct rt_import_request push; + struct rt_export_request get; + event push_stopped; + rtable *tab; + event *stop; + u8 refeed_pending; + u8 stop_pending; +}; /* * Channel states @@ -585,70 +612,59 @@ struct channel { * restricted by that and is on volition of the protocol. Generally, channels * are opened in protocols' start() hooks when going to PS_UP. * - * CS_FLUSHING - The transitional state between initialized channel and closed + * CS_STOP - The transitional state between initialized channel and closed * channel. The channel is still initialized, but no route exchange is allowed. * Instead, the associated table is running flush loop to remove routes imported * through the channel. After that, the channel changes state to CS_DOWN and * is detached from the table (the table is unlocked and the channel is unlinked - * from it). Unlike other states, the CS_FLUSHING state is not explicitly + * from it). Unlike other states, the CS_STOP state is not explicitly * entered or left by the protocol. A protocol may request to close a channel * (by calling channel_close()), which causes the channel to change state to - * CS_FLUSHING and later to CS_DOWN. Also note that channels are closed + * CS_STOP and later to CS_DOWN. Also note that channels are closed * automatically by the core when the protocol is going down. * + * CS_PAUSE - Almost the same as CS_STOP, just the table import is kept and + * the table export is stopped before transitioning to CS_START. + * * Allowed transitions: * * CS_DOWN -> CS_START / CS_UP - * CS_START -> CS_UP / CS_FLUSHING - * CS_UP -> CS_START / CS_FLUSHING - * CS_FLUSHING -> CS_DOWN (automatic) + * CS_START -> CS_UP / CS_STOP + * CS_UP -> CS_PAUSE / CS_STOP + * CS_PAUSE -> CS_START (automatic) + * CS_STOP -> CS_DOWN (automatic) */ #define CS_DOWN 0 #define CS_START 1 #define CS_UP 2 -#define CS_FLUSHING 3 - -#define ES_DOWN 0 -#define ES_FEEDING 1 -#define ES_READY 2 - +#define CS_STOP 3 +#define CS_PAUSE 4 struct channel_config *proto_cf_find_channel(struct proto_config *p, uint net_type); static inline struct channel_config *proto_cf_main_channel(struct proto_config *pc) { return proto_cf_find_channel(pc, pc->net_type); } -struct channel *proto_find_channel_by_table(struct proto *p, struct rtable *t); +struct channel *proto_find_channel_by_table(struct proto *p, rtable *t); struct channel *proto_find_channel_by_name(struct proto *p, const char *n); struct channel *proto_add_channel(struct proto *p, struct channel_config *cf); int proto_configure_channel(struct proto *p, struct channel **c, struct channel_config *cf); void channel_set_state(struct channel *c, uint state); -void channel_setup_in_table(struct channel *c); +void channel_setup_in_table(struct channel *c, int best); void channel_setup_out_table(struct channel *c); void channel_schedule_reload(struct channel *c); static inline void channel_init(struct channel *c) { channel_set_state(c, CS_START); } static inline void channel_open(struct channel *c) { channel_set_state(c, CS_UP); } -static inline void channel_close(struct channel *c) { channel_set_state(c, CS_FLUSHING); } +static inline void channel_close(struct channel *c) { channel_set_state(c, CS_STOP); } void channel_request_feeding(struct channel *c); +void channel_request_reload(struct channel *c); +void channel_refresh_begin(struct channel *c); +void channel_refresh_end(struct channel *c); void *channel_config_new(const struct channel_class *cc, const char *name, uint net_type, struct proto_config *proto); void *channel_config_get(const struct channel_class *cc, const char *name, uint net_type, struct proto_config *proto); int channel_reconfigure(struct channel *c, struct channel_config *cf); - -/* Moved from route.h to avoid dependency conflicts */ -static inline void rte_update(struct proto *p, const net_addr *n, rte *new) { rte_update2(p->main_channel, n, new, p->main_source); } - -static inline void -rte_update3(struct channel *c, const net_addr *n, rte *new, struct rte_src *src) -{ - if (c->in_table && !rte_update_in(c, n, new, src)) - return; - - rte_update2(c, n, new, src); -} - - #endif diff --git a/nest/route.h b/nest/route.h index f5fc9e31..9093108b 100644 --- a/nest/route.h +++ b/nest/route.h @@ -2,6 +2,7 @@ * BIRD Internet Routing Daemon -- Routing Table * * (c) 1998--2000 Martin Mares <mj@ucw.cz> + * (c) 2019--2021 Maria Matejka <mq@jmq.cz> * * Can be freely distributed and used under the terms of the GNU GPL. */ @@ -10,13 +11,19 @@ #define _BIRD_ROUTE_H_ #include "lib/lists.h" +#include "lib/event.h" #include "lib/bitmap.h" #include "lib/resource.h" #include "lib/net.h" +#include "lib/hash.h" +#include "lib/event.h" + +#include <stdatomic.h> struct ea_list; struct protocol; struct proto; +struct channel; struct rte_src; struct symbol; struct timer; @@ -139,61 +146,91 @@ void fit_copy(struct fib *f, struct fib_iterator *dst, struct fib_iterator *src) * It's guaranteed that there is at most one RTE for every (prefix,proto) pair. */ -struct rtable_config { - node n; - char *name; - struct rtable *table; - struct proto_config *krt_attached; /* Kernel syncer attached to this table */ - uint addr_type; /* Type of address data stored in table (NET_*) */ - int gc_max_ops; /* Maximum number of operations before GC is run */ - int gc_min_time; /* Minimum time between two consecutive GC runs */ - byte sorted; /* Routes of network are sorted according to rte_better() */ - byte internal; /* Internal table of a protocol */ - btime min_settle_time; /* Minimum settle time for notifications */ - btime max_settle_time; /* Maximum settle time for notifications */ -}; - -typedef struct rtable { - resource r; - node n; /* Node in list of all tables */ +typedef struct rtable_private { +#define RTABLE_PUBLIC \ + resource r; \ + node n; /* Node in list of all tables */ \ + struct birdloop *loop; /* This loop runs the table */ \ + char *name; /* Name of this table */ \ + uint addr_type; /* Type of address data stored in table (NET_*) */ \ + struct rtable_config *config; /* Configuration of this table */ \ + struct event *nhu_event; /* Event to update next hops */ \ + _Atomic byte nhu_state; /* Next Hop Update state */ \ + + RTABLE_PUBLIC; pool *rp; /* Resource pool to allocate everything from, including itself */ + struct slab *rte_slab; /* Slab to allocate route objects */ struct fib fib; - char *name; /* Name of this table */ - list channels; /* List of attached channels (struct channel) */ - uint addr_type; /* Type of address data stored in table (NET_*) */ - int pipe_busy; /* Pipe loop detection */ int use_count; /* Number of protocols using this table */ u32 rt_count; /* Number of routes in the table */ + u32 rr_count; /* Number of running route refresh requests */ + u32 imports_up; /* Number of imports in TIS_UP state */ - byte internal; /* Internal table of a protocol */ + list imports; /* Registered route importers */ + list exports; /* Registered route exporters */ struct hmap id_map; struct hostcache *hostcache; - struct rtable_config *config; /* Configuration of this table */ - struct config *deleted; /* Table doesn't exist in current configuration, - * delete as soon as use_count becomes 0 and remove - * obstacle from this routing table. - */ - struct event *rt_event; /* Routing table event */ + struct event *prune_event; /* Event to prune abandoned routes */ + struct event *announce_event; /* Event to announce pending exports */ + struct event *ec_event; /* Event to prune finished exports */ + struct event *hcu_event; /* Event to update host cache */ + void (*delete)(void *); /* Delete callback (in parent loop context) */ btime last_rt_change; /* Last time when route changed */ btime base_settle_time; /* Start time of rtable settling interval */ btime gc_time; /* Time of last GC */ int gc_counter; /* Number of operations since last GC */ byte prune_state; /* Table prune state, 1 -> scheduled, 2-> running */ - byte hcu_scheduled; /* Hostcache update is scheduled */ - byte nhu_state; /* Next Hop Update state */ + + byte cork_active; /* Congestion control activated */ + struct fib_iterator prune_fit; /* Rtable prune FIB iterator */ struct fib_iterator nhu_fit; /* Next Hop Update FIB iterator */ + struct tbf rl_pipe; /* Rate limiting token buffer for pipe collisions */ + + linpool *nhu_lp; /* Linpool used for NHU */ list subscribers; /* Subscribers for notifications */ struct timer *settle_timer; /* Settle time for notifications */ + + list pending_exports; /* List of packed struct rt_pending_export */ + + struct rt_pending_export *first_export; /* First export to announce */ + u64 next_export_seq; /* The next export will have this ID */ +} rtable_private; + +typedef union { + struct { RTABLE_PUBLIC }; + rtable_private priv; } rtable; +#define RT_LOCK(tab) ({ birdloop_enter((tab)->loop); &(tab)->priv; }) +#define RT_UNLOCK(tab) birdloop_leave((tab)->loop) +#define RT_PRIV(tab) ({ ASSERT_DIE(birdloop_inside((tab)->loop)); &(tab)->priv; }) + +#define RT_LOCKED(tpub, tpriv) for (rtable_private *tpriv = RT_LOCK(tpub); tpriv; RT_UNLOCK(tpriv), (tpriv = NULL)) + +struct rtable_config { + node n; + char *name; + void *owner; /* Main config if global table, channel_aux_table if channel table */ + rtable *table; + struct proto_config *krt_attached; /* Kernel syncer attached to this table */ + uint addr_type; /* Type of address data stored in table (NET_*) */ + int gc_max_ops; /* Maximum number of operations before GC is run */ + int gc_min_time; /* Minimum time between two consecutive GC runs */ + byte sorted; /* Routes of network are sorted according to rte_better() */ + btime min_settle_time; /* Minimum settle time for notifications */ + btime max_settle_time; /* Maximum settle time for notifications */ + btime min_rr_settle_time; /* Minimum settle time for notifications when route refresh is running */ + btime max_rr_settle_time; /* Maximum settle time for notifications when route refresh is running */ + uint cork_limit; /* Amount of routes to be pending on export to cork imports */ +}; + struct rt_subscription { node n; rtable *tab; - void (*hook)(struct rt_subscription *b); - void *data; + event *event; }; #define NHU_CLEAN 0 @@ -202,7 +239,8 @@ struct rt_subscription { #define NHU_DIRTY 3 typedef struct network { - struct rte *routes; /* Available routes for this network */ + struct rte_storage *routes; /* Available routes for this network */ + struct rt_pending_export *last, *first; /* Routes with unfinished exports */ struct fib_node n; /* FIB flags reserved for kernel syncer */ } net; @@ -223,7 +261,7 @@ struct hostentry { ip_addr addr; /* IP address of host, part of key */ ip_addr link; /* (link-local) IP address of host, used as gw if host is directly attached */ - struct rtable *tab; /* Dependent table, part of key */ + rtable *tab; /* Dependent table, part of key */ struct hostentry *next; /* Next in hash chain */ unsigned hash_key; /* Hash key */ unsigned uc; /* Use count */ @@ -234,64 +272,178 @@ struct hostentry { }; typedef struct rte { - struct rte *next; - net *net; /* Network this RTE belongs to */ - struct channel *sender; /* Channel used to send the route to the routing table */ struct rta *attrs; /* Attributes of this route */ + const net_addr *net; /* Network this RTE belongs to */ + struct rte_src *src; /* Route source that created the route */ + struct rt_import_hook *sender; /* Import hook used to send the route to the routing table */ + btime lastmod; /* Last modified (set by table) */ u32 id; /* Table specific route id */ - byte flags; /* Flags (REF_...) */ + byte flags; /* Table-specific flags */ byte pflags; /* Protocol-specific flags */ - word pref; /* Route preference */ - btime lastmod; /* Last modified */ - union { /* Protocol-dependent data (metrics etc.) */ -#ifdef CONFIG_RIP - struct { - struct iface *from; /* Incoming iface */ - u8 metric; /* RIP metric */ - u16 tag; /* External route tag */ - } rip; -#endif -#ifdef CONFIG_OSPF - struct { - u32 metric1, metric2; /* OSPF Type 1 and Type 2 metrics */ - u32 tag; /* External route tag */ - u32 router_id; /* Router that originated this route */ - } ospf; -#endif -#ifdef CONFIG_BGP - struct { - u8 suppressed; /* Used for deterministic MED comparison */ - s8 stale; /* Route is LLGR_STALE, -1 if unknown */ - } bgp; -#endif -#ifdef CONFIG_BABEL - struct { - u16 seqno; /* Babel seqno */ - u16 metric; /* Babel metric */ - u64 router_id; /* Babel router id */ - } babel; -#endif - struct { /* Routes generated by krt sync (both temporary and inherited ones) */ - s8 src; /* Alleged route source (see krt.h) */ - u8 proto; /* Kernel source protocol ID */ - u8 seen; /* Seen during last scan */ - u8 best; /* Best route in network, propagated to core */ - u32 metric; /* Kernel metric */ - } krt; - } u; + u8 generation; /* If this route import is based on other previously exported route, + this value should be 1 + MAX(generation of the parent routes). + Otherwise the route is independent and this value is zero. */ + u8 stale_cycle; /* Auxiliary value for route refresh */ } rte; -#define REF_COW 1 /* Copy this rte on write */ +struct rte_storage { + struct rte_storage *next; /* Next in chain */ + struct rte rte; /* Route data */ +}; + +#define RTES_CLONE(r, l) ((r) ? (((*(l)) = (r)->rte), (l)) : NULL) +#define RTES_OR_NULL(r) ((r) ? &((r)->rte) : NULL) + #define REF_FILTERED 2 /* Route is rejected by import filter */ -#define REF_STALE 4 /* Route is stale in a refresh cycle */ -#define REF_DISCARD 8 /* Route is scheduled for discard */ -#define REF_MODIFY 16 /* Route is scheduled for modify */ +#define REF_USE_STALE 4 /* Do not reset route's stale_cycle to the actual value */ /* Route is valid for propagation (may depend on other flags in the future), accepts NULL */ -static inline int rte_is_valid(rte *r) { return r && !(r->flags & REF_FILTERED); } +static inline int rte_is_valid(const rte *r) { return r && !(r->flags & REF_FILTERED); } /* Route just has REF_FILTERED flag */ -static inline int rte_is_filtered(rte *r) { return !!(r->flags & REF_FILTERED); } +static inline int rte_is_filtered(const rte *r) { return !!(r->flags & REF_FILTERED); } + + +/* Table-channel connections */ + +struct rt_import_request { + struct rt_import_hook *hook; /* The table part of importer */ + char *name; + u8 trace_routes; + + event_list *list; /* Where to schedule import events */ + + void (*dump_req)(struct rt_import_request *req); + void (*log_state_change)(struct rt_import_request *req, u8 state); + /* Preimport is called when the @new route is just-to-be inserted, replacing @old. + * Return a route (may be different or modified in-place) to continue or NULL to withdraw. */ + struct rte *(*preimport)(struct rt_import_request *req, struct rte *new, struct rte *old); +}; + +struct rt_import_hook { + node n; + rtable *table; /* The connected table */ + struct rt_import_request *req; /* The requestor */ + + struct rt_import_stats { + /* Import - from protocol to core */ + u32 pref; /* Number of routes selected as best in the (adjacent) routing table */ + u32 updates_ignored; /* Number of route updates rejected as already in route table */ + u32 updates_accepted; /* Number of route updates accepted and imported */ + u32 withdraws_ignored; /* Number of route withdraws rejected as already not in route table */ + u32 withdraws_accepted; /* Number of route withdraws accepted and processed */ + } stats; + + u64 flush_seq; /* Table export seq when the channel announced flushing */ + btime last_state_change; /* Time of last state transition */ + + u8 import_state; /* IS_* */ + u8 stale_set; /* Set this stale_cycle to imported routes */ + u8 stale_valid; /* Routes with this stale_cycle and bigger are considered valid */ + u8 stale_pruned; /* Last prune finished when this value was set at stale_valid */ + u8 stale_pruning; /* Last prune started when this value was set at stale_valid */ + + struct event *export_announce_event; /* Event to run to announce new exports */ + struct event *stopped; /* Event to run when import is stopped */ +}; + +struct rt_pending_export { + struct rt_pending_export * _Atomic next; /* Next export for the same destination */ + struct rte_storage *new, *new_best, *old, *old_best; + u64 seq; /* Sequential ID (table-local) of the pending export */ +}; + +struct rt_export_request { + struct rt_export_hook *hook; /* Table part of the export */ + char *name; + u8 trace_routes; + + event_list *list; /* Where to schedule export events */ + + /* There are two methods of export. You can either request feeding every single change + * or feeding the whole route feed. In case of regular export, &export_one is preferred. + * Anyway, when feeding, &export_bulk is preferred, falling back to &export_one. + * Thus, for RA_OPTIMAL, &export_one is only set, + * for RA_MERGED and RA_ACCEPTED, &export_bulk is only set + * and for RA_ANY, both are set to accomodate for feeding all routes but receiving single changes + */ + void (*export_one)(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe); + void (*export_bulk)(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe, rte **feed, uint count); + + void (*dump_req)(struct rt_export_request *req); + void (*log_state_change)(struct rt_export_request *req, u8); +}; + +struct rt_export_hook { + node n; + rtable *table; /* The connected table */ + + pool *pool; + + struct rt_export_request *req; /* The requestor */ + + struct rt_export_stats { + /* Export - from core to protocol */ + u32 updates_received; /* Number of route updates received */ + u32 withdraws_received; /* Number of route withdraws received */ + } stats; + + struct fib_iterator feed_fit; /* Routing table iterator used during feeding */ + + struct bmap seq_map; /* Keep track which exports were already procesed */ + + struct rt_pending_export * _Atomic last_export;/* Last export processed */ + struct rt_pending_export *rpe_next; /* Next pending export to process */ + + btime last_state_change; /* Time of last state transition */ + + u8 refeed_pending; /* Refeeding and another refeed is scheduled */ + _Atomic u8 export_state; /* Route export state (TES_*, see below) */ + + struct event *event; /* Event running all the export operations */ + + void (*stopped)(struct rt_export_request *); /* Stored callback when export is stopped */ +}; + +extern struct event_cork rt_cork; + +#define TIS_DOWN 0 +#define TIS_UP 1 +#define TIS_STOP 2 +#define TIS_FLUSHING 3 +#define TIS_WAITING 4 +#define TIS_CLEARED 5 +#define TIS_MAX 6 + +#define TES_DOWN 0 +#define TES_HUNGRY 1 +#define TES_FEEDING 2 +#define TES_READY 3 +#define TES_STOP 4 +#define TES_MAX 5 + +void rt_request_import(rtable *tab, struct rt_import_request *req); +void rt_request_export(rtable *tab, struct rt_export_request *req); + +void rt_stop_import(struct rt_import_request *, struct event *stopped); +void rt_stop_export(struct rt_export_request *, void (*stopped)(struct rt_export_request *)); + +const char *rt_import_state_name(u8 state); +const char *rt_export_state_name(u8 state); + +static inline u8 rt_import_get_state(struct rt_import_hook *ih) { return ih ? ih->import_state : TIS_DOWN; } +static inline u8 rt_export_get_state(struct rt_export_hook *eh) { return eh ? eh->export_state : TES_DOWN; } + +void rte_import(struct rt_import_request *req, const net_addr *net, rte *new, struct rte_src *src); + +/* Get next rpe. If src is given, it must match. */ +struct rt_pending_export *rpe_next(struct rt_pending_export *rpe, struct rte_src *src); + +/* Mark the pending export processed */ +void rpe_mark_seen(struct rt_export_hook *hook, struct rt_pending_export *rpe); + +/* Get pending export seen status */ +int rpe_get_seen(struct rt_export_hook *hook, struct rt_pending_export *rpe); /* Types of route announcement, also used as flags */ @@ -307,56 +459,74 @@ static inline int rte_is_filtered(rte *r) { return !!(r->flags & REF_FILTERED); #define RIC_REJECT -1 /* Rejected by protocol */ #define RIC_DROP -2 /* Silently dropped by protocol */ +#define rte_update channel_rte_import +/** + * rte_update - enter a new update to a routing table + * @c: channel doing the update + * @net: network address + * @rte: a &rte representing the new route + * @src: old route source identifier + * + * This function imports a new route to the appropriate table (via the channel). + * Table keys are @net (obligatory) and @rte->attrs->src. + * Both the @net and @rte pointers can be local. + * + * The route attributes (@rte->attrs) are obligatory. They can be also allocated + * locally. Anyway, if you use an already-cached attribute object, you shall + * call rta_clone() on that object yourself. (This semantics may change in future.) + * + * If the route attributes are local, you may set @rte->attrs->src to NULL, then + * the protocol's default route source will be supplied. + * + * When rte_update() gets a route, it automatically validates it. This includes + * checking for validity of the given network and next hop addresses and also + * checking for host-scope or link-scope routes. Then the import filters are + * processed and if accepted, the route is passed to route table recalculation. + * + * The accepted routes are then inserted into the table, replacing the old route + * for the same @net identified by @src. Then the route is announced + * to all the channels connected to the table using the standard export mechanism. + * Setting @rte to NULL makes this a withdraw, otherwise @rte->src must be the same + * as @src. + * + * All memory used for temporary allocations is taken from a special linpool + * @rte_update_pool and freed when rte_update() finishes. + */ +void rte_update(struct channel *c, const net_addr *net, struct rte *rte, struct rte_src *src); + extern list routing_tables; struct config; void rt_init(void); void rt_preconfig(struct config *); void rt_commit(struct config *new, struct config *old); -void rt_lock_table(rtable *); -void rt_unlock_table(rtable *); +void rt_lock_table(rtable_private *); +void rt_unlock_table(rtable_private *); void rt_subscribe(rtable *tab, struct rt_subscription *s); void rt_unsubscribe(struct rt_subscription *s); rtable *rt_setup(pool *, struct rtable_config *); -static inline void rt_shutdown(rtable *r) { rfree(r->rp); } -static inline net *net_find(rtable *tab, const net_addr *addr) { return (net *) fib_find(&tab->fib, addr); } -static inline net *net_find_valid(rtable *tab, const net_addr *addr) -{ net *n = net_find(tab, addr); return (n && rte_is_valid(n->routes)) ? n : NULL; } -static inline net *net_get(rtable *tab, const net_addr *addr) { return (net *) fib_get(&tab->fib, addr); } -void *net_route(rtable *tab, const net_addr *n); +static inline net *net_find(rtable_private *tab, const net_addr *addr) { return (net *) fib_find(&tab->fib, addr); } +static inline net *net_find_valid(rtable_private *tab, const net_addr *addr) +{ net *n = net_find(tab, addr); return (n && n->routes && rte_is_valid(&n->routes->rte)) ? n : NULL; } +static inline net *net_get(rtable_private *tab, const net_addr *addr) { return (net *) fib_get(&tab->fib, addr); } +void *net_route(rtable_private *tab, const net_addr *n); int net_roa_check(rtable *tab, const net_addr *n, u32 asn); -rte *rte_find(net *net, struct rte_src *src); -rte *rte_get_temp(struct rta *); -void rte_update2(struct channel *c, const net_addr *n, rte *new, struct rte_src *src); -/* rte_update() moved to protocol.h to avoid dependency conflicts */ -int rt_examine(rtable *t, net_addr *a, struct proto *p, const struct filter *filter); -rte *rt_export_merged(struct channel *c, net *net, rte **rt_free, linpool *pool, int silent); -void rt_refresh_begin(rtable *t, struct channel *c); -void rt_refresh_end(rtable *t, struct channel *c); -void rt_modify_stale(rtable *t, struct channel *c); -void rt_schedule_prune(rtable *t); -void rte_dump(rte *); -void rte_free(rte *); -rte *rte_do_cow(rte *); -static inline rte * rte_cow(rte *r) { return (r->flags & REF_COW) ? rte_do_cow(r) : r; } -rte *rte_cow_rta(rte *r, linpool *lp); -void rte_init_tmp_attrs(struct rte *r, linpool *lp, uint max); -void rte_make_tmp_attr(struct rte *r, uint id, uint type, uintptr_t val); -void rte_make_tmp_attrs(struct rte **r, struct linpool *pool, struct rta **old_attrs); -uintptr_t rte_store_tmp_attr(struct rte *r, uint id); +int rt_examine(rtable_private *t, net_addr *a, struct channel *c, const struct filter *filter); +rte *rt_export_merged(struct channel *c, rte ** feed, uint count, linpool *pool, int silent); + +void rt_refresh_begin(struct rt_import_request *); +void rt_refresh_end(struct rt_import_request *); +void rt_schedule_prune(rtable_private *t); +void rte_dump(struct rte_storage *); +void rte_free(struct rte_storage *, rtable_private *); void rt_dump(rtable *); void rt_dump_all(void); -int rt_feed_channel(struct channel *c); -void rt_feed_channel_abort(struct channel *c); -int rte_update_in(struct channel *c, const net_addr *n, rte *new, struct rte_src *src); -int rt_reload_channel(struct channel *c); -void rt_reload_channel_abort(struct channel *c); +void rt_dump_hooks(rtable *); +void rt_dump_hooks_all(void); void rt_prune_sync(rtable *t, int all); -int rte_update_out(struct channel *c, const net_addr *n, rte *new, rte *old0, int refeed); struct rtable_config *rt_new_table(struct symbol *s, uint addr_type); - /* Default limit for ECMP next hops, defined in sysdep code */ extern const int rt_default_ecmp; @@ -379,6 +549,7 @@ struct rt_show_data { struct channel *export_channel; struct config *running_on_config; struct krt_proto *kernel; + struct rt_export_hook *kernel_export_hook; int export_mode, primary_only, filtered, stats, show_for; int table_open; /* Iteration (fit) is open */ @@ -430,30 +601,29 @@ struct nexthop { struct rte_src { struct rte_src *next; /* Hash chain */ - struct proto *proto; /* Protocol the source is based on */ + struct rte_owner *owner; /* Route source owner */ u32 private_id; /* Private ID, assigned by the protocol */ u32 global_id; /* Globally unique ID of the source */ - unsigned uc; /* Use count */ + _Atomic u64 uc; /* Use count */ }; typedef struct rta { - struct rta *next, **pprev; /* Hash chain */ - u32 uc; /* Use count */ + struct rta * _Atomic next, * _Atomic *pprev; /* Hash chain */ + _Atomic u32 uc; /* Use count */ u32 hash_key; /* Hash over important fields */ struct ea_list *eattrs; /* Extended Attribute chain */ - struct rte_src *src; /* Route source that created the route */ struct hostentry *hostentry; /* Hostentry for recursive next-hops */ ip_addr from; /* Advertising router */ u32 igp_metric; /* IGP metric to next hop (for iBGP routes) */ - u8 source; /* Route source (RTS_...) */ - u8 scope; /* Route scope (SCOPE_... -- see ip.h) */ - u8 dest; /* Route destination type (RTD_...) */ - u8 aflags; + u16 cached:1; /* Are attributes cached? */ + u16 source:7; /* Route source (RTS_...) */ + u16 scope:4; /* Route scope (SCOPE_... -- see ip.h) */ + u16 dest:4; /* Route destination type (RTD_...) */ + word pref; struct nexthop nh; /* Next hop */ } rta; -#define RTS_DUMMY 0 /* Dummy route to be removed soon */ #define RTS_STATIC 1 /* Normal static route */ #define RTS_INHERIT 2 /* Route inherited from kernel */ #define RTS_DEVICE 3 /* Device route */ @@ -471,11 +641,6 @@ typedef struct rta { #define RTS_PERF 15 /* Perf checker */ #define RTS_MAX 16 -#define RTC_UNICAST 0 -#define RTC_BROADCAST 1 -#define RTC_MULTICAST 2 -#define RTC_ANYCAST 3 /* IPv6 Anycast */ - #define RTD_NONE 0 /* Undefined next hop */ #define RTD_UNICAST 1 /* Next hop is neighbor router */ #define RTD_BLACKHOLE 2 /* Silently drop packets */ @@ -483,8 +648,6 @@ typedef struct rta { #define RTD_PROHIBIT 4 /* Administratively prohibited */ #define RTD_MAX 5 -#define RTAF_CACHED 1 /* This is a cached rta */ - #define IGP_METRIC_UNKNOWN 0x80000000 /* Default igp_metric used when no other protocol-specific metric is availabe */ @@ -508,8 +671,8 @@ typedef struct eattr { byte flags; /* Protocol-dependent flags */ byte type; /* Attribute type and several flags (EAF_...) */ union { - u32 data; - const struct adata *ptr; /* Attribute data elsewhere */ + uintptr_t data; + const struct adata *ptr; /* Attribute data elsewhere */ } u; } eattr; @@ -517,7 +680,6 @@ typedef struct eattr { #define EA_CODE(proto,id) (((proto) << 8) | (id)) #define EA_ID(ea) ((ea) & 0xff) #define EA_PROTO(ea) ((ea) >> 8) -#define EA_ID_FLAG(ea) (1 << EA_ID(ea)) #define EA_CUSTOM(id) ((id) | EA_CUSTOM_BIT) #define EA_IS_CUSTOM(ea) ((ea) & EA_CUSTOM_BIT) #define EA_CUSTOM_ID(ea) ((ea) & ~EA_CUSTOM_BIT) @@ -540,6 +702,7 @@ const char *ea_custom_name(uint ea); #define EAF_TYPE_AS_PATH 0x06 /* BGP AS path (encoding per RFC 1771:4.3) */ #define EAF_TYPE_BITFIELD 0x09 /* 32-bit embedded bitfield */ #define EAF_TYPE_INT_SET 0x0a /* Set of u32's (e.g., a community list) */ +#define EAF_TYPE_PTR 0x0d /* Pointer to an object */ #define EAF_TYPE_EC_SET 0x0e /* Set of pairs of u32's - ext. community list */ #define EAF_TYPE_LC_SET 0x12 /* Set of triplets of u32's - large community list */ #define EAF_TYPE_UNDEF 0x1f /* `force undefined' entry */ @@ -578,13 +741,52 @@ typedef struct ea_list { #define EALF_SORTED 1 /* Attributes are sorted by code */ #define EALF_BISECT 2 /* Use interval bisection for searching */ #define EALF_CACHED 4 /* Attributes belonging to cached rta */ -#define EALF_TEMP 8 /* Temporary ea_list added by make_tmp_attrs hooks */ -struct rte_src *rt_find_source(struct proto *p, u32 id); -struct rte_src *rt_get_source(struct proto *p, u32 id); -static inline void rt_lock_source(struct rte_src *src) { src->uc++; } -static inline void rt_unlock_source(struct rte_src *src) { src->uc--; } -void rt_prune_sources(void); +struct rte_owner_class { + void (*get_route_info)(struct rte *, byte *buf); /* Get route information (for `show route' command) */ + int (*rte_better)(struct rte *, struct rte *); + int (*rte_mergable)(struct rte *, struct rte *); + u32 (*rte_igp_metric)(struct rte *); +}; + +struct rte_owner { + struct rte_owner_class *class; + int (*rte_recalculate)(rtable_private *, struct network *, struct rte *, struct rte *, struct rte *); + HASH(struct rte_src) hash; + const char *name; + u32 hash_key; + u32 uc; + event_list *list; + event *prune; + event *stop; +}; + +#define RTE_SRC_PU_SHIFT 44 +#define RTE_SRC_IN_PROGRESS (1ULL << RTE_SRC_PU_SHIFT) + +struct rte_src *rt_get_source_o(struct rte_owner *o, u32 id); +#define rt_get_source(p, id) rt_get_source_o(&(p)->sources, (id)) +static inline void rt_lock_source(struct rte_src *src) +{ + u64 uc = atomic_fetch_add_explicit(&src->uc, 1, memory_order_acq_rel); + ASSERT_DIE(uc > 0); +} + +static inline void rt_unlock_source(struct rte_src *src) +{ + u64 uc = atomic_fetch_add_explicit(&src->uc, RTE_SRC_IN_PROGRESS, memory_order_acq_rel); + u64 pending = uc >> RTE_SRC_PU_SHIFT; + uc &= RTE_SRC_IN_PROGRESS - 1; + + ASSERT_DIE(uc > pending); + if (uc == pending + 1) + ev_send(src->owner->list, src->owner->prune); + + atomic_fetch_sub_explicit(&src->uc, RTE_SRC_IN_PROGRESS + 1, memory_order_acq_rel); +} + +void rt_init_sources(struct rte_owner *, const char *name, event_list *list); +void rt_destroy_sources(struct rte_owner *, event *); struct ea_walk_state { ea_list *eattrs; /* Ccurrent ea_list, initially set by caller */ @@ -594,7 +796,7 @@ struct ea_walk_state { eattr *ea_find(ea_list *, unsigned ea); eattr *ea_walk(struct ea_walk_state *s, uint id, uint max); -int ea_get_int(ea_list *, unsigned ea, int def); +uintptr_t ea_get_int(ea_list *, unsigned ea, uintptr_t def); void ea_dump(ea_list *); void ea_sort(ea_list *); /* Sort entries in all sub-lists */ unsigned ea_scan(ea_list *); /* How many bytes do we need for merged ea_list */ @@ -673,24 +875,47 @@ void rta_init(void); static inline size_t rta_size(const rta *a) { return sizeof(rta) + sizeof(u32)*a->nh.labels; } #define RTA_MAX_SIZE (sizeof(rta) + sizeof(u32)*MPLS_MAX_LABEL_STACK) rta *rta_lookup(rta *); /* Get rta equivalent to this one, uc++ */ -static inline int rta_is_cached(rta *r) { return r->aflags & RTAF_CACHED; } -static inline rta *rta_clone(rta *r) { r->uc++; return r; } -void rta__free(rta *r); -static inline void rta_free(rta *r) { if (r && !--r->uc) rta__free(r); } +static inline int rta_is_cached(rta *r) { return r->cached; } + +static inline rta *rta_clone(rta *r) { + u32 uc = atomic_fetch_add_explicit(&r->uc, 1, memory_order_acq_rel); + ASSERT_DIE(uc > 0); + return r; +} + +#define RTA_OBSOLETE_LIMIT 512 + +extern _Atomic u32 rta_obsolete_count; +extern event rta_cleanup_event; + +static inline void rta_free(rta *r) { + if (!r) + return; + + u32 uc = atomic_fetch_sub_explicit(&r->uc, 1, memory_order_acq_rel); + if (uc > 1) + return; + + u32 obs = atomic_fetch_add_explicit(&rta_obsolete_count, 1, memory_order_acq_rel); + if (obs == RTA_OBSOLETE_LIMIT) + ev_send(&global_work_list, &rta_cleanup_event); +} + rta *rta_do_cow(rta *o, linpool *lp); static inline rta * rta_cow(rta *r, linpool *lp) { return rta_is_cached(r) ? rta_do_cow(r, lp) : r; } -void rta_dump(rta *); +static inline void rta_uncache(rta *r) { r->cached = 0; r->uc = 0; } +void rta_dump(const rta *); void rta_dump_all(void); -void rta_show(struct cli *, rta *); +void rta_show(struct cli *, const rta *); -u32 rt_get_igp_metric(rte *rt); +u32 rt_get_igp_metric(rte *); struct hostentry * rt_get_hostentry(rtable *tab, ip_addr a, ip_addr ll, rtable *dep); -void rta_apply_hostentry(rta *a, struct hostentry *he, mpls_label_stack *mls); +void rta_apply_hostentry(rta *a, struct hostentry *he, mpls_label_stack *mls, linpool *lp); static inline void -rta_set_recursive_next_hop(rtable *dep, rta *a, rtable *tab, ip_addr gw, ip_addr ll, mpls_label_stack *mls) +rta_set_recursive_next_hop(rtable *dep, rta *a, rtable *tab, ip_addr gw, ip_addr ll, mpls_label_stack *mls, linpool *lp) { - rta_apply_hostentry(a, rt_get_hostentry(tab, gw, ll, dep), mls); + rta_apply_hostentry(a, rt_get_hostentry(tab, gw, ll, dep), mls, lp); } /* diff --git a/nest/rt-attr.c b/nest/rt-attr.c index c630aa95..c3da4782 100644 --- a/nest/rt-attr.c +++ b/nest/rt-attr.c @@ -54,6 +54,7 @@ #include "lib/hash.h" #include "lib/idm.h" #include "lib/resource.h" +#include "lib/rcu.h" #include "lib/string.h" #include <stddef.h> @@ -61,7 +62,6 @@ const adata null_adata; /* adata of length 0 */ const char * const rta_src_names[RTS_MAX] = { - [RTS_DUMMY] = "", [RTS_STATIC] = "static", [RTS_INHERIT] = "inherit", [RTS_DEVICE] = "device", @@ -86,7 +86,13 @@ const char * rta_dest_names[RTD_MAX] = { [RTD_PROHIBIT] = "prohibited", }; +static DOMAIN(attrs) src_domain; + +#define SRC_LOCK LOCK_DOMAIN(attrs, src_domain) +#define SRC_UNLOCK UNLOCK_DOMAIN(attrs, src_domain) + pool *rta_pool; +pool *src_pool; static slab *rta_slab_[4]; static slab *nexthop_slab_[4]; @@ -97,72 +103,151 @@ static struct idm src_ids; /* rte source hash */ -#define RSH_KEY(n) n->proto, n->private_id +#define RSH_KEY(n) n->private_id #define RSH_NEXT(n) n->next -#define RSH_EQ(p1,n1,p2,n2) p1 == p2 && n1 == n2 -#define RSH_FN(p,n) p->hash_key ^ u32_hash(n) +#define RSH_EQ(n1,n2) n1 == n2 +#define RSH_FN(n) u32_hash(n) #define RSH_REHASH rte_src_rehash #define RSH_PARAMS /2, *2, 1, 1, 8, 20 -#define RSH_INIT_ORDER 6 - -static HASH(struct rte_src) src_hash; +#define RSH_INIT_ORDER 2 static void rte_src_init(void) { - rte_src_slab = sl_new(rta_pool, sizeof(struct rte_src)); - - idm_init(&src_ids, rta_pool, SRC_ID_INIT_SIZE); + src_domain = DOMAIN_NEW(attrs, "Route sources"); + src_pool = rp_new(&root_pool, &main_birdloop, "Route sources"); + rte_src_slab = sl_new(src_pool, sizeof(struct rte_src)); - HASH_INIT(src_hash, rta_pool, RSH_INIT_ORDER); + idm_init(&src_ids, src_pool, SRC_ID_INIT_SIZE); } - HASH_DEFINE_REHASH_FN(RSH, struct rte_src) -struct rte_src * -rt_find_source(struct proto *p, u32 id) +static struct rte_src * +rt_find_source(struct rte_owner *p, u32 id) { - return HASH_FIND(src_hash, RSH, p, id); + return HASH_FIND(p->hash, RSH, id); } struct rte_src * -rt_get_source(struct proto *p, u32 id) +rt_get_source_o(struct rte_owner *p, u32 id) { + if (p->stop) + bug("Stopping route owner asked for another source."); + struct rte_src *src = rt_find_source(p, id); if (src) + { + UNUSED u64 uc = atomic_fetch_add_explicit(&src->uc, 1, memory_order_acq_rel); return src; + } + SRC_LOCK; src = sl_allocz(rte_src_slab); - src->proto = p; + src->owner = p; src->private_id = id; src->global_id = idm_alloc(&src_ids); - src->uc = 0; - HASH_INSERT2(src_hash, RSH, rta_pool, src); + atomic_store_explicit(&src->uc, 1, memory_order_release); + p->uc++; + + HASH_INSERT2(p->hash, RSH, src_pool, src); + if (config->table_debug) + log(L_TRACE "Allocated new rte_src for %s, ID %uL %uG, have %u sources now", + p->name, src->private_id, src->global_id, p->uc); + + SRC_UNLOCK; return src; } +static inline void +rt_done_sources(struct rte_owner *o) +{ + if (o->stop->list) + ev_send(o->stop->list, o->stop); + else + ev_send(o->list, o->stop); +} + void -rt_prune_sources(void) +rt_prune_sources(void *data) { - HASH_WALK_FILTER(src_hash, next, src, sp) + struct rte_owner *o = data; + + HASH_WALK_FILTER(o->hash, next, src, sp) { - if (src->uc == 0) + u64 uc; + while ((uc = atomic_load_explicit(&src->uc, memory_order_acquire)) >> RTE_SRC_PU_SHIFT) + ; + + if (uc == 0) { - HASH_DO_REMOVE(src_hash, RSH, sp); + o->uc--; + + HASH_DO_REMOVE(o->hash, RSH, sp); + + SRC_LOCK; idm_free(&src_ids, src->global_id); sl_free(rte_src_slab, src); + SRC_UNLOCK; } } HASH_WALK_FILTER_END; - HASH_MAY_RESIZE_DOWN(src_hash, RSH, rta_pool); + SRC_LOCK; + HASH_MAY_RESIZE_DOWN(o->hash, RSH, src_pool); + + if (o->stop && !o->uc) + { + rfree(o->prune); + SRC_UNLOCK; + + if (config->table_debug) + log(L_TRACE "All rte_src's for %s pruned, scheduling stop event", o->name); + + rt_done_sources(o); + } + else + SRC_UNLOCK; } +void +rt_init_sources(struct rte_owner *o, const char *name, event_list *list) +{ + SRC_LOCK; + HASH_INIT(o->hash, src_pool, RSH_INIT_ORDER); + o->hash_key = random_u32(); + o->uc = 0; + o->name = name; + o->prune = ev_new_init(src_pool, rt_prune_sources, o); + o->stop = NULL; + o->list = list; + SRC_UNLOCK; +} + +void +rt_destroy_sources(struct rte_owner *o, event *done) +{ + o->stop = done; + + if (!o->uc) + { + if (config->table_debug) + log(L_TRACE "Source owner %s destroy requested. All rte_src's already pruned, scheduling stop event", o->name); + + SRC_LOCK; + rfree(o->prune); + SRC_UNLOCK; + + rt_done_sources(o); + } + else + if (config->table_debug) + log(L_TRACE "Source owner %s destroy requested. Remaining %u rte_src's to prune.", o->name, o->uc); +} /* * Multipath Next Hop @@ -541,8 +626,8 @@ ea_walk(struct ea_walk_state *s, uint id, uint max) * by calling ea_find() to find the attribute, extracting its value or returning * a provided default if no such attribute is present. */ -int -ea_get_int(ea_list *e, unsigned id, int def) +uintptr_t +ea_get_int(ea_list *e, unsigned id, uintptr_t def) { eattr *a = ea_find(e, id); if (!a) @@ -1081,21 +1166,28 @@ ea_append(ea_list *to, ea_list *what) * rta's */ -static uint rta_cache_count; -static uint rta_cache_size = 32; -static uint rta_cache_limit; -static uint rta_cache_mask; -static rta **rta_hash_table; +static DOMAIN(attrs) attrs_domain; -static void -rta_alloc_hash(void) +#define RTA_LOCK LOCK_DOMAIN(attrs, attrs_domain) +#define RTA_UNLOCK UNLOCK_DOMAIN(attrs, attrs_domain) + +struct rta_cache { + u32 count; + u32 size; + u32 limit; + u32 mask; + rta * _Atomic table[0]; +} * _Atomic rta_cache; +// rta_aux, rta_cache = { .size = ATOMIC_VAR_INIT(32), }; + +static struct rta_cache * +rta_alloc_hash(u32 size) { - rta_hash_table = mb_allocz(rta_pool, sizeof(rta *) * rta_cache_size); - if (rta_cache_size < 32768) - rta_cache_limit = rta_cache_size * 2; - else - rta_cache_limit = ~0; - rta_cache_mask = rta_cache_size - 1; + struct rta_cache *c = mb_allocz(rta_pool, sizeof(struct rta_cache) + sizeof(rta * _Atomic) * size); + c->size = size; + c->limit = (size >> 20) ? (~0U) : (size * 2); + c->mask = size - 1; + return c; } static inline uint @@ -1104,13 +1196,14 @@ rta_hash(rta *a) u64 h; mem_hash_init(&h); #define MIX(f) mem_hash_mix(&h, &(a->f), sizeof(a->f)); - MIX(src); +#define BMIX(f) mem_hash_mix_num(&h, a->f); MIX(hostentry); MIX(from); MIX(igp_metric); - MIX(source); - MIX(scope); - MIX(dest); + BMIX(source); + BMIX(scope); + BMIX(dest); + MIX(pref); #undef MIX return mem_hash_value(&h) ^ nexthop_hash(&(a->nh)) ^ ea_hash(a->eattrs); @@ -1119,8 +1212,7 @@ rta_hash(rta *a) static inline int rta_same(rta *x, rta *y) { - return (x->src == y->src && - x->source == y->source && + return (x->source == y->source && x->scope == y->scope && x->dest == y->dest && x->igp_metric == y->igp_metric && @@ -1149,34 +1241,88 @@ rta_copy(rta *o) } static inline void -rta_insert(rta *r) +rta_insert(rta *r, struct rta_cache *c) { - uint h = r->hash_key & rta_cache_mask; - r->next = rta_hash_table[h]; - if (r->next) - r->next->pprev = &r->next; - r->pprev = &rta_hash_table[h]; - rta_hash_table[h] = r; + uint h = r->hash_key & c->mask; + rta *next = atomic_load_explicit(&c->table[h], memory_order_relaxed); + + atomic_store_explicit(&r->next, next, memory_order_relaxed); + r->pprev = &c->table[h]; + + if (next) + next->pprev = &r->next; + + /* This store MUST be the last and MUST have release order for thread-safety */ + atomic_store_explicit(&c->table[h], r, memory_order_release); } static void -rta_rehash(void) +rta_rehash(struct rta_cache *c) { - uint ohs = rta_cache_size; - uint h; - rta *r, *n; - rta **oht = rta_hash_table; - - rta_cache_size = 2*rta_cache_size; - DBG("Rehashing rta cache from %d to %d entries.\n", ohs, rta_cache_size); - rta_alloc_hash(); - for(h=0; h<ohs; h++) - for(r=oht[h]; r; r=n) + u32 os = c->size; + + struct rta_cache *nc = rta_alloc_hash(os * 2); + nc->count = c->count; + + /* First we simply copy every chain to both new locations */ + for (u32 h = 0; h < os; h++) + { + rta *r = atomic_load_explicit(&c->table[h], memory_order_relaxed); + atomic_store_explicit(&nc->table[h], r, memory_order_relaxed); + atomic_store_explicit(&nc->table[h + os], r, memory_order_relaxed); + } + + /* Then we exchange the hashes; release semantics forces the previous code to be already done */ + atomic_store_explicit(&rta_cache, nc, memory_order_release); + + /* And now we pass through both chains and filter them */ + for (u32 h = 0; h < c->size; h++) + { + rta * _Atomic * ap = &nc->table[h]; + rta * _Atomic * bp = &nc->table[h + os]; + + rta *r = atomic_load_explicit(ap, memory_order_relaxed); + ASSERT_DIE(r == atomic_load_explicit(bp, memory_order_relaxed)); + + while (r) + { + if (r->hash_key & os) { - n = r->next; - rta_insert(r); + r->pprev = bp; + atomic_store_explicit(bp, r, memory_order_release); + bp = &r->next; } - mb_free(oht); + else + { + r->pprev = ap; + atomic_store_explicit(ap, r, memory_order_release); + ap = &r->next; + } + + r = atomic_load_explicit(&r->next, memory_order_acquire); + } + + atomic_store_explicit(ap, NULL, memory_order_release); + atomic_store_explicit(bp, NULL, memory_order_release); + } + + synchronize_rcu(); + mb_free(c); +} + +static rta * +rta_find(rta *o, u32 h, struct rta_cache *c) +{ + rta *r = NULL; + + for (r = atomic_load_explicit(&c->table[h & c->mask], memory_order_acquire); r; r = atomic_load_explicit(&r->next, memory_order_acquire)) + if (r->hash_key == h && rta_same(r, o)) + { + atomic_fetch_add_explicit(&r->uc, 1, memory_order_acq_rel); + return r; + } + + return NULL; } /** @@ -1198,45 +1344,117 @@ rta_lookup(rta *o) rta *r; uint h; - ASSERT(!(o->aflags & RTAF_CACHED)); + ASSERT(!o->cached); if (o->eattrs) ea_normalize(o->eattrs); h = rta_hash(o); - for(r=rta_hash_table[h & rta_cache_mask]; r; r=r->next) - if (r->hash_key == h && rta_same(r, o)) - return rta_clone(r); + /* Lockless lookup */ + rcu_read_lock(); + r = rta_find(o, h, atomic_load_explicit(&rta_cache, memory_order_acquire)); + rcu_read_unlock(); + + if (r) + return r; + + RTA_LOCK; + + /* Locked lookup to avoid duplicates if possible */ + struct rta_cache *c = atomic_load_explicit(&rta_cache, memory_order_acquire); + r = rta_find(o, h, c); + if (r) + { + RTA_UNLOCK; + return r; + } + + /* Store the rta */ r = rta_copy(o); r->hash_key = h; - r->aflags = RTAF_CACHED; - rt_lock_source(r->src); + r->cached = 1; rt_lock_hostentry(r->hostentry); - rta_insert(r); + rta_insert(r, c); - if (++rta_cache_count > rta_cache_limit) - rta_rehash(); + if (++c->count > c->limit) + rta_rehash(c); + RTA_UNLOCK; return r; } -void -rta__free(rta *a) +static void +rta_cleanup(void *data UNUSED) { - ASSERT(rta_cache_count && (a->aflags & RTAF_CACHED)); - rta_cache_count--; - *a->pprev = a->next; - if (a->next) - a->next->pprev = a->pprev; - rt_unlock_hostentry(a->hostentry); - rt_unlock_source(a->src); - if (a->nh.next) - nexthop_free(a->nh.next); - ea_free(a->eattrs); - a->aflags = 0; /* Poison the entry */ - sl_free(rta_slab(a), a); + u32 count = 0; + rta *ax[RTA_OBSOLETE_LIMIT]; + + RTA_LOCK; + struct rta_cache *c = atomic_load_explicit(&rta_cache, memory_order_acquire); + + for(u32 h=0; h<c->size; h++) + for(rta *a = atomic_load_explicit(&c->table[h], memory_order_acquire), *next; + a; + a = next) + { + next = atomic_load_explicit(&a->next, memory_order_acquire); + if (atomic_load_explicit(&a->uc, memory_order_acquire) > 0) + continue; + + /* Check if the cleanup fits in the buffer */ + if (count == RTA_OBSOLETE_LIMIT) + { + ev_send(&global_work_list, &rta_cleanup_event); + goto wait; + } + + /* Relink the forward pointer */ + atomic_store_explicit(a->pprev, next, memory_order_release); + + /* Relink the backwards pointer */ + if (next) + next->pprev = a->pprev; + + /* Store for freeing and go to the next */ + ax[count++] = a; + a = next; + } + +wait: + /* Wait until nobody knows about us */ + synchronize_rcu(); + + u32 freed = 0; + + for (u32 i=0; i<count; i++) + { + rta *a = ax[i]; + /* Acquired inbetween, relink back */ + if (atomic_load_explicit(&a->uc, memory_order_acquire)) + { + rta_insert(a, c); + continue; + } + + /* Cleared to free the memory */ + rt_unlock_hostentry(a->hostentry); + if (a->nh.next) + nexthop_free(a->nh.next); + ea_free(a->eattrs); + a->cached = 0; + c->count--; + sl_free(rta_slab(a), a); + freed++; + } + + atomic_fetch_sub_explicit(&rta_obsolete_count, freed, memory_order_release); + + RTA_UNLOCK; } +_Atomic u32 rta_obsolete_count; +event rta_cleanup_event = { .hook = rta_cleanup, .list = &global_work_list }; + rta * rta_do_cow(rta *o, linpool *lp) { @@ -1248,8 +1466,7 @@ rta_do_cow(rta *o, linpool *lp) memcpy(*nhn, nho, nexthop_size(nho)); nhn = &((*nhn)->next); } - r->aflags = 0; - r->uc = 0; + rta_uncache(r); return r; } @@ -1260,22 +1477,22 @@ rta_do_cow(rta *o, linpool *lp) * This function takes a &rta and dumps its contents to the debug output. */ void -rta_dump(rta *a) +rta_dump(const rta *a) { - static char *rts[] = { "RTS_DUMMY", "RTS_STATIC", "RTS_INHERIT", "RTS_DEVICE", + static char *rts[] = { "", "RTS_STATIC", "RTS_INHERIT", "RTS_DEVICE", "RTS_STAT_DEV", "RTS_REDIR", "RTS_RIP", "RTS_OSPF", "RTS_OSPF_IA", "RTS_OSPF_EXT1", "RTS_OSPF_EXT2", "RTS_BGP", "RTS_PIPE", "RTS_BABEL" }; static char *rtd[] = { "", " DEV", " HOLE", " UNREACH", " PROHIBIT" }; - debug("p=%s uc=%d %s %s%s h=%04x", - a->src->proto->name, a->uc, rts[a->source], ip_scope_text(a->scope), + debug("pref=%d uc=%d %s %s%s h=%04x", + a->pref, a->uc, rts[a->source], ip_scope_text(a->scope), rtd[a->dest], a->hash_key); - if (!(a->aflags & RTAF_CACHED)) + if (!a->cached) debug(" !CACHED"); debug(" <-%I", a->from); if (a->dest == RTD_UNICAST) - for (struct nexthop *nh = &(a->nh); nh; nh = nh->next) + for (const struct nexthop *nh = &(a->nh); nh; nh = nh->next) { if (ipa_nonzero(nh->gw)) debug(" ->%I", nh->gw); if (nh->labels) debug(" L %d", nh->label[0]); @@ -1302,19 +1519,27 @@ rta_dump_all(void) rta *a; uint h; - debug("Route attribute cache (%d entries, rehash at %d):\n", rta_cache_count, rta_cache_limit); - for(h=0; h<rta_cache_size; h++) - for(a=rta_hash_table[h]; a; a=a->next) + RTA_LOCK; + + struct rta_cache *c = atomic_load_explicit(&rta_cache, memory_order_acquire); + + debug("Route attribute cache (%d entries, rehash at %d):\n", c->count, c->limit); + for(h=0; h<c->size; h++) + for(a = atomic_load_explicit(&c->table[h], memory_order_acquire); + a; + a = atomic_load_explicit(&a->next, memory_order_acquire)) { debug("%p ", a); rta_dump(a); debug("\n"); } debug("\n"); + + RTA_UNLOCK; } void -rta_show(struct cli *c, rta *a) +rta_show(struct cli *c, const rta *a) { cli_printf(c, -1008, "\tType: %s %s", rta_src_names[a->source], ip_scope_text(a->scope)); @@ -1332,7 +1557,9 @@ rta_show(struct cli *c, rta *a) void rta_init(void) { - rta_pool = rp_new(&root_pool, "Attributes"); + attrs_domain = DOMAIN_NEW(attrs, "Attributes"); + + rta_pool = rp_new(&root_pool, &main_birdloop, "Attributes"); rta_slab_[0] = sl_new(rta_pool, sizeof(rta)); rta_slab_[1] = sl_new(rta_pool, sizeof(rta) + sizeof(u32)); @@ -1344,7 +1571,7 @@ rta_init(void) nexthop_slab_[2] = sl_new(rta_pool, sizeof(struct nexthop) + sizeof(u32)*2); nexthop_slab_[3] = sl_new(rta_pool, sizeof(struct nexthop) + sizeof(u32)*MPLS_MAX_LABEL_STACK); - rta_alloc_hash(); + atomic_store_explicit(&rta_cache, rta_alloc_hash(32), memory_order_relaxed); rte_src_init(); } diff --git a/nest/rt-dev.c b/nest/rt-dev.c index 61f025ce..c1251675 100644 --- a/nest/rt-dev.c +++ b/nest/rt-dev.c @@ -67,13 +67,11 @@ dev_ifa_notify(struct proto *P, uint flags, struct ifa *ad) /* Use iface ID as local source ID */ struct rte_src *src = rt_get_source(P, ad->iface->index); - rte_update2(c, net, NULL, src); + rte_update(c, net, NULL, src); + rt_unlock_source(src); } else if (flags & IF_CHANGE_UP) { - rta *a; - rte *e; - DBG("dev_if_notify: %s:%I going up\n", ad->iface->name, ad->ip); if (cf->check_link && !(ad->iface->flags & IF_LINK_UP)) @@ -83,17 +81,20 @@ dev_ifa_notify(struct proto *P, uint flags, struct ifa *ad) struct rte_src *src = rt_get_source(P, ad->iface->index); rta a0 = { - .src = src, + .pref = c->preference, .source = RTS_DEVICE, .scope = SCOPE_UNIVERSE, .dest = RTD_UNICAST, .nh.iface = ad->iface, }; - a = rta_lookup(&a0); - e = rte_get_temp(a); - e->pflags = 0; - rte_update2(c, net, e, src); + rte e0 = { + .attrs = rta_lookup(&a0), + .src = src, + }; + + rte_update(c, net, &e0, src); + rt_unlock_source(src); } } diff --git a/nest/rt-show.c b/nest/rt-show.c index 7691878d..65b59af4 100644 --- a/nest/rt-show.c +++ b/nest/rt-show.c @@ -56,17 +56,17 @@ rt_show_rte(struct cli *c, byte *ia, rte *e, struct rt_show_data *d, int primary if (d->verbose && !rta_is_cached(a) && a->eattrs) ea_normalize(a->eattrs); - get_route_info = a->src->proto->proto->get_route_info; + get_route_info = e->src->owner->class ? e->src->owner->class->get_route_info : NULL; if (get_route_info) get_route_info(e, info); else - bsprintf(info, " (%d)", e->pref); + bsprintf(info, " (%d)", a->pref); if (d->last_table != d->tab) rt_show_table(c, d); cli_printf(c, -1007, "%-20s %s [%s %s%s]%s%s", ia, rta_dest_name(a->dest), - a->src->proto->name, tm, from, primary ? (sync_error ? " !" : " *") : "", info); + e->src->owner->name, tm, from, primary ? (sync_error ? " !" : " *") : "", info); if (a->dest == RTD_UNICAST) for (nh = &(a->nh); nh; nh = nh->next) @@ -95,13 +95,38 @@ rt_show_rte(struct cli *c, byte *ia, rte *e, struct rt_show_data *d, int primary } if (d->verbose) + { + cli_printf(c, -1008, "\tInternal route ID: %uL %uG %uS", e->src->private_id, e->src->global_id, e->stale_cycle); rta_show(c, a); + } +} + +static uint +rte_feed_count(net *n) +{ + uint count = 0; + for (struct rte_storage *e = n->routes; e; e = e->next) + if (rte_is_valid(RTES_OR_NULL(e))) + count++; + return count; +} + +static void +rte_feed_obtain(net *n, rte **feed, uint count) +{ + uint i = 0; + for (struct rte_storage *e = n->routes; e; e = e->next) + if (rte_is_valid(RTES_OR_NULL(e))) + { + ASSERT_DIE(i < count); + feed[i++] = &e->rte; + } + ASSERT_DIE(i == count); } static void rt_show_net(struct cli *c, net *n, struct rt_show_data *d) { - rte *e, *ee; byte ia[NET_MAX_TEXT_LENGTH+1]; struct channel *ec = d->tab->export_channel; @@ -114,9 +139,9 @@ rt_show_net(struct cli *c, net *n, struct rt_show_data *d) bsnprintf(ia, sizeof(ia), "%N", n->n.addr); - for (e = n->routes; e; e = e->next) + for (struct rte_storage *er = n->routes; er; er = er->next) { - if (rte_is_filtered(e) != d->filtered) + if (rte_is_filtered(&er->rte) != d->filtered) continue; d->rt_counter++; @@ -126,16 +151,15 @@ rt_show_net(struct cli *c, net *n, struct rt_show_data *d) if (pass) continue; - ee = e; - rte_make_tmp_attrs(&e, c->show_pool, NULL); + struct rte e = er->rte; /* Export channel is down, do not try to export routes to it */ - if (ec && (ec->export_state == ES_DOWN)) + if (ec && !ec->out_req.hook) goto skip; if (d->export_mode == RSEM_EXPORTED) { - if (!bmap_test(&ec->export_map, ee->id)) + if (!bmap_test(&ec->export_map, e.id)) goto skip; // if (ec->ra_mode != RA_ANY) @@ -144,17 +168,24 @@ rt_show_net(struct cli *c, net *n, struct rt_show_data *d) else if ((d->export_mode == RSEM_EXPORT) && (ec->ra_mode == RA_MERGED)) { /* Special case for merged export */ - rte *rt_free; - e = rt_export_merged(ec, n, &rt_free, c->show_pool, 1); pass = 1; + uint count = rte_feed_count(n); + if (!count) + goto skip; - if (!e) - { e = ee; goto skip; } + rte **feed = alloca(count * sizeof(rte *)); + rte_feed_obtain(n, feed, count); + rte *em = rt_export_merged(ec, feed, count, c->show_pool, 1); + + if (em) + e = *em; + else + goto skip; } else if (d->export_mode) { struct proto *ep = ec->proto; - int ic = ep->preexport ? ep->preexport(ep, &e, c->show_pool) : 0; + int ic = ep->preexport ? ep->preexport(ec, &e) : 0; if (ec->ra_mode == RA_OPTIMAL || ec->ra_mode == RA_MERGED) pass = 1; @@ -180,24 +211,19 @@ rt_show_net(struct cli *c, net *n, struct rt_show_data *d) } } - if (d->show_protocol && (d->show_protocol != e->attrs->src->proto)) + if (d->show_protocol && (&d->show_protocol->sources != e.src->owner)) goto skip; if (f_run(d->filter, &e, c->show_pool, 0) > F_ACCEPT) goto skip; if (d->stats < 2) - rt_show_rte(c, ia, e, d, (e->net->routes == ee)); + rt_show_rte(c, ia, &e, d, (n->routes == er)); d->show_counter++; ia[0] = 0; skip: - if (e != ee) - { - rte_free(e); - e = ee; - } lp_flush(c->show_pool); if (d->primary_only) @@ -213,11 +239,13 @@ rt_show_cleanup(struct cli *c) /* Unlink the iterator */ if (d->table_open) - fit_get(&d->tab->table->fib, &d->fit); + RT_LOCKED(d->tab->table, t) + fit_get(&t->fib, &d->fit); /* Unlock referenced tables */ WALK_LIST(tab, d->tables) - rt_unlock_table(tab->table); + RT_LOCKED(tab->table, t) + rt_unlock_table(t); } static void @@ -229,8 +257,6 @@ rt_show_cont(struct cli *c) #else unsigned max = 64; #endif - struct fib *fib = &d->tab->table->fib; - struct fib_iterator *it = &d->fit; if (d->running_on_config && (d->running_on_config != config)) { @@ -238,9 +264,14 @@ rt_show_cont(struct cli *c) goto done; } + rtable_private *t = RT_LOCK(d->tab->table); + + struct fib *fib = &t->fib; + struct fib_iterator *it = &d->fit; + if (!d->table_open) { - FIB_ITERATE_INIT(&d->fit, &d->tab->table->fib); + FIB_ITERATE_INIT(&d->fit, fib); d->table_open = 1; d->table_counter++; d->kernel = rt_show_get_kernel(d); @@ -258,6 +289,7 @@ rt_show_cont(struct cli *c) if (!max--) { FIB_ITERATE_PUT(it); + RT_UNLOCK(d->tab->table); return; } rt_show_net(c, n, d); @@ -274,6 +306,8 @@ rt_show_cont(struct cli *c) d->net_counter - d->net_counter_last, d->tab->table->name); } + RT_UNLOCK(d->tab->table); + d->kernel = NULL; d->table_open = 0; d->tab = NODE_NEXT(d->tab); @@ -322,7 +356,7 @@ rt_show_get_default_tables(struct rt_show_data *d) { WALK_LIST(c, d->export_protocol->channels) { - if (c->export_state == ES_DOWN) + if (!c->out_req.hook) continue; tab = rt_show_add_table(d, c->table); @@ -339,7 +373,7 @@ rt_show_get_default_tables(struct rt_show_data *d) } for (int i=1; i<NET_MAX; i++) - if (config->def_tables[i]) + if (config->def_tables[i] && config->def_tables[i]->table) rt_show_add_table(d, config->def_tables[i]->table); } @@ -405,7 +439,8 @@ rt_show(struct rt_show_data *d) if (!d->addr) { WALK_LIST(tab, d->tables) - rt_lock_table(tab->table); + RT_LOCKED(tab->table, t) + rt_lock_table(t); /* There is at least one table */ d->tab = HEAD(d->tables); @@ -420,13 +455,17 @@ rt_show(struct rt_show_data *d) d->tab = tab; d->kernel = rt_show_get_kernel(d); + RT_LOCK(tab->table); + if (d->show_for) - n = net_route(tab->table, d->addr); + n = net_route(RT_PRIV(tab->table), d->addr); else - n = net_find(tab->table, d->addr); + n = net_find(RT_PRIV(tab->table), d->addr); if (n) rt_show_net(this_cli, n, d); + + RT_UNLOCK(tab->table); } if (d->rt_counter) diff --git a/nest/rt-table.c b/nest/rt-table.c index 390b3277..5f1e1679 100644 --- a/nest/rt-table.c +++ b/nest/rt-table.c @@ -45,28 +45,85 @@ #include "lib/string.h" #include "lib/alloca.h" -#ifdef CONFIG_BGP -#include "proto/bgp/bgp.h" -#endif +#include <stdatomic.h> pool *rt_table_pool; -static slab *rte_slab; -static linpool *rte_update_pool; - list routing_tables; -static void rt_free_hostcache(rtable *tab); -static void rt_notify_hostcache(rtable *tab, net *net); -static void rt_update_hostcache(rtable *tab); -static void rt_next_hop_update(rtable *tab); -static inline void rt_prune_table(rtable *tab); -static inline void rt_schedule_notify(rtable *tab); +/* Data structures for export journal */ +#define RT_PENDING_EXPORT_ITEMS (page_size - sizeof(struct rt_export_block)) / sizeof(struct rt_pending_export) +struct rt_export_block { + node n; + _Atomic u32 end; + _Atomic _Bool not_last; + struct rt_pending_export export[]; +}; + +static void rt_free_hostcache(rtable_private *tab); +static void rt_notify_hostcache(rtable_private *tab, net *net); +static void rt_update_hostcache(void *tab); +static void rt_next_hop_update(void *tab); +static inline void rt_prune_table(void *tab); +static void rt_fast_prune_check(rtable_private *tab); +static inline void rt_schedule_notify(rtable_private *tab); +static void rt_feed_channel(void *); + +static inline void rt_export_used(rtable_private *tab); +static void rt_export_cleanup(void *tab); + +const char *rt_import_state_name_array[TIS_MAX] = { + [TIS_DOWN] = "DOWN", + [TIS_UP] = "UP", + [TIS_STOP] = "STOP", + [TIS_FLUSHING] = "FLUSHING", + [TIS_WAITING] = "WAITING", + [TIS_CLEARED] = "CLEARED", +}; + +const char *rt_export_state_name_array[TES_MAX] = { + [TES_DOWN] = "DOWN", + [TES_HUNGRY] = "HUNGRY", + [TES_FEEDING] = "FEEDING", + [TES_READY] = "READY", + [TES_STOP] = "STOP" +}; + +const char *rt_import_state_name(u8 state) +{ + if (state >= TIS_MAX) + return "!! INVALID !!"; + else + return rt_import_state_name_array[state]; +} + +const char *rt_export_state_name(u8 state) +{ + if (state >= TES_MAX) + return "!! INVALID !!"; + else + return rt_export_state_name_array[state]; +} + +struct event_cork rt_cork; + +static inline void +rte_update_lock(struct channel *c) +{ + c->rte_update_nest_cnt++; +} + +static inline void +rte_update_unlock(struct channel *c) +{ + if (!--c->rte_update_nest_cnt) + lp_flush(c->rte_update_pool); +} /* Like fib_route(), but skips empty net entries */ static inline void * -net_route_ip4(rtable *t, net_addr_ip4 *n) +net_route_ip4(rtable_private *t, net_addr_ip4 *n) { net *r; @@ -80,7 +137,7 @@ net_route_ip4(rtable *t, net_addr_ip4 *n) } static inline void * -net_route_ip6(rtable *t, net_addr_ip6 *n) +net_route_ip6(rtable_private *t, net_addr_ip6 *n) { net *r; @@ -94,7 +151,7 @@ net_route_ip6(rtable *t, net_addr_ip6 *n) } static inline void * -net_route_ip6_sadr(rtable *t, net_addr_ip6_sadr *n) +net_route_ip6_sadr(rtable_private *t, net_addr_ip6_sadr *n) { struct fib_node *fn; @@ -133,7 +190,7 @@ net_route_ip6_sadr(rtable *t, net_addr_ip6_sadr *n) } void * -net_route(rtable *tab, const net_addr *n) +net_route(rtable_private *tab, const net_addr *n) { ASSERT(tab->addr_type == n->type); @@ -162,12 +219,15 @@ net_route(rtable *tab, const net_addr *n) static int -net_roa_check_ip4(rtable *tab, const net_addr_ip4 *px, u32 asn) +net_roa_check_ip4(rtable *t, const net_addr_ip4 *px, u32 asn) { struct net_addr_roa4 n = NET_ADDR_ROA4(px->prefix, px->pxlen, 0, 0); struct fib_node *fn; int anything = 0; + RT_LOCK(t); + rtable_private *tab = RT_PRIV(t); + while (1) { for (fn = fib_get_chain(&tab->fib, (net_addr *) &n); fn; fn = fn->next) @@ -175,11 +235,14 @@ net_roa_check_ip4(rtable *tab, const net_addr_ip4 *px, u32 asn) net_addr_roa4 *roa = (void *) fn->addr; net *r = fib_node_to_user(&tab->fib, fn); - if (net_equal_prefix_roa4(roa, &n) && rte_is_valid(r->routes)) + if (net_equal_prefix_roa4(roa, &n) && r->routes && rte_is_valid(&r->routes->rte)) { anything = 1; if (asn && (roa->asn == asn) && (roa->max_pxlen >= px->pxlen)) + { + RT_UNLOCK(tab); return ROA_VALID; + } } } @@ -190,16 +253,20 @@ net_roa_check_ip4(rtable *tab, const net_addr_ip4 *px, u32 asn) ip4_clrbit(&n.prefix, n.pxlen); } + RT_UNLOCK(tab); return anything ? ROA_INVALID : ROA_UNKNOWN; } static int -net_roa_check_ip6(rtable *tab, const net_addr_ip6 *px, u32 asn) +net_roa_check_ip6(rtable *t, const net_addr_ip6 *px, u32 asn) { struct net_addr_roa6 n = NET_ADDR_ROA6(px->prefix, px->pxlen, 0, 0); struct fib_node *fn; int anything = 0; + RT_LOCK(t); + rtable_private *tab = RT_PRIV(t); + while (1) { for (fn = fib_get_chain(&tab->fib, (net_addr *) &n); fn; fn = fn->next) @@ -207,11 +274,14 @@ net_roa_check_ip6(rtable *tab, const net_addr_ip6 *px, u32 asn) net_addr_roa6 *roa = (void *) fn->addr; net *r = fib_node_to_user(&tab->fib, fn); - if (net_equal_prefix_roa6(roa, &n) && rte_is_valid(r->routes)) + if (net_equal_prefix_roa6(roa, &n) && r->routes && rte_is_valid(&r->routes->rte)) { anything = 1; if (asn && (roa->asn == asn) && (roa->max_pxlen >= px->pxlen)) + { + RT_UNLOCK(tab); return ROA_VALID; + } } } @@ -222,6 +292,7 @@ net_roa_check_ip6(rtable *tab, const net_addr_ip6 *px, u32 asn) ip6_clrbit(&n.prefix, n.pxlen); } + RT_UNLOCK(tab); return anything ? ROA_INVALID : ROA_UNKNOWN; } @@ -256,253 +327,50 @@ net_roa_check(rtable *tab, const net_addr *n, u32 asn) * @net: network node * @src: route source * - * The rte_find() function returns a route for destination @net - * which is from route source @src. + * The rte_find() function returns a pointer to a route for destination @net + * which is from route source @src. List end pointer is returned if no route is found. */ -rte * +static struct rte_storage ** rte_find(net *net, struct rte_src *src) { - rte *e = net->routes; - - while (e && e->attrs->src != src) - e = e->next; - return e; -} - -/** - * rte_get_temp - get a temporary &rte - * @a: attributes to assign to the new route (a &rta; in case it's - * un-cached, rte_update() will create a cached copy automatically) - * - * Create a temporary &rte and bind it with the attributes @a. - * Also set route preference to the default preference set for - * the protocol. - */ -rte * -rte_get_temp(rta *a) -{ - rte *e = sl_alloc(rte_slab); + struct rte_storage **e = &net->routes; - e->attrs = a; - e->id = 0; - e->flags = 0; - e->pref = 0; - return e; -} + while ((*e) && (*e)->rte.src != src) + e = &(*e)->next; -rte * -rte_do_cow(rte *r) -{ - rte *e = sl_alloc(rte_slab); - - memcpy(e, r, sizeof(rte)); - e->attrs = rta_clone(r->attrs); - e->flags = 0; return e; } -/** - * rte_cow_rta - get a private writable copy of &rte with writable &rta - * @r: a route entry to be copied - * @lp: a linpool from which to allocate &rta - * - * rte_cow_rta() takes a &rte and prepares it and associated &rta for - * modification. There are three possibilities: First, both &rte and &rta are - * private copies, in that case they are returned unchanged. Second, &rte is - * private copy, but &rta is cached, in that case &rta is duplicated using - * rta_do_cow(). Third, both &rte is shared and &rta is cached, in that case - * both structures are duplicated by rte_do_cow() and rta_do_cow(). - * - * Note that in the second case, cached &rta loses one reference, while private - * copy created by rta_do_cow() is a shallow copy sharing indirect data (eattrs, - * nexthops, ...) with it. To work properly, original shared &rta should have - * another reference during the life of created private copy. - * - * Result: a pointer to the new writable &rte with writable &rta. - */ -rte * -rte_cow_rta(rte *r, linpool *lp) -{ - if (!rta_is_cached(r->attrs)) - return r; - - r = rte_cow(r); - rta *a = rta_do_cow(r->attrs, lp); - rta_free(r->attrs); - r->attrs = a; - return r; -} - -/** - * rte_init_tmp_attrs - initialize temporary ea_list for route - * @r: route entry to be modified - * @lp: linpool from which to allocate attributes - * @max: maximum number of added temporary attribus - * - * This function is supposed to be called from make_tmp_attrs() and - * store_tmp_attrs() hooks before rte_make_tmp_attr() / rte_store_tmp_attr() - * functions. It allocates &ea_list with length for @max items for temporary - * attributes and puts it on top of eattrs stack. - */ -void -rte_init_tmp_attrs(rte *r, linpool *lp, uint max) +static struct rte_storage * +rte_store(const rte *r, net *net, rtable_private *tab) { - struct ea_list *e = lp_alloc(lp, sizeof(struct ea_list) + max * sizeof(eattr)); + struct rte_storage *e = sl_alloc(tab->rte_slab); - e->next = r->attrs->eattrs; - e->flags = EALF_SORTED | EALF_TEMP; - e->count = 0; + e->rte = *r; + e->rte.net = net->n.addr; - r->attrs->eattrs = e; -} + rt_lock_source(e->rte.src); -/** - * rte_make_tmp_attr - make temporary eattr from private route fields - * @r: route entry to be modified - * @id: attribute ID - * @type: attribute type - * @val: attribute value (u32 or adata ptr) - * - * This function is supposed to be called from make_tmp_attrs() hook for - * each temporary attribute, after temporary &ea_list was initialized by - * rte_init_tmp_attrs(). It checks whether temporary attribute is supposed to - * be defined (based on route pflags) and if so then it fills &eattr field in - * preallocated temporary &ea_list on top of route @r eattrs stack. - * - * Note that it may require free &eattr in temporary &ea_list, so it must not be - * called more times than @max argument of rte_init_tmp_attrs(). - */ -void -rte_make_tmp_attr(rte *r, uint id, uint type, uintptr_t val) -{ - if (r->pflags & EA_ID_FLAG(id)) - { - ea_list *e = r->attrs->eattrs; - eattr *a = &e->attrs[e->count++]; - a->id = id; - a->type = type; - a->flags = 0; - - if (type & EAF_EMBEDDED) - a->u.data = (u32) val; - else - a->u.ptr = (struct adata *) val; - } + return e; } /** - * rte_store_tmp_attr - store temporary eattr to private route fields - * @r: route entry to be modified - * @id: attribute ID - * - * This function is supposed to be called from store_tmp_attrs() hook for - * each temporary attribute, after temporary &ea_list was initialized by - * rte_init_tmp_attrs(). It checks whether temporary attribute is defined in - * route @r eattrs stack, updates route pflags accordingly, undefines it by - * filling &eattr field in preallocated temporary &ea_list on top of the eattrs - * stack, and returns the value. Caller is supposed to store it in the - * appropriate private field. + * rte_free - delete a &rte + * @e: &struct rte_storage to be deleted + * @tab: the table which the rte belongs to * - * Note that it may require free &eattr in temporary &ea_list, so it must not be - * called more times than @max argument of rte_init_tmp_attrs() + * rte_free() deletes the given &rte from the routing table it's linked to. */ -uintptr_t -rte_store_tmp_attr(rte *r, uint id) -{ - ea_list *e = r->attrs->eattrs; - eattr *a = ea_find(e->next, id); - if (a) - { - e->attrs[e->count++] = (struct eattr) { .id = id, .type = EAF_TYPE_UNDEF }; - r->pflags |= EA_ID_FLAG(id); - return (a->type & EAF_EMBEDDED) ? a->u.data : (uintptr_t) a->u.ptr; - } - else - { - r->pflags &= ~EA_ID_FLAG(id); - return 0; - } -} - -/** - * rte_make_tmp_attrs - prepare route by adding all relevant temporary route attributes - * @r: route entry to be modified (may be replaced if COW) - * @lp: linpool from which to allocate attributes - * @old_attrs: temporary ref to old &rta (may be NULL) - * - * This function expands privately stored protocol-dependent route attributes - * to a uniform &eattr / &ea_list representation. It is essentially a wrapper - * around protocol make_tmp_attrs() hook, which does some additional work like - * ensuring that route @r is writable. - * - * The route @r may be read-only (with %REF_COW flag), in that case rw copy is - * obtained by rte_cow() and @r is replaced. If @rte is originally rw, it may be - * directly modified (and it is never copied). - * - * If the @old_attrs ptr is supplied, the function obtains another reference of - * old cached &rta, that is necessary in some cases (see rte_cow_rta() for - * details). It is freed by rte_store_tmp_attrs(), or manually by rta_free(). - * - * Generally, if caller ensures that @r is read-only (e.g. in route export) then - * it may ignore @old_attrs (and set it to NULL), but must handle replacement of - * @r. If caller ensures that @r is writable (e.g. in route import) then it may - * ignore replacement of @r, but it must handle @old_attrs. - */ void -rte_make_tmp_attrs(rte **r, linpool *lp, rta **old_attrs) +rte_free(struct rte_storage *e, rtable_private *tab) { - void (*make_tmp_attrs)(rte *r, linpool *lp); - make_tmp_attrs = (*r)->attrs->src->proto->make_tmp_attrs; - - if (!make_tmp_attrs) - return; - - /* We may need to keep ref to old attributes, will be freed in rte_store_tmp_attrs() */ - if (old_attrs) - *old_attrs = rta_is_cached((*r)->attrs) ? rta_clone((*r)->attrs) : NULL; - - *r = rte_cow_rta(*r, lp); - make_tmp_attrs(*r, lp); + rt_unlock_source(e->rte.src); + rta_free(e->rte.attrs); + sl_free(tab->rte_slab, e); } -/** - * rte_store_tmp_attrs - store temporary route attributes back to private route fields - * @r: route entry to be modified - * @lp: linpool from which to allocate attributes - * @old_attrs: temporary ref to old &rta - * - * This function stores temporary route attributes that were expanded by - * rte_make_tmp_attrs() back to private route fields and also undefines them. - * It is essentially a wrapper around protocol store_tmp_attrs() hook, which - * does some additional work like shortcut if there is no change and cleanup - * of @old_attrs reference obtained by rte_make_tmp_attrs(). - */ -static void -rte_store_tmp_attrs(rte *r, linpool *lp, rta *old_attrs) -{ - void (*store_tmp_attrs)(rte *rt, linpool *lp); - store_tmp_attrs = r->attrs->src->proto->store_tmp_attrs; - - if (!store_tmp_attrs) - return; - - ASSERT(!rta_is_cached(r->attrs)); - - /* If there is no new ea_list, we just skip the temporary ea_list */ - ea_list *ea = r->attrs->eattrs; - if (ea && (ea->flags & EALF_TEMP)) - r->attrs->eattrs = ea->next; - else - store_tmp_attrs(r, lp); - - /* Free ref we got in rte_make_tmp_attrs(), have to do rta_lookup() first */ - r->attrs = rta_lookup(r->attrs); - rta_free(old_attrs); -} - - static int /* Actually better or at least as good as */ rte_better(rte *new, rte *old) { @@ -513,20 +381,20 @@ rte_better(rte *new, rte *old) if (!rte_is_valid(new)) return 0; - if (new->pref > old->pref) + if (new->attrs->pref > old->attrs->pref) return 1; - if (new->pref < old->pref) + if (new->attrs->pref < old->attrs->pref) return 0; - if (new->attrs->src->proto->proto != old->attrs->src->proto->proto) + if (new->src->owner->class != old->src->owner->class) { /* * If the user has configured protocol preferences, so that two different protocols * have the same preference, try to break the tie by comparing addresses. Not too * useful, but keeps the ordering of routes unambiguous. */ - return new->attrs->src->proto->proto > old->attrs->src->proto->proto; + return new->src->owner->class > old->src->owner->class; } - if (better = new->attrs->src->proto->rte_better) + if (better = new->src->owner->class->rte_better) return better(new, old); return 0; } @@ -539,180 +407,209 @@ rte_mergable(rte *pri, rte *sec) if (!rte_is_valid(pri) || !rte_is_valid(sec)) return 0; - if (pri->pref != sec->pref) + if (pri->attrs->pref != sec->attrs->pref) return 0; - if (pri->attrs->src->proto->proto != sec->attrs->src->proto->proto) + if (pri->src->owner->class != sec->src->owner->class) return 0; - if (mergable = pri->attrs->src->proto->rte_mergable) + if (mergable = pri->src->owner->class->rte_mergable) return mergable(pri, sec); return 0; } static void -rte_trace(struct channel *c, rte *e, int dir, char *msg) +rte_trace(const char *name, const rte *e, int dir, const char *msg) { - log(L_TRACE "%s.%s %c %s %N %s", - c->proto->name, c->name ?: "?", dir, msg, e->net->n.addr, - rta_dest_name(e->attrs->dest)); + log(L_TRACE "%s %c %s %N src %uL %uG %uS id %u %s%s", + name, dir, msg, e->net, + e->src->private_id, e->src->global_id, e->stale_cycle, e->id, + rta_dest_name(e->attrs->dest), + rte_is_filtered(e) ? " (filtered)" : ""); } static inline void -rte_trace_in(uint flag, struct channel *c, rte *e, char *msg) +channel_rte_trace_in(uint flag, struct channel *c, const rte *e, const char *msg) { if ((c->debug & flag) || (c->proto->debug & flag)) - rte_trace(c, e, '>', msg); + rte_trace(c->in_req.name, e, '>', msg); } static inline void -rte_trace_out(uint flag, struct channel *c, rte *e, char *msg) +channel_rte_trace_out(uint flag, struct channel *c, const rte *e, const char *msg) { if ((c->debug & flag) || (c->proto->debug & flag)) - rte_trace(c, e, '<', msg); + rte_trace(c->out_req.name, e, '<', msg); +} + +static inline void +rt_rte_trace_in(uint flag, struct rt_import_request *req, const rte *e, const char *msg) +{ + if (req->trace_routes & flag) + rte_trace(req->name, e, '>', msg); +} + +#if 0 +// seems to be unused at all +static inline void +rt_rte_trace_out(uint flag, struct rt_export_request *req, const rte *e, const char *msg) +{ + if (req->trace_routes & flag) + rte_trace(req->name, e, '<', msg); +} +#endif + +static uint +rte_feed_count(net *n) +{ + uint count = 0; + for (struct rte_storage *e = n->routes; e; e = e->next) + if (rte_is_valid(RTES_OR_NULL(e))) + count++; + return count; +} + +static void +rte_feed_obtain(net *n, struct rte **feed, uint count) +{ + uint i = 0; + for (struct rte_storage *e = n->routes; e; e = e->next) + if (rte_is_valid(RTES_OR_NULL(e))) + { + ASSERT_DIE(i < count); + feed[i++] = &e->rte; + } + ASSERT_DIE(i == count); } static rte * -export_filter_(struct channel *c, rte *rt0, rte **rt_free, linpool *pool, int silent) +export_filter_(struct channel *c, rte *rt, linpool *pool, int silent) { struct proto *p = c->proto; const struct filter *filter = c->out_filter; - struct proto_stats *stats = &c->stats; - rte *rt; - int v; + struct channel_export_stats *stats = &c->export_stats; - rt = rt0; - *rt_free = NULL; + /* Do nothing if we have already rejected the route */ + if (silent && bmap_test(&c->export_reject_map, rt->id)) + goto reject_noset; - v = p->preexport ? p->preexport(p, &rt, pool) : 0; + int v = p->preexport ? p->preexport(c, rt) : 0; if (v < 0) { if (silent) - goto reject; + goto reject_noset; - stats->exp_updates_rejected++; + stats->updates_rejected++; if (v == RIC_REJECT) - rte_trace_out(D_FILTERS, c, rt, "rejected by protocol"); + channel_rte_trace_out(D_FILTERS, c, rt, "rejected by protocol"); goto reject; + } if (v > 0) { if (!silent) - rte_trace_out(D_FILTERS, c, rt, "forced accept by protocol"); + channel_rte_trace_out(D_FILTERS, c, rt, "forced accept by protocol"); goto accept; } - rte_make_tmp_attrs(&rt, pool, NULL); - v = filter && ((filter == FILTER_REJECT) || - (f_run(filter, &rt, pool, + (f_run(filter, rt, pool, (silent ? FF_SILENT : 0)) > F_ACCEPT)); if (v) { if (silent) goto reject; - stats->exp_updates_filtered++; - rte_trace_out(D_FILTERS, c, rt, "filtered out"); + stats->updates_filtered++; + channel_rte_trace_out(D_FILTERS, c, rt, "filtered out"); goto reject; } -#ifdef CONFIG_PIPE - /* Pipes need rte with stored tmpattrs, remaining protocols need expanded tmpattrs */ - if (p->proto == &proto_pipe) - rte_store_tmp_attrs(rt, pool, NULL); -#endif - accept: - if (rt != rt0) - *rt_free = rt; + /* We have accepted the route */ + bmap_clear(&c->export_reject_map, rt->id); return rt; reject: + /* We have rejected the route by filter */ + bmap_set(&c->export_reject_map, rt->id); + +reject_noset: /* Discard temporary rte */ - if (rt != rt0) - rte_free(rt); return NULL; } static inline rte * -export_filter(struct channel *c, rte *rt0, rte **rt_free, int silent) +export_filter(struct channel *c, rte *rt, int silent) { - return export_filter_(c, rt0, rt_free, rte_update_pool, silent); + return export_filter_(c, rt, c->rte_update_pool, silent); } +void do_rt_notify_direct(struct channel *c, const net_addr *net, rte *new, const rte *old); + static void -do_rt_notify(struct channel *c, net *net, rte *new, rte *old, int refeed) +do_rt_notify(struct channel *c, const net_addr *net, rte *new, const rte *old) { - struct proto *p = c->proto; - struct proto_stats *stats = &c->stats; + struct channel_export_stats *stats = &c->export_stats; - if (refeed && new) + if (c->refeeding && new) c->refeed_count++; - /* Apply export limit */ - struct channel_limit *l = &c->out_limit; - if (l->action && !old && new) - { - if (stats->exp_routes >= l->limit) - channel_notify_limit(c, l, PLD_OUT, stats->exp_routes); - - if (l->state == PLS_BLOCKED) + if (!old && new) + if (CHANNEL_LIMIT_PUSH(c, OUT)) { - stats->exp_updates_rejected++; - rte_trace_out(D_FILTERS, c, new, "rejected [limit]"); + stats->updates_rejected++; + channel_rte_trace_out(D_FILTERS, c, new, "rejected [limit]"); return; } - } - /* Apply export table */ - if (c->out_table && !rte_update_out(c, net->n.addr, new, old, refeed)) - return; - - if (new) - stats->exp_updates_accepted++; - else - stats->exp_withdraws_accepted++; + if (!new && old) + CHANNEL_LIMIT_POP(c, OUT); + /* Store route export state */ if (old) - { bmap_clear(&c->export_map, old->id); - stats->exp_routes--; - } if (new) - { bmap_set(&c->export_map, new->id); - stats->exp_routes++; - } + + /* Apply export table */ + if (c->out_table) + rte_import(&c->out_table->push, net, new, old ? old->src : new->src); + else + do_rt_notify_direct(c, net, new, old); +} + +void +do_rt_notify_direct(struct channel *c, const net_addr *net, rte *new, const rte *old) +{ + struct proto *p = c->proto; + struct channel_export_stats *stats = &c->export_stats; + + if (new) + stats->updates_accepted++; + else + stats->withdraws_accepted++; if (p->debug & D_ROUTES) { if (new && old) - rte_trace_out(D_ROUTES, c, new, "replaced"); + channel_rte_trace_out(D_ROUTES, c, new, "replaced"); else if (new) - rte_trace_out(D_ROUTES, c, new, "added"); + channel_rte_trace_out(D_ROUTES, c, new, "added"); else if (old) - rte_trace_out(D_ROUTES, c, old, "removed"); + channel_rte_trace_out(D_ROUTES, c, old, "removed"); } p->rt_notify(p, c, net, new, old); } static void -rt_notify_basic(struct channel *c, net *net, rte *new, rte *old, int refeed) +rt_notify_basic(struct channel *c, const net_addr *net, rte *new, rte *old) { - // struct proto *p = c->proto; - rte *new_free = NULL; - - if (new) - c->stats.exp_updates_received++; - else - c->stats.exp_withdraws_received++; - if (new) - new = export_filter(c, new, &new_free, 0); + new = export_filter(c, new, 0); if (old && !bmap_test(&c->export_map, old->id)) old = NULL; @@ -720,87 +617,87 @@ rt_notify_basic(struct channel *c, net *net, rte *new, rte *old, int refeed) if (!new && !old) return; - do_rt_notify(c, net, new, old, refeed); - - /* Discard temporary rte */ - if (new_free) - rte_free(new_free); + do_rt_notify(c, net, new, old); } static void -rt_notify_accepted(struct channel *c, net *net, rte *new_changed, rte *old_changed, int refeed) +channel_rpe_mark_seen(struct rt_export_request *req, struct rt_pending_export *rpe) { - // struct proto *p = c->proto; - rte *new_best = NULL; - rte *old_best = NULL; - rte *new_free = NULL; - int new_first = 0; - - /* - * We assume that there are no changes in net route order except (added) - * new_changed and (removed) old_changed. Therefore, the function is not - * compatible with deterministic_med (where nontrivial reordering can happen - * as a result of a route change) and with recomputation of recursive routes - * due to next hop update (where many routes can be changed in one step). - * - * Note that we need this assumption just for optimizations, we could just - * run full new_best recomputation otherwise. - * - * There are three cases: - * feed or old_best is old_changed -> we need to recompute new_best - * old_best is before new_changed -> new_best is old_best, ignore - * old_best is after new_changed -> try new_changed, otherwise old_best - */ + struct channel *c = SKIP_BACK(struct channel, out_req, req); - if (net->routes) - c->stats.exp_updates_received++; - else - c->stats.exp_withdraws_received++; + rpe_mark_seen(req->hook, rpe); + if (rpe->old) + bmap_clear(&c->export_reject_map, rpe->old->rte.id); +} - /* Find old_best - either old_changed, or route for net->routes */ - if (old_changed && bmap_test(&c->export_map, old_changed->id)) - old_best = old_changed; - else +void +rt_notify_accepted(struct rt_export_request *req, const net_addr *n, struct rt_pending_export *rpe, + struct rte **feed, uint count) +{ + struct channel *c = SKIP_BACK(struct channel, out_req, req); + + rte_update_lock(c); + + rte nb0, *new_best = NULL; + const rte *old_best = NULL; + + for (uint i = 0; i < count; i++) { - for (rte *r = net->routes; rte_is_valid(r); r = r->next) + if (!rte_is_valid(feed[i])) + continue; + + /* Has been already rejected, won't bother with it */ + if (!c->refeeding && bmap_test(&c->export_reject_map, feed[i]->id)) + continue; + + /* Previously exported */ + if (!old_best && bmap_test(&c->export_map, feed[i]->id)) { - if (bmap_test(&c->export_map, r->id)) + /* is still best */ + if (!new_best) { - old_best = r; - break; + DBG("rt_notify_accepted: idempotent\n"); + goto done; } - /* Note if new_changed found before old_best */ - if (r == new_changed) - new_first = 1; + /* is superseded */ + old_best = feed[i]; + break; } - } - /* Find new_best */ - if ((new_changed == old_changed) || (old_best == old_changed)) - { - /* Feed or old_best changed -> find first accepted by filters */ - for (rte *r = net->routes; rte_is_valid(r); r = r->next) - if (new_best = export_filter(c, r, &new_free, 0)) - break; + /* Have no new best route yet */ + if (!new_best) + { + /* Try this route not seen before */ + nb0 = *feed[i]; + new_best = export_filter(c, &nb0, 0); + DBG("rt_notify_accepted: checking route id %u: %s\n", feed[i]->id, new_best ? "ok" : "no"); + } } - else + +done: + /* Check obsolete routes for previously exported */ + while (rpe) { - /* Other cases -> either new_changed, or old_best (and nothing changed) */ - if (new_first && (new_changed = export_filter(c, new_changed, &new_free, 0))) - new_best = new_changed; - else - return; + channel_rpe_mark_seen(req, rpe); + if (rpe->old) + { + if (bmap_test(&c->export_map, rpe->old->rte.id)) + { + ASSERT_DIE(old_best == NULL); + old_best = &rpe->old->rte; + } + } + rpe = rpe_next(rpe, NULL); } - if (!new_best && !old_best) - return; - - do_rt_notify(c, net, new_best, old_best, refeed); + /* Nothing to export */ + if (new_best || old_best) + do_rt_notify(c, n, new_best, old_best); + else + DBG("rt_notify_accepted: nothing to export\n"); - /* Discard temporary rte */ - if (new_free) - rte_free(new_free); + rte_update_unlock(c); } @@ -811,38 +708,45 @@ nexthop_merge_rta(struct nexthop *nhs, rta *a, linpool *pool, int max) } rte * -rt_export_merged(struct channel *c, net *net, rte **rt_free, linpool *pool, int silent) +rt_export_merged(struct channel *c, struct rte **feed, uint count, linpool *pool, int silent) { + _Thread_local static rte rloc; + // struct proto *p = c->proto; struct nexthop *nhs = NULL; - rte *best0, *best, *rt0, *rt, *tmp; - - best0 = net->routes; - *rt_free = NULL; + rte *best0 = feed[0]; + rte *best = NULL; if (!rte_is_valid(best0)) return NULL; - best = export_filter_(c, best0, rt_free, pool, silent); + /* Already rejected, no need to re-run the filter */ + if (!c->refeeding && bmap_test(&c->export_reject_map, best0->id)) + return NULL; + + rloc = *best0; + best = export_filter_(c, &rloc, pool, silent); + + if (!best) + /* Best route doesn't pass the filter */ + return NULL; - if (!best || !rte_is_reachable(best)) + if (!rte_is_reachable(best)) + /* Unreachable routes can't be merged */ return best; - for (rt0 = best0->next; rt0; rt0 = rt0->next) + for (uint i = 1; i < count; i++) { - if (!rte_mergable(best0, rt0)) + if (!rte_mergable(best0, feed[i])) continue; - rt = export_filter_(c, rt0, &tmp, pool, 1); + rte tmp0 = *feed[i]; + rte *tmp = export_filter_(c, &tmp0, pool, 1); - if (!rt) + if (!tmp || !rte_is_reachable(tmp)) continue; - if (rte_is_reachable(rt)) - nhs = nexthop_merge_rta(nhs, rt->attrs, pool, c->merge_limit); - - if (tmp) - rte_free(tmp); + nhs = nexthop_merge_rta(nhs, tmp->attrs, pool, c->merge_limit); } if (nhs) @@ -851,66 +755,208 @@ rt_export_merged(struct channel *c, net *net, rte **rt_free, linpool *pool, int if (nhs->next) { - best = rte_cow_rta(best, pool); + best->attrs = rta_cow(best->attrs, pool); nexthop_link(best->attrs, nhs); } } - if (best != best0) - *rt_free = best; - return best; } - -static void -rt_notify_merged(struct channel *c, net *net, rte *new_changed, rte *old_changed, - rte *new_best, rte *old_best, int refeed) +void +rt_notify_merged(struct rt_export_request *req, const net_addr *n, struct rt_pending_export *rpe, + struct rte **feed, uint count) { - // struct proto *p = c->proto; - rte *new_free = NULL; + struct channel *c = SKIP_BACK(struct channel, out_req, req); - /* We assume that all rte arguments are either NULL or rte_is_valid() */ - - /* This check should be done by the caller */ - if (!new_best && !old_best) - return; + rte_update_lock(c); + // struct proto *p = c->proto; +#if 0 /* TODO: Find whether this check is possible when processing multiple changes at once. */ /* Check whether the change is relevant to the merged route */ if ((new_best == old_best) && (new_changed != old_changed) && !rte_mergable(new_best, new_changed) && !rte_mergable(old_best, old_changed)) return; +#endif - if (new_best) - c->stats.exp_updates_received++; - else - c->stats.exp_withdraws_received++; + rte *old_best = NULL; + /* Find old best route */ + for (uint i = 0; i < count; i++) + if (bmap_test(&c->export_map, feed[i]->id)) + { + old_best = feed[i]; + break; + } + + /* Check obsolete routes for previously exported */ + while (rpe) + { + channel_rpe_mark_seen(req, rpe); + if (rpe->old) + { + if (bmap_test(&c->export_map, rpe->old->rte.id)) + { + ASSERT_DIE(old_best == NULL); + old_best = &rpe->old->rte; + } + } + rpe = rpe_next(rpe, NULL); + } /* Prepare new merged route */ - if (new_best) - new_best = rt_export_merged(c, net, &new_free, rte_update_pool, 0); + rte *new_merged = count ? rt_export_merged(c, feed, count, c->rte_update_pool, 0) : NULL; - /* Check old merged route */ - if (old_best && !bmap_test(&c->export_map, old_best->id)) - old_best = NULL; + if (new_merged || old_best) + do_rt_notify(c, n, new_merged, old_best); - if (!new_best && !old_best) - return; + rte_update_unlock(c); +} - do_rt_notify(c, net, new_best, old_best, refeed); +void +rt_notify_optimal(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe) +{ + struct channel *c = SKIP_BACK(struct channel, out_req, req); + rte_update_lock(c); + rte *old = RTES_OR_NULL(rpe->old_best); + struct rte_storage *new_best = rpe->new_best; - /* Discard temporary rte */ - if (new_free) - rte_free(new_free); + while (rpe) + { + channel_rpe_mark_seen(req, rpe); + new_best = rpe->new_best; + rpe = rpe_next(rpe, NULL); + } + + if (&new_best->rte != old) + { + rte n0, *new = RTES_CLONE(new_best, &n0); + rt_notify_basic(c, net, new, old); + } + + rte_update_unlock(c); +} + +void +rt_notify_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe) +{ + struct channel *c = SKIP_BACK(struct channel, out_req, req); + rte_update_lock(c); + struct rte_src *src = rpe->new ? rpe->new->rte.src : rpe->old->rte.src; + rte *old = RTES_OR_NULL(rpe->old); + struct rte_storage *new_any = rpe->new; + + while (rpe) + { + channel_rpe_mark_seen(req, rpe); + new_any = rpe->new; + rpe = rpe_next(rpe, src); + } + + if (&new_any->rte != old) + { + rte n0, *new = RTES_CLONE(new_any, &n0); + rt_notify_basic(c, net, new, old); + } + + rte_update_unlock(c); +} + +void +rt_feed_any(struct rt_export_request *req, const net_addr *net, struct rt_pending_export *rpe UNUSED, rte **feed, uint count) +{ + struct channel *c = SKIP_BACK(struct channel, out_req, req); + rte_update_lock(c); + + for (uint i=0; i<count; i++) + { + rte n0 = *feed[i]; + rt_notify_basic(c, net, &n0, NULL); + } + + rte_update_unlock(c); } +void +rpe_mark_seen(struct rt_export_hook *hook, struct rt_pending_export *rpe) +{ + bmap_set(&hook->seq_map, rpe->seq); +} + +struct rt_pending_export * +rpe_next(struct rt_pending_export *rpe, struct rte_src *src) +{ + struct rt_pending_export *next = atomic_load_explicit(&rpe->next, memory_order_acquire); + + if (!next) + return NULL; + + if (!src) + return next; + + while (rpe = next) + if (src == (rpe->new ? rpe->new->rte.src : rpe->old->rte.src)) + return rpe; + else + next = atomic_load_explicit(&rpe->next, memory_order_acquire); + + return NULL; +} + +static struct rt_pending_export * rt_next_export_fast(struct rt_pending_export *last); +static void +rte_export(struct rt_export_hook *hook, struct rt_pending_export *rpe) +{ + if (bmap_test(&hook->seq_map, rpe->seq)) + goto seen; + + const net_addr *n = rpe->new_best ? rpe->new_best->rte.net : rpe->old_best->rte.net; + + if (rpe->new) + hook->stats.updates_received++; + else + hook->stats.withdraws_received++; + + if (hook->req->export_one) + hook->req->export_one(hook->req, n, rpe); + else if (hook->req->export_bulk) + { + RT_LOCK(hook->table); + net *net = SKIP_BACK(struct network, n.addr, (net_addr (*)[0]) n); + uint count = rte_feed_count(net); + rte **feed = NULL; + if (count) + { + feed = alloca(count * sizeof(rte *)); + rte_feed_obtain(net, feed, count); + } + RT_UNLOCK(hook->table); + hook->req->export_bulk(hook->req, n, rpe, feed, count); + } + else + bug("Export request must always provide an export method"); + +seen: + /* Get the next export if exists */ + hook->rpe_next = rt_next_export_fast(rpe); + + /* The last block may be available to free */ + if (PAGE_HEAD(hook->rpe_next) != PAGE_HEAD(rpe)) + { + RT_LOCK(hook->table); + rt_export_used(RT_PRIV(hook->table)); + RT_UNLOCK(hook->table); + } + + /* Releasing this export for cleanup routine */ + DBG("store hook=%p last_export=%p seq=%lu\n", hook, rpe, rpe->seq); + atomic_store_explicit(&hook->last_export, rpe, memory_order_release); +} /** * rte_announce - announce a routing table change * @tab: table the route has been added to - * @type: type of route announcement (RA_UNDEF or RA_ANY) * @net: network in question * @new: the new or changed route * @old: the previous route replaced by the new one @@ -926,13 +972,6 @@ rt_notify_merged(struct channel *c, net *net, rte *new_changed, rte *old_changed * and @new_best and @old_best describes best routes. Other routes are not * affected, but in sorted table the order of other routes might change. * - * Second, There is a bulk change of multiple routes in @net, with shared best - * route selection. In such case separate route changes are described using - * @type of %RA_ANY, with @new and @old specifying the changed route, while - * @new_best and @old_best are NULL. After that, another notification is done - * where @new_best and @old_best are filled (may be the same), but @new and @old - * are NULL. - * * The function announces the change to all associated channels. For each * channel, an appropriate preprocessing is done according to channel &ra_mode. * For example, %RA_OPTIMAL channels receive just changes of best routes. @@ -947,306 +986,416 @@ rt_notify_merged(struct channel *c, net *net, rte *new_changed, rte *old_changed * done outside of scope of rte_announce(). */ static void -rte_announce(rtable *tab, uint type, net *net, rte *new, rte *old, - rte *new_best, rte *old_best) +rte_announce(rtable_private *tab, net *net, struct rte_storage *new, struct rte_storage *old, + struct rte_storage *new_best, struct rte_storage *old_best) { - if (!rte_is_valid(new)) - new = NULL; - - if (!rte_is_valid(old)) - old = NULL; - - if (!rte_is_valid(new_best)) + if (!new_best || !rte_is_valid(&new_best->rte)) new_best = NULL; - if (!rte_is_valid(old_best)) + if (!old_best || !rte_is_valid(&old_best->rte)) old_best = NULL; - if (!new && !old && !new_best && !old_best) + if (!new || !rte_is_valid(&new->rte)) + new = NULL; + + if (old && !rte_is_valid(&old->rte)) + { + /* Filtered old route isn't announced, should be freed immediately. */ + rte_free(old, tab); + old = NULL; + } + + if ((new == old) && (new_best == old_best)) return; if (new_best != old_best) { if (new_best) - new_best->sender->stats.pref_routes++; + new_best->rte.sender->stats.pref++; if (old_best) - old_best->sender->stats.pref_routes--; + old_best->rte.sender->stats.pref--; if (tab->hostcache) rt_notify_hostcache(tab, net); } + if (EMPTY_LIST(tab->exports) && EMPTY_LIST(tab->pending_exports)) + { + /* No export hook and no pending exports to cleanup. We may free the route immediately. */ + if (!old) + return; + + hmap_clear(&tab->id_map, old->rte.id); + rte_free(old, tab); + return; + } + + /* Get the pending export structure */ + struct rt_export_block *rpeb = NULL, *rpebsnl = NULL; + u32 end = 0; + + if (!EMPTY_LIST(tab->pending_exports)) + { + rpeb = TAIL(tab->pending_exports); + end = atomic_load_explicit(&rpeb->end, memory_order_relaxed); + if (end >= RT_PENDING_EXPORT_ITEMS) + { + ASSERT_DIE(end == RT_PENDING_EXPORT_ITEMS); + rpebsnl = rpeb; + + rpeb = NULL; + end = 0; + } + } + + if (!rpeb) + { + rpeb = alloc_page(); + *rpeb = (struct rt_export_block) {}; + add_tail(&tab->pending_exports, &rpeb->n); + } + + /* Fill the pending export */ + struct rt_pending_export *rpe = &rpeb->export[rpeb->end]; + *rpe = (struct rt_pending_export) { + .new = new, + .new_best = new_best, + .old = old, + .old_best = old_best, + .seq = tab->next_export_seq++, + }; + + DBG("rte_announce: table=%s net=%N new=%p from %p old=%p from %p new_best=%p old_best=%p seq=%lu\n", tab->name, net->n.addr, new, new ? new->sender : NULL, old, old ? old->sender : NULL, new_best, old_best, rpe->seq); + + ASSERT_DIE(atomic_fetch_add_explicit(&rpeb->end, 1, memory_order_release) == end); + + if (rpebsnl) + { + _Bool f = 0; + ASSERT_DIE(atomic_compare_exchange_strong_explicit(&rpebsnl->not_last, &f, 1, + memory_order_release, memory_order_relaxed)); + } + + /* Append to the same-network squasher list */ + if (net->last) + { + struct rt_pending_export *rpenull = NULL; + ASSERT_DIE(atomic_compare_exchange_strong_explicit( + &net->last->next, &rpenull, rpe, + memory_order_relaxed, + memory_order_relaxed)); + + } + + net->last = rpe; + + if (!net->first) + net->first = rpe; + + if (tab->first_export == NULL) + tab->first_export = rpe; + + if (!EMPTY_LIST(tab->exports) && + (tab->first_export->seq + tab->config->cork_limit <= tab->next_export_seq) && + !tab->cork_active) + { + if (config->table_debug) + log(L_TRACE "%s: cork activated", tab->name); + + ev_cork(&rt_cork); + tab->cork_active = 1; + } +} + +static struct rt_pending_export * +rt_next_export_fast(struct rt_pending_export *last) +{ + /* Get the whole export block and find our position in there. */ + struct rt_export_block *rpeb = PAGE_HEAD(last); + u32 pos = (last - &rpeb->export[0]); + u32 end = atomic_load_explicit(&rpeb->end, memory_order_acquire); + ASSERT_DIE(pos < end); + + /* Next is in the same block. */ + if (++pos < end) + return &rpeb->export[pos]; + + /* There is another block. */ + if (atomic_load_explicit(&rpeb->not_last, memory_order_acquire)) + { + /* This is OK to do non-atomically because of the not_last flag. */ + rpeb = NODE_NEXT(rpeb); + return &rpeb->export[0]; + } + + /* There is nothing more. */ + return NULL; +} + +static struct rt_pending_export * +rt_next_export(struct rt_export_hook *hook, rtable_private *tab) +{ + /* As the table is locked, it is safe to reload the last export pointer */ + struct rt_pending_export *last = atomic_load_explicit(&hook->last_export, memory_order_acquire); + + /* It is still valid, let's reuse it */ + if (last) + return rt_next_export_fast(last); + + /* No, therefore we must process the table's first pending export */ + else + return tab->first_export; +} + +static inline void +rt_send_export_event(struct rt_export_hook *hook) +{ + ev_send(hook->req->list, hook->event); +} + +static void +rt_announce_exports(void *data) +{ + rtable_private *tab = data; + ASSERT_DIE(birdloop_inside(tab->loop)); + rt_schedule_notify(tab); - struct channel *c; node *n; - WALK_LIST2(c, n, tab->channels, table_node) + struct rt_export_hook *c; node *n; + WALK_LIST2(c, n, tab->exports, n) { - if (c->export_state == ES_DOWN) + if (atomic_load_explicit(&c->export_state, memory_order_acquire) != TES_READY) continue; - if (type && (type != c->ra_mode)) - continue; + rt_send_export_event(c); + } +} - switch (c->ra_mode) +static void +rt_import_announce_exports(void *data) +{ + struct rt_import_hook *hook = data; + RT_LOCKED(hook->table, tab) + { + if (hook->import_state == TIS_CLEARED) { - case RA_OPTIMAL: - if (new_best != old_best) - rt_notify_basic(c, net, new_best, old_best, 0); - break; + rfree(hook->export_announce_event); - case RA_ANY: - if (new != old) - rt_notify_basic(c, net, new, old, 0); - break; + ev_send(hook->stopped->list, hook->stopped); + rem_node(&hook->n); + mb_free(hook); + rt_unlock_table(tab); + } + else + ev_send_loop(tab->loop, tab->announce_event); + } +} - case RA_ACCEPTED: - rt_notify_accepted(c, net, new, old, 0); - break; +static struct rt_pending_export * +rt_last_export(rtable_private *tab) +{ + struct rt_pending_export *rpe = NULL; - case RA_MERGED: - rt_notify_merged(c, net, new, old, new_best, old_best, 0); - break; + if (!EMPTY_LIST(tab->pending_exports)) + { + /* We'll continue processing exports from this export on */ + struct rt_export_block *reb = TAIL(tab->pending_exports); + ASSERT_DIE(reb->end); + rpe = &reb->export[reb->end - 1]; + } + + return rpe; +} + +#define RT_EXPORT_BULK 1024 + +static void +rt_export_hook(void *_data) +{ + struct rt_export_hook *c = _data; + + ASSERT_DIE(atomic_load_explicit(&c->export_state, memory_order_relaxed) == TES_READY); + + if (!c->rpe_next) + { + RT_LOCK(c->table); + c->rpe_next = rt_next_export(c, RT_PRIV(c->table)); + + if (!c->rpe_next) + { + rt_export_used(RT_PRIV(c->table)); + RT_UNLOCK(c->table); + return; } + + RT_UNLOCK(c->table); + } + + /* Process the export */ + for (uint i=0; i<RT_EXPORT_BULK; i++) + { + rte_export(c, c->rpe_next); + + if (!c->rpe_next) + break; } + + rt_send_export_event(c); } + static inline int -rte_validate(rte *e) +rte_validate(struct channel *ch, rte *e) { int c; - net *n = e->net; + const net_addr *n = e->net; - if (!net_validate(n->n.addr)) + if (!net_validate(n)) { log(L_WARN "Ignoring bogus prefix %N received via %s", - n->n.addr, e->sender->proto->name); + n, ch->proto->name); return 0; } /* FIXME: better handling different nettypes */ - c = !net_is_flow(n->n.addr) ? - net_classify(n->n.addr): (IADDR_HOST | SCOPE_UNIVERSE); + c = !net_is_flow(n) ? + net_classify(n): (IADDR_HOST | SCOPE_UNIVERSE); if ((c < 0) || !(c & IADDR_HOST) || ((c & IADDR_SCOPE_MASK) <= SCOPE_LINK)) { log(L_WARN "Ignoring bogus route %N received via %s", - n->n.addr, e->sender->proto->name); + n, ch->proto->name); return 0; } - if (net_type_match(n->n.addr, NB_DEST) == !e->attrs->dest) + if (net_type_match(n, NB_DEST) == !e->attrs->dest) { log(L_WARN "Ignoring route %N with invalid dest %d received via %s", - n->n.addr, e->attrs->dest, e->sender->proto->name); + n, e->attrs->dest, ch->proto->name); return 0; } if ((e->attrs->dest == RTD_UNICAST) && !nexthop_is_sorted(&(e->attrs->nh))) { log(L_WARN "Ignoring unsorted multipath route %N received via %s", - n->n.addr, e->sender->proto->name); + n, ch->proto->name); return 0; } return 1; } -/** - * rte_free - delete a &rte - * @e: &rte to be deleted - * - * rte_free() deletes the given &rte from the routing table it's linked to. - */ -void -rte_free(rte *e) -{ - if (rta_is_cached(e->attrs)) - rta_free(e->attrs); - sl_free(rte_slab, e); -} - -static inline void -rte_free_quick(rte *e) -{ - rta_free(e->attrs); - sl_free(rte_slab, e); -} - static int rte_same(rte *x, rte *y) { + ASSERT_DIE(x->attrs->cached && y->attrs->cached); + /* rte.flags are not checked, as they are mostly internal to rtable */ return x->attrs == y->attrs && - x->pflags == y->pflags && - x->pref == y->pref && - (!x->attrs->src->proto->rte_same || x->attrs->src->proto->rte_same(x, y)) && + x->src == y->src && rte_is_filtered(x) == rte_is_filtered(y); } static inline int rte_is_ok(rte *e) { return e && !rte_is_filtered(e); } static void -rte_recalculate(struct channel *c, net *net, rte *new, struct rte_src *src) +rte_recalculate(rtable_private *table, struct rt_import_hook *c, net *net, rte *new, struct rte_src *src) { - struct proto *p = c->proto; - struct rtable *table = c->table; - struct proto_stats *stats = &c->stats; - static struct tbf rl_pipe = TBF_DEFAULT_LOG_LIMITS; - rte *before_old = NULL; - rte *old_best = net->routes; + struct rt_import_request *req = c->req; + struct rt_import_stats *stats = &c->stats; + struct rte_storage *old_best_stored = net->routes, *old_stored = NULL; + rte *old_best = old_best_stored ? &old_best_stored->rte : NULL; rte *old = NULL; - rte **k; - - k = &net->routes; /* Find and remove original route from the same protocol */ - while (old = *k) - { - if (old->attrs->src == src) - { - /* If there is the same route in the routing table but from - * a different sender, then there are two paths from the - * source protocol to this routing table through transparent - * pipes, which is not allowed. - * - * We log that and ignore the route. If it is withdraw, we - * ignore it completely (there might be 'spurious withdraws', - * see FIXME in do_rte_announce()) - */ - if (old->sender->proto != p) - { - if (new) - { - log_rl(&rl_pipe, L_ERR "Pipe collision detected when sending %N to table %s", - net->n.addr, table->name); - rte_free_quick(new); - } - return; - } - if (new && rte_same(old, new)) - { - /* No changes, ignore the new route and refresh the old one */ + /* Set the stale cycle unless already set */ + if (new && !(new->flags & REF_USE_STALE)) + new->stale_cycle = c->stale_set; - old->flags &= ~(REF_STALE | REF_DISCARD | REF_MODIFY); + /* Find and remove original route from the same protocol */ + struct rte_storage **before_old = rte_find(net, src); - if (!rte_is_filtered(new)) - { - stats->imp_updates_ignored++; - rte_trace_in(D_ROUTES, c, new, "ignored"); - } - - rte_free_quick(new); - return; - } - *k = old->next; - table->rt_count--; - break; - } - k = &old->next; - before_old = old; - } - - /* Save the last accessed position */ - rte **pos = k; - - if (!old) - before_old = NULL; - - if (!old && !new) + if (!*before_old && !new) { - stats->imp_withdraws_ignored++; + stats->withdraws_ignored++; return; } - int new_ok = rte_is_ok(new); - int old_ok = rte_is_ok(old); + if (new) + new->attrs = rta_is_cached(new->attrs) ? rta_clone(new->attrs) : rta_lookup(new->attrs); - struct channel_limit *l = &c->rx_limit; - if (l->action && !old && new && !c->in_table) + if (*before_old) { - u32 all_routes = stats->imp_routes + stats->filt_routes; - - if (all_routes >= l->limit) - channel_notify_limit(c, l, PLD_RX, all_routes); - - if (l->state == PLS_BLOCKED) + old = &(old_stored = (*before_old))->rte; + + /* If there is the same route in the routing table but from + * a different sender, then there are two paths from the + * source protocol to this routing table through transparent + * pipes, which is not allowed. + * We log that and ignore the route. */ + if (old->sender != c) { - /* In receive limit the situation is simple, old is NULL so - we just free new and exit like nothing happened */ + if (!old->generation && !new->generation) + bug("Two protocols claim to author a route with the same rte_src in table %s: %N %s/%u:%u", + c->table->name, net->n.addr, old->src->owner->name, old->src->private_id, old->src->global_id); - stats->imp_updates_ignored++; - rte_trace_in(D_FILTERS, c, new, "ignored [limit]"); - rte_free_quick(new); - return; + log_rl(&table->rl_pipe, L_ERR "Route source collision in table %s: %N %s/%u:%u", + c->table->name, net->n.addr, old->src->owner->name, old->src->private_id, old->src->global_id); } - } - l = &c->in_limit; - if (l->action && !old_ok && new_ok) - { - if (stats->imp_routes >= l->limit) - channel_notify_limit(c, l, PLD_IN, stats->imp_routes); - - if (l->state == PLS_BLOCKED) - { - /* In import limit the situation is more complicated. We - shouldn't just drop the route, we should handle it like - it was filtered. We also have to continue the route - processing if old or new is non-NULL, but we should exit - if both are NULL as this case is probably assumed to be - already handled. */ + if (new && rte_same(old, new)) + { + /* No changes, ignore the new route and refresh the old one */ + old->stale_cycle = new->stale_cycle; - stats->imp_updates_ignored++; - rte_trace_in(D_FILTERS, c, new, "ignored [limit]"); + if (!rte_is_filtered(new)) + { + stats->updates_ignored++; + rt_rte_trace_in(D_ROUTES, req, new, "ignored"); + } - if (c->in_keep_filtered) - new->flags |= REF_FILTERED; - else - { rte_free_quick(new); new = NULL; } + rta_free(new->attrs); + return; + } - /* Note that old && !new could be possible when - c->in_keep_filtered changed in the recent past. */ + *before_old = (*before_old)->next; + table->rt_count--; + } - if (!old && !new) - return; + if (req->preimport) + new = req->preimport(req, new, old); - new_ok = 0; - goto skip_stats1; - } - } + int new_ok = rte_is_ok(new); + int old_ok = rte_is_ok(old); if (new_ok) - stats->imp_updates_accepted++; + stats->updates_accepted++; else if (old_ok) - stats->imp_withdraws_accepted++; + stats->withdraws_accepted++; else - stats->imp_withdraws_ignored++; + stats->withdraws_ignored++; if (old_ok || new_ok) table->last_rt_change = current_time(); - skip_stats1: - - if (new) - rte_is_filtered(new) ? stats->filt_routes++ : stats->imp_routes++; - if (old) - rte_is_filtered(old) ? stats->filt_routes-- : stats->imp_routes--; + struct rte_storage *new_stored = new ? rte_store(new, net, table) : NULL; if (table->config->sorted) { /* If routes are sorted, just insert new route to appropriate position */ - if (new) + if (new_stored) { - if (before_old && !rte_better(new, before_old)) - k = &before_old->next; + struct rte_storage **k; + if ((before_old != &net->routes) && !rte_better(new, &SKIP_BACK(struct rte_storage, next, before_old)->rte)) + k = before_old; else k = &net->routes; for (; *k; k=&(*k)->next) - if (rte_better(new, *k)) + if (rte_better(new, &(*k)->rte)) break; - new->next = *k; - *k = new; + new_stored->next = *k; + *k = new_stored; table->rt_count++; } @@ -1256,16 +1405,17 @@ rte_recalculate(struct channel *c, net *net, rte *new, struct rte_src *src) /* If routes are not sorted, find the best route and move it on the first position. There are several optimized cases. */ - if (src->proto->rte_recalculate && src->proto->rte_recalculate(table, net, new, old, old_best)) + if (src->owner->rte_recalculate && + src->owner->rte_recalculate(table, net, new_stored ? &new_stored->rte : NULL, old, old_best)) goto do_recalculate; - if (new && rte_better(new, old_best)) + if (new_stored && rte_better(&new_stored->rte, old_best)) { /* The first case - the new route is cleary optimal, we link it at the first position */ - new->next = net->routes; - net->routes = new; + new_stored->next = net->routes; + net->routes = new_stored; table->rt_count++; } @@ -1279,10 +1429,10 @@ rte_recalculate(struct channel *c, net *net, rte *new, struct rte_src *src) do_recalculate: /* Add the new route to the list */ - if (new) + if (new_stored) { - new->next = *pos; - *pos = new; + new_stored->next = *before_old; + *before_old = new_stored; table->rt_count++; } @@ -1290,335 +1440,408 @@ rte_recalculate(struct channel *c, net *net, rte *new, struct rte_src *src) /* Find a new optimal route (if there is any) */ if (net->routes) { - rte **bp = &net->routes; - for (k=&(*bp)->next; *k; k=&(*k)->next) - if (rte_better(*k, *bp)) + struct rte_storage **bp = &net->routes; + for (struct rte_storage **k=&(*bp)->next; *k; k=&(*k)->next) + if (rte_better(&(*k)->rte, &(*bp)->rte)) bp = k; /* And relink it */ - rte *best = *bp; + struct rte_storage *best = *bp; *bp = best->next; best->next = net->routes; net->routes = best; } } - else if (new) + else if (new_stored) { /* The third case - the new route is not better than the old best route (therefore old_best != NULL) and the old best route was not removed (therefore old_best == net->routes). We just link the new route to the old/last position. */ - new->next = *pos; - *pos = new; + new_stored->next = *before_old; + *before_old = new_stored; table->rt_count++; } /* The fourth (empty) case - suboptimal route was removed, nothing to do */ } - if (new) + if (new_stored) { - new->lastmod = current_time(); - - if (!old) - { - new->id = hmap_first_zero(&table->id_map); - hmap_set(&table->id_map, new->id); - } - else - new->id = old->id; + new_stored->rte.lastmod = current_time(); + new_stored->rte.id = hmap_first_zero(&table->id_map); + hmap_set(&table->id_map, new_stored->rte.id); } + _Bool nb = (new_stored == net->routes); + _Bool ob = (old_best == old); + /* Log the route change */ - if ((c->debug & D_ROUTES) || (p->debug & D_ROUTES)) + if (new_ok && old_ok) { - if (new_ok) - rte_trace(c, new, '>', new == net->routes ? "added [best]" : "added"); - else if (old_ok) - { - if (old != old_best) - rte_trace(c, old, '>', "removed"); - else if (rte_is_ok(net->routes)) - rte_trace(c, old, '>', "removed [replaced]"); - else - rte_trace(c, old, '>', "removed [sole]"); - } + const char *best_indicator[2][2] = { { "updated", "updated [-best]" }, { "updated [+best]", "updated [best]" } }; + rt_rte_trace_in(D_ROUTES, req, &new_stored->rte, best_indicator[nb][ob]); } + else if (new_ok) + rt_rte_trace_in(D_ROUTES, req, &new_stored->rte, + (!net->routes->next || !rte_is_ok(&net->routes->next->rte)) ? "added [sole]" : + nb ? "added [best]" : "added"); + else if (old_ok) + rt_rte_trace_in(D_ROUTES, req, old, + (!net->routes || !rte_is_ok(&net->routes->rte)) ? "removed [sole]" : + ob ? "removed [best]" : "removed"); /* Propagate the route change */ - rte_announce(table, RA_UNDEF, net, new, old, net->routes, old_best); + rte_announce(table, net, new_stored, old_stored, + net->routes, old_best_stored); if (!net->routes && (table->gc_counter++ >= table->config->gc_max_ops) && (table->gc_time + table->config->gc_min_time <= current_time())) rt_schedule_prune(table); +#if 0 + /* Enable and reimplement these callbacks if anybody wants to use them */ if (old_ok && p->rte_remove) p->rte_remove(net, old); if (new_ok && p->rte_insert) - p->rte_insert(net, new); - - if (old) - { - if (!new) - hmap_clear(&table->id_map, old->id); - - rte_free_quick(old); - } -} - -static int rte_update_nest_cnt; /* Nesting counter to allow recursive updates */ + p->rte_insert(net, &new_stored->rte); +#endif -static inline void -rte_update_lock(void) -{ - rte_update_nest_cnt++; } -static inline void -rte_update_unlock(void) +rte * +channel_preimport(struct rt_import_request *req, rte *new, rte *old) { - if (!--rte_update_nest_cnt) - lp_flush(rte_update_pool); -} + struct channel *c = SKIP_BACK(struct channel, in_req, req); -static inline void -rte_hide_dummy_routes(net *net, rte **dummy) -{ - if (net->routes && net->routes->attrs->source == RTS_DUMMY) + if (!c->in_table) { - *dummy = net->routes; - net->routes = (*dummy)->next; + if (new && !old) + if (CHANNEL_LIMIT_PUSH(c, RX)) + { + rta_free(new->attrs); + return NULL; + } + + if (!new && old) + CHANNEL_LIMIT_POP(c, RX); } + + int new_in = new && !rte_is_filtered(new); + int old_in = old && !rte_is_filtered(old); + + if (new_in && !old_in) + if (CHANNEL_LIMIT_PUSH(c, IN)) + if (c->in_keep_filtered) + { + new->flags |= REF_FILTERED; + return new; + } + else + { + rta_free(new->attrs); + return NULL; + } + + if (!new_in && old_in) + CHANNEL_LIMIT_POP(c, IN); + + return new; } -static inline void -rte_unhide_dummy_routes(net *net, rte **dummy) +rte * +channel_in_preimport(struct rt_import_request *req, rte *new, rte *old) { - if (*dummy) - { - (*dummy)->next = net->routes; - net->routes = *dummy; - } + struct channel_aux_table *cat = SKIP_BACK(struct channel_aux_table, push, req); + + if (new && !old) + if (CHANNEL_LIMIT_PUSH(cat->c, RX)) + return NULL; + + if (!new && old) + CHANNEL_LIMIT_POP(cat->c, RX); + + return new; } -/** - * rte_update - enter a new update to a routing table - * @table: table to be updated - * @c: channel doing the update - * @net: network node - * @p: protocol submitting the update - * @src: protocol originating the update - * @new: a &rte representing the new route or %NULL for route removal. - * - * This function is called by the routing protocols whenever they discover - * a new route or wish to update/remove an existing route. The right announcement - * sequence is to build route attributes first (either un-cached with @aflags set - * to zero or a cached one using rta_lookup(); in this case please note that - * you need to increase the use count of the attributes yourself by calling - * rta_clone()), call rte_get_temp() to obtain a temporary &rte, fill in all - * the appropriate data and finally submit the new &rte by calling rte_update(). - * - * @src specifies the protocol that originally created the route and the meaning - * of protocol-dependent data of @new. If @new is not %NULL, @src have to be the - * same value as @new->attrs->proto. @p specifies the protocol that called - * rte_update(). In most cases it is the same protocol as @src. rte_update() - * stores @p in @new->sender; - * - * When rte_update() gets any route, it automatically validates it (checks, - * whether the network and next hop address are valid IP addresses and also - * whether a normal routing protocol doesn't try to smuggle a host or link - * scope route to the table), converts all protocol dependent attributes stored - * in the &rte to temporary extended attributes, consults import filters of the - * protocol to see if the route should be accepted and/or its attributes modified, - * stores the temporary attributes back to the &rte. - * - * Now, having a "public" version of the route, we - * automatically find any old route defined by the protocol @src - * for network @n, replace it by the new one (or removing it if @new is %NULL), - * recalculate the optimal route for this destination and finally broadcast - * the change (if any) to all routing protocols by calling rte_announce(). - * - * All memory used for attribute lists and other temporary allocations is taken - * from a special linear pool @rte_update_pool and freed when rte_update() - * finishes. - */ +void rte_update_direct(struct channel *c, const net_addr *n, rte *new, struct rte_src *src); void -rte_update2(struct channel *c, const net_addr *n, rte *new, struct rte_src *src) +rte_update(struct channel *c, const net_addr *n, rte *new, struct rte_src *src) { - // struct proto *p = c->proto; - struct proto_stats *stats = &c->stats; - const struct filter *filter = c->in_filter; - rte *dummy = NULL; - net *nn; + if (!c->in_req.hook) + return; ASSERT(c->channel_state == CS_UP); - rte_update_lock(); + if (c->in_table) + rte_import(&c->in_table->push, n, new, src); + else + rte_update_direct(c, n, new, src); +} + +void +rte_update_direct(struct channel *c, const net_addr *n, rte *new, struct rte_src *src) +{ + const struct filter *filter = c->in_filter; + struct channel_import_stats *stats = &c->import_stats; + + rte_update_lock(c); if (new) { - /* Create a temporary table node */ - nn = alloca(sizeof(net) + n->length); - memset(nn, 0, sizeof(net) + n->length); - net_copy(nn->n.addr, n); - - new->net = nn; - new->sender = c; + new->net = n; - if (!new->pref) - new->pref = c->preference; + int fr; - stats->imp_updates_received++; - if (!rte_validate(new)) + stats->updates_received++; + if (!rte_validate(c, new)) { - rte_trace_in(D_FILTERS, c, new, "invalid"); - stats->imp_updates_invalid++; - goto drop; + channel_rte_trace_in(D_FILTERS, c, new, "invalid"); + stats->updates_invalid++; + new = NULL; } - - if (filter == FILTER_REJECT) + else if ((filter == FILTER_REJECT) || + ((fr = f_run(filter, new, c->rte_update_pool, 0)) > F_ACCEPT)) { - stats->imp_updates_filtered++; - rte_trace_in(D_FILTERS, c, new, "filtered out"); - - if (! c->in_keep_filtered) - goto drop; + stats->updates_filtered++; + channel_rte_trace_in(D_FILTERS, c, new, "filtered out"); - /* new is a private copy, i could modify it */ - new->flags |= REF_FILTERED; + if (c->in_keep_filtered) + new->flags |= REF_FILTERED; + else + new = NULL; } - else if (filter) - { - rta *old_attrs = NULL; - rte_make_tmp_attrs(&new, rte_update_pool, &old_attrs); + } + else + stats->withdraws_received++; - int fr = f_run(filter, &new, rte_update_pool, 0); - if (fr > F_ACCEPT) - { - stats->imp_updates_filtered++; - rte_trace_in(D_FILTERS, c, new, "filtered out"); + rte_import(&c->in_req, n, new, src); - if (! c->in_keep_filtered) - { - rta_free(old_attrs); - goto drop; - } + rte_update_unlock(c); +} - new->flags |= REF_FILTERED; - } +void +rte_import(struct rt_import_request *req, const net_addr *n, rte *new, struct rte_src *src) +{ + struct rt_import_hook *hook = req->hook; + if (!hook) + return; - rte_store_tmp_attrs(new, rte_update_pool, old_attrs); - } - if (!rta_is_cached(new->attrs)) /* Need to copy attributes */ - new->attrs = rta_lookup(new->attrs); - new->flags |= REF_COW; + RT_LOCK(hook->table); + rtable_private *tab = RT_PRIV(hook->table); + net *nn; + if (new) + { /* Use the actual struct network, not the dummy one */ - nn = net_get(c->table, n); - new->net = nn; + nn = net_get(tab, n); + new->net = nn->n.addr; + new->sender = hook; } - else + else if (!(nn = net_find(tab, n))) { - stats->imp_withdraws_received++; - - if (!(nn = net_find(c->table, n)) || !src) - { - stats->imp_withdraws_ignored++; - rte_update_unlock(); - return; - } + req->hook->stats.withdraws_ignored++; + RT_UNLOCK(tab); + return; } - recalc: /* And recalculate the best route */ - rte_hide_dummy_routes(nn, &dummy); - rte_recalculate(c, nn, new, src); - rte_unhide_dummy_routes(nn, &dummy); + rte_recalculate(tab, hook, nn, new, src); - rte_update_unlock(); - return; + /* Schedule export announcement */ + ev_send(req->list, hook->export_announce_event); - drop: - rte_free(new); - new = NULL; - if (nn = net_find(c->table, n)) - goto recalc; + /* Done! */ + RT_UNLOCK(tab); +} - rte_update_unlock(); +/* Check rtable for best route to given net whether it would be exported do p */ +int +rt_examine(rtable_private *t, net_addr *a, struct channel *c, const struct filter *filter) +{ + net *n = net_find(t, a); + + if (!n || !n->routes) + return 0; + + rte rt = n->routes->rte; + + if (!rte_is_valid(&rt)) + return 0; + + rte_update_lock(c); + + /* Rest is stripped down export_filter() */ + int v = c->proto->preexport ? c->proto->preexport(c, &rt) : 0; + if (v == RIC_PROCESS) + v = (f_run(filter, &rt, c->rte_update_pool, FF_SILENT) <= F_ACCEPT); + + rte_update_unlock(c); + + return v > 0; } -/* Independent call to rte_announce(), used from next hop - recalculation, outside of rte_update(). new must be non-NULL */ -static inline void -rte_announce_i(rtable *tab, uint type, net *net, rte *new, rte *old, - rte *new_best, rte *old_best) +static void +rt_export_stopped(void *data) { - rte_update_lock(); - rte_announce(tab, type, net, new, old, new_best, old_best); - rte_update_unlock(); + struct rt_export_hook *hook = data; + + RT_LOCKED(hook->table, tab) + { + /* Drop pending exports */ + rt_export_used(tab); + + /* Unlist */ + rem_node(&hook->n); + } + + /* Report the channel as stopped. */ + hook->stopped(hook->req); + + RT_LOCKED(hook->table, tab) + { + /* Free the hook together with its coroutine. */ + rp_free(hook->pool, tab->rp); + rt_unlock_table(tab); + + rt_fast_prune_check(tab); + + DBG("Export hook %p in table %s finished uc=%u\n", hook, tab->name, tab->use_count); + } } + static inline void -rte_discard(rte *old) /* Non-filtered route deletion, used during garbage collection */ +rt_set_import_state(struct rt_import_hook *hook, u8 state) { - rte_update_lock(); - rte_recalculate(old->sender, old->net, NULL, old->attrs->src); - rte_update_unlock(); + hook->last_state_change = current_time(); + hook->import_state = state; + + if (hook->req->log_state_change) + hook->req->log_state_change(hook->req, state); } -/* Modify existing route by protocol hook, used for long-lived graceful restart */ static inline void -rte_modify(rte *old) +rt_set_export_state(struct rt_export_hook *hook, u8 state) { - rte_update_lock(); + hook->last_state_change = current_time(); + atomic_store_explicit(&hook->export_state, state, memory_order_release); - rte *new = old->sender->proto->rte_modify(old, rte_update_pool); - if (new != old) - { - if (new) - { - if (!rta_is_cached(new->attrs)) - new->attrs = rta_lookup(new->attrs); - new->flags = (old->flags & ~REF_MODIFY) | REF_COW; - } + if (hook->req->log_state_change) + hook->req->log_state_change(hook->req, state); +} - rte_recalculate(old->sender, old->net, new, old->attrs->src); - } +void +rt_request_import(rtable *t, struct rt_import_request *req) +{ + RT_LOCK(t); + rtable_private *tab = RT_PRIV(t); + rt_lock_table(tab); + + ASSERT_DIE(!tab->delete); + + struct rt_import_hook *hook = req->hook = mb_allocz(tab->rp, sizeof(struct rt_import_hook)); + + DBG("Lock table %s for import %p req=%p uc=%u\n", tab->name, hook, req, tab->use_count); - rte_update_unlock(); + hook->req = req; + hook->table = t; + + hook->export_announce_event = ev_new_init(tab->rp, rt_import_announce_exports, hook); + + if (!hook->stale_set) + hook->stale_set = hook->stale_valid = hook->stale_pruning = hook->stale_pruned = 1; + + rt_set_import_state(hook, TIS_UP); + + hook->n = (node) {}; + add_tail(&tab->imports, &hook->n); + tab->imports_up++; + + RT_UNLOCK(t); } -/* Check rtable for best route to given net whether it would be exported do p */ -int -rt_examine(rtable *t, net_addr *a, struct proto *p, const struct filter *filter) +void +rt_stop_import(struct rt_import_request *req, event *stopped) { - net *n = net_find(t, a); - rte *rt = n ? n->routes : NULL; + ASSERT_DIE(req->hook); + struct rt_import_hook *hook = req->hook; - if (!rte_is_valid(rt)) - return 0; + rtable_private *tab = RT_LOCK(hook->table); - rte_update_lock(); + rt_schedule_prune(tab); - /* Rest is stripped down export_filter() */ - int v = p->preexport ? p->preexport(p, &rt, rte_update_pool) : 0; - if (v == RIC_PROCESS) - { - rte_make_tmp_attrs(&rt, rte_update_pool, NULL); - v = (f_run(filter, &rt, rte_update_pool, FF_SILENT) <= F_ACCEPT); - } + tab->imports_up--; + rt_fast_prune_check(tab); - /* Discard temporary rte */ - if (rt != n->routes) - rte_free(rt); + rt_set_import_state(hook, TIS_STOP); - rte_update_unlock(); + hook->stopped = stopped; - return v > 0; + if (hook->stale_set < hook->stale_valid) + if (!--tab->rr_count) + rt_schedule_notify(tab); + + RT_UNLOCK(tab); } +void +rt_request_export(rtable *t, struct rt_export_request *req) +{ + RT_LOCK(t); + rtable_private *tab = RT_PRIV(t); + rt_lock_table(tab); + + pool *p = rp_new(tab->rp, tab->loop, "Export hook"); + struct rt_export_hook *hook = req->hook = mb_allocz(p, sizeof(struct rt_export_hook)); + hook->pool = p; + + hook->req = req; + hook->table = t; + + /* stats zeroed by mb_allocz */ + + bmap_init(&hook->seq_map, p, 1024); + + rt_set_export_state(hook, TES_HUNGRY); + + hook->n = (node) {}; + add_tail(&tab->exports, &hook->n); + + DBG("New export hook %p req %p in table %s uc=%u\n", hook, req, tab->name, tab->use_count); + + hook->event = ev_new_init(p, rt_feed_channel, hook); + RT_UNLOCK(t); + + rt_send_export_event(hook); +} + +void +rt_stop_export(struct rt_export_request *req, void (*stopped)(struct rt_export_request *)) +{ + ASSERT_DIE(req->hook); + struct rt_export_hook *hook = req->hook; + + RT_LOCK(hook->table); + rtable_private *tab = RT_PRIV(hook->table); + + /* Stop feeding */ + ev_postpone(hook->event); + + if (atomic_load_explicit(&hook->export_state, memory_order_relaxed) == TES_FEEDING) + fit_get(&tab->fib, &hook->feed_fit); + + hook->event->hook = rt_export_stopped; + hook->stopped = stopped; + + rt_send_export_event(hook); + + RT_UNLOCK(hook->table); + + rt_set_export_state(hook, TES_STOP); +} /** * rt_refresh_begin - start a refresh cycle @@ -1630,21 +1853,48 @@ rt_examine(rtable *t, net_addr *a, struct proto *p, const struct filter *filter) * routes to the routing table (by rte_update()). After that, all protocol * routes (more precisely routes with @c as @sender) not sent during the * refresh cycle but still in the table from the past are pruned. This is - * implemented by marking all related routes as stale by REF_STALE flag in - * rt_refresh_begin(), then marking all related stale routes with REF_DISCARD - * flag in rt_refresh_end() and then removing such routes in the prune loop. - */ + * implemented by setting rte->stale_cycle to req->stale_set in rte_update() + * and then dropping all routes with old stale_cycle values in table prune loop. */ void -rt_refresh_begin(rtable *t, struct channel *c) +rt_refresh_begin(struct rt_import_request *req) { - FIB_WALK(&t->fib, net, n) - { - rte *e; - for (e = n->routes; e; e = e->next) - if (e->sender == c) - e->flags |= REF_STALE; - } - FIB_WALK_END; + struct rt_import_hook *hook = req->hook; + ASSERT_DIE(hook); + + RT_LOCK(hook->table); + rtable_private *tab = RT_PRIV(hook->table); + + ASSERT_DIE(hook->stale_set == hook->stale_valid); + + /* If the pruning routine is too slow */ + if ((hook->stale_pruned < hook->stale_valid) && (hook->stale_pruned + 128 < hook->stale_valid) + || (hook->stale_pruned > hook->stale_valid) && (hook->stale_pruned > hook->stale_valid + 128)) + { + log(L_WARN "Route refresh flood in table %s", tab->name); + FIB_WALK(&tab->fib, net, n) + { + for (struct rte_storage *e = n->routes; e; e = e->next) + if (e->rte.sender == req->hook) + e->rte.stale_cycle = 0; + } + FIB_WALK_END; + hook->stale_set = 1; + hook->stale_valid = 0; + hook->stale_pruned = 0; + } + else if (!++hook->stale_set) + { + /* Let's reserve the stale_cycle zero value for always-invalid routes */ + hook->stale_set = 1; + hook->stale_valid = 0; + } + + tab->rr_count++; + + if (req->trace_routes & D_STATES) + log(L_TRACE "%s: route refresh begin [%u]", req->name, hook->stale_set); + + RT_UNLOCK(tab); } /** @@ -1656,45 +1906,24 @@ rt_refresh_begin(rtable *t, struct channel *c) * hook. See rt_refresh_begin() for description of refresh cycles. */ void -rt_refresh_end(rtable *t, struct channel *c) +rt_refresh_end(struct rt_import_request *req) { - int prune = 0; + struct rt_import_hook *hook = req->hook; + ASSERT_DIE(hook); - FIB_WALK(&t->fib, net, n) - { - rte *e; - for (e = n->routes; e; e = e->next) - if ((e->sender == c) && (e->flags & REF_STALE)) - { - e->flags |= REF_DISCARD; - prune = 1; - } - } - FIB_WALK_END; + rtable_private *tab = RT_LOCK(hook->table); + hook->stale_valid++; + ASSERT_DIE(hook->stale_set == hook->stale_valid); - if (prune) - rt_schedule_prune(t); -} + rt_schedule_prune(tab); -void -rt_modify_stale(rtable *t, struct channel *c) -{ - int prune = 0; + if (req->trace_routes & D_STATES) + log(L_TRACE "%s: route refresh end [%u]", req->name, hook->stale_valid); - FIB_WALK(&t->fib, net, n) - { - rte *e; - for (e = n->routes; e; e = e->next) - if ((e->sender == c) && (e->flags & REF_STALE) && !(e->flags & REF_FILTERED)) - { - e->flags |= REF_MODIFY; - prune = 1; - } - } - FIB_WALK_END; + if (!--tab->rr_count) + rt_schedule_notify(tab); - if (prune) - rt_schedule_prune(t); + RT_UNLOCK(tab); } /** @@ -1704,14 +1933,11 @@ rt_modify_stale(rtable *t, struct channel *c) * This functions dumps contents of a &rte to debug output. */ void -rte_dump(rte *e) -{ - net *n = e->net; - debug("%-1N ", n->n.addr); - debug("PF=%02x pref=%d ", e->pflags, e->pref); - rta_dump(e->attrs); - if (e->attrs->src->proto->proto->dump_attrs) - e->attrs->src->proto->proto->dump_attrs(e); +rte_dump(struct rte_storage *e) +{ + debug("%-1N ", e->rte.net); + debug("PF=%02x ", e->rte.pflags); + rta_dump(e->rte.attrs); debug("\n"); } @@ -1722,20 +1948,22 @@ rte_dump(rte *e) * This function dumps contents of a given routing table to debug output. */ void -rt_dump(rtable *t) +rt_dump(rtable *tab) { - debug("Dump of routing table <%s>\n", t->name); + RT_LOCK(tab); + rtable_private *t = RT_PRIV(tab); + debug("Dump of routing table <%s>%s\n", t->name, t->delete ? " (deleted)" : ""); #ifdef DEBUGGING fib_check(&t->fib); #endif FIB_WALK(&t->fib, net, n) { - rte *e; - for(e=n->routes; e; e=e->next) + for(struct rte_storage *e=n->routes; e; e=e->next) rte_dump(e); } FIB_WALK_END; debug("\n"); + RT_UNLOCK(tab); } /** @@ -1753,73 +1981,121 @@ rt_dump_all(void) rt_dump(t); } -static inline void -rt_schedule_hcu(rtable *tab) +void +rt_dump_hooks(rtable *t) +{ + RT_LOCK(t); + rtable_private *tab = RT_PRIV(t); + debug("Dump of hooks in routing table <%s>%s\n", tab->name, tab->delete ? " (deleted)" : ""); + debug(" nhu_state=%u hcu_scheduled=%u use_count=%d rt_count=%u\n", + atomic_load(&tab->nhu_state), ev_active(tab->hcu_event), tab->use_count, tab->rt_count); + debug(" last_rt_change=%t gc_time=%t gc_counter=%d prune_state=%u\n", + tab->last_rt_change, tab->gc_time, tab->gc_counter, tab->prune_state); + + struct rt_import_hook *ih; + WALK_LIST(ih, tab->imports) + { + ih->req->dump_req(ih->req); + debug(" Import hook %p requested by %p: pref=%u" + " last_state_change=%t import_state=%u stopped=%p\n", + ih, ih->req, ih->stats.pref, + ih->last_state_change, ih->import_state, ih->stopped); + } + + struct rt_export_hook *eh; + WALK_LIST(eh, tab->exports) + { + eh->req->dump_req(eh->req); + debug(" Export hook %p requested by %p:" + " refeed_pending=%u last_state_change=%t export_state=%u\n", + eh, eh->req, eh->refeed_pending, eh->last_state_change, atomic_load_explicit(&eh->export_state, memory_order_relaxed)); + } + debug("\n"); + RT_UNLOCK(t); +} + +void +rt_dump_hooks_all(void) { - if (tab->hcu_scheduled) - return; + rtable *t; + node *n; - tab->hcu_scheduled = 1; - ev_schedule(tab->rt_event); + debug("Dump of all table hooks\n"); + + WALK_LIST2(t, n, routing_tables, n) + rt_dump_hooks(t); } static inline void rt_schedule_nhu(rtable *tab) { - if (tab->nhu_state == NHU_CLEAN) - ev_schedule(tab->rt_event); + atomic_fetch_or_explicit(&tab->nhu_state, NHU_SCHEDULED, memory_order_acq_rel); + ev_send_loop(tab->loop, tab->nhu_event); /* state change: * NHU_CLEAN -> NHU_SCHEDULED * NHU_RUNNING -> NHU_DIRTY */ - tab->nhu_state |= NHU_SCHEDULED; } void -rt_schedule_prune(rtable *tab) +rt_schedule_prune(rtable_private *tab) { if (tab->prune_state == 0) - ev_schedule(tab->rt_event); + ev_send_loop(tab->loop, tab->prune_event); /* state change 0->1, 2->3 */ tab->prune_state |= 1; } +static int +rt_fast_prune_ready(rtable_private *tab) +{ + return EMPTY_LIST(tab->pending_exports) && EMPTY_LIST(tab->exports) && !tab->imports_up; +} static void -rt_event(void *ptr) +rt_fast_prune_check(rtable_private *tab) { - rtable *tab = ptr; - - rt_lock_table(tab); - - if (tab->hcu_scheduled) - rt_update_hostcache(tab); - - if (tab->nhu_state) - rt_next_hop_update(tab); + if (tab->delete && rt_fast_prune_ready(tab)) + { + tab->prune_state |= 1; + ev_send_loop(tab->loop, tab->prune_event); + } +} - if (tab->prune_state) - rt_prune_table(tab); +void +rt_export_used(rtable_private *tab) +{ + if (config->table_debug) + log(L_TRACE "%s: Export cleanup requested", tab->name); - rt_unlock_table(tab); + ev_send_loop(tab->loop, tab->ec_event); } - static inline btime -rt_settled_time(rtable *tab) +rt_settled_time(rtable_private *tab) { ASSUME(tab->base_settle_time != 0); - return MIN(tab->last_rt_change + tab->config->min_settle_time, - tab->base_settle_time + tab->config->max_settle_time); + btime min_settle_time = tab->rr_count ? tab->config->min_rr_settle_time : tab->config->min_settle_time; + btime max_settle_time = tab->rr_count ? tab->config->max_rr_settle_time : tab->config->max_settle_time; + + DBG("settled time computed from %t %t %t %t as %t / %t, now is %t\n", + tab->name, tab->last_rt_change, min_settle_time, + tab->base_settle_time, max_settle_time, + tab->last_rt_change + min_settle_time, + tab->base_settle_time + max_settle_time, current_time()); + + return MIN(tab->last_rt_change + min_settle_time, + tab->base_settle_time + max_settle_time); } static void rt_settle_timer(timer *t) { - rtable *tab = t->data; + rtable_private *tab = t->data; + ASSERT_DIE(birdloop_inside(tab->loop)); if (!tab->base_settle_time) return; @@ -1827,7 +2103,7 @@ rt_settle_timer(timer *t) btime settled_time = rt_settled_time(tab); if (current_time() < settled_time) { - tm_set(tab->settle_timer, settled_time); + tm_set_in(tab->settle_timer, settled_time, tab->loop); return; } @@ -1836,11 +2112,11 @@ rt_settle_timer(timer *t) struct rt_subscription *s; WALK_LIST(s, tab->subscribers) - s->hook(s); + ev_send(s->event->list, s->event); } static void -rt_kick_settle_timer(rtable *tab) +rt_kick_settle_timer(rtable_private *tab) { tab->base_settle_time = current_time(); @@ -1848,11 +2124,11 @@ rt_kick_settle_timer(rtable *tab) tab->settle_timer = tm_new_init(tab->rp, rt_settle_timer, tab, 0, 0); if (!tm_active(tab->settle_timer)) - tm_set(tab->settle_timer, rt_settled_time(tab)); + tm_set_in(tab->settle_timer, rt_settled_time(tab), tab->loop); } static inline void -rt_schedule_notify(rtable *tab) +rt_schedule_notify(rtable_private *tab) { if (EMPTY_LIST(tab->subscribers)) return; @@ -1864,33 +2140,40 @@ rt_schedule_notify(rtable *tab) } void -rt_subscribe(rtable *tab, struct rt_subscription *s) +rt_subscribe(rtable *t, struct rt_subscription *s) { - s->tab = tab; - rt_lock_table(tab); - add_tail(&tab->subscribers, &s->n); + s->tab = t; + RT_LOCKED(t, tab) + { + rt_lock_table(tab); + DBG("rt_subscribe(%s)\n", tab->name); + add_tail(&tab->subscribers, &s->n); + } } void rt_unsubscribe(struct rt_subscription *s) { - rem_node(&s->n); - rt_unlock_table(s->tab); + RT_LOCKED(s->tab, tab) + { + rem_node(&s->n); + if (EMPTY_LIST(tab->subscribers) && tm_active(tab->settle_timer)) + tm_stop(tab->settle_timer); + rt_unlock_table(tab); + } } static void rt_free(resource *_r) { - rtable *r = (rtable *) _r; + rtable_private *r = (rtable_private *) _r; DBG("Deleting routing table %s\n", r->name); ASSERT_DIE(r->use_count == 0); - - if (r->internal) - return; - - r->config->table = NULL; - rem_node(&r->n); + ASSERT_DIE(r->rt_count == 0); + ASSERT_DIE(!r->cork_active); + ASSERT_DIE(EMPTY_LIST(r->imports)); + ASSERT_DIE(EMPTY_LIST(r->exports)); if (r->hostcache) rt_free_hostcache(r); @@ -1907,14 +2190,14 @@ rt_free(resource *_r) static void rt_res_dump(resource *_r) { - rtable *r = (rtable *) _r; + rtable_private *r = RT_PRIV((rtable *) _r); debug("name \"%s\", addr_type=%s, rt_count=%u, use_count=%d\n", r->name, net_label[r->addr_type], r->rt_count, r->use_count); } static struct resclass rt_class = { .name = "Routing table", - .size = sizeof(struct rtable), + .size = sizeof(rtable_private), .free = rt_free, .dump = rt_res_dump, .lookup = NULL, @@ -1928,11 +2211,16 @@ rt_setup(pool *pp, struct rtable_config *cf) void *nb = mb_alloc(pp, ns); ASSERT_DIE(ns - 1 == bsnprintf(nb, ns, "Routing table %s", cf->name)); - pool *p = rp_new(pp, nb); - mb_move(nb, p); + struct birdloop *l = birdloop_new(pp, DOMAIN_ORDER(rtable), nb); + pool *p = birdloop_pool(l); + + birdloop_enter(l); - rtable *t = ralloc(p, &rt_class); + rtable_private *t = ralloc(p, &rt_class); t->rp = p; + t->loop = l; + + t->rte_slab = sl_new(p, sizeof(struct rte_storage)); t->name = cf->name; t->config = cf; @@ -1940,19 +2228,35 @@ rt_setup(pool *pp, struct rtable_config *cf) fib_init(&t->fib, p, t->addr_type, sizeof(net), OFFSETOF(net, n), 0, NULL); - if (!(t->internal = cf->internal)) - { - init_list(&t->channels); - hmap_init(&t->id_map, p, 1024); - hmap_set(&t->id_map, 0); + init_list(&t->imports); + init_list(&t->exports); - init_list(&t->subscribers); + hmap_init(&t->id_map, p, 1024); + hmap_set(&t->id_map, 0); - t->rt_event = ev_new_init(p, rt_event, t); - t->last_rt_change = t->gc_time = current_time(); - } + init_list(&t->pending_exports); + init_list(&t->subscribers); + + t->announce_event = ev_new_init(p, rt_announce_exports, t); + t->ec_event = ev_new_init(p, rt_export_cleanup, t); + t->prune_event = ev_new_init(p, rt_prune_table, t); + t->hcu_event = ev_new_init(p, rt_update_hostcache, t); + t->nhu_event = ev_new_init(p, rt_next_hop_update, t); + + t->nhu_event->cork = &rt_cork; + t->prune_event->cork = &rt_cork; + + t->last_rt_change = t->gc_time = current_time(); + t->next_export_seq = 1; - return t; + t->rl_pipe = (struct tbf) TBF_DEFAULT_LOG_LIMITS; + + t->nhu_lp = lp_new_default(p); + + mb_move(nb, p); + birdloop_leave(l); + + return (rtable *) t; } /** @@ -1965,13 +2269,11 @@ void rt_init(void) { rta_init(); - rt_table_pool = rp_new(&root_pool, "Routing tables"); - rte_update_pool = lp_new_default(rt_table_pool); - rte_slab = sl_new(rt_table_pool, sizeof(rte)); + rt_table_pool = rp_new(&root_pool, &main_birdloop, "Routing tables"); init_list(&routing_tables); + ev_init_cork(&rt_cork, "Route Table Cork"); } - /** * rt_prune_table - prune a routing table * @@ -1987,12 +2289,15 @@ rt_init(void) * iteration. */ static void -rt_prune_table(rtable *tab) +rt_prune_table(void *data) { + rtable_private *tab = data; + ASSERT_DIE(birdloop_inside(tab->loop)); + struct fib_iterator *fit = &tab->prune_fit; - int limit = 512; + int limit = tab->delete ? 16384 : 512; - struct channel *c; + struct rt_import_hook *ih; node *n, *x; DBG("Pruning route table %s\n", tab->name); @@ -2003,12 +2308,24 @@ rt_prune_table(rtable *tab) if (tab->prune_state == 0) return; + if (tab->delete && !rt_fast_prune_ready(tab)) + return; + + rt_lock_table(tab); + if (tab->prune_state == 1) { /* Mark channels to flush */ - WALK_LIST2(c, n, tab->channels, table_node) - if (c->channel_state == CS_FLUSHING) - c->flush_active = 1; + WALK_LIST2(ih, n, tab->imports, n) + if (ih->import_state == TIS_STOP) + rt_set_import_state(ih, TIS_FLUSHING); + else if ((ih->stale_valid != ih->stale_pruning) && (ih->stale_pruning == ih->stale_pruned)) + { + ih->stale_pruning = ih->stale_valid; + + if (ih->req->trace_routes & D_STATES) + log(L_TRACE "%s: table prune after refresh begin [%u]", ih->req->name, ih->stale_pruning); + } FIB_ITERATE_INIT(fit, &tab->fib); tab->prune_state = 2; @@ -2017,43 +2334,53 @@ rt_prune_table(rtable *tab) again: FIB_ITERATE_START(&tab->fib, fit, net, n) { - rte *e; - - rescan: - for (e=n->routes; e; e=e->next) + if (tab->delete) { - if (e->sender->flush_active || (e->flags & REF_DISCARD)) - { - if (limit <= 0) - { - FIB_ITERATE_PUT(fit); - ev_schedule(tab->rt_event); - return; - } + ASSERT_DIE(!n->first); - rte_discard(e); - limit--; + for (struct rte_storage *e = n->routes, *next; e; e = next) + { + next = e->next; - goto rescan; - } + struct rt_import_request *req = e->rte.sender->req; + if (req->preimport) + req->preimport(req, NULL, &e->rte); - if (e->flags & REF_MODIFY) + tab->rt_count--; + hmap_clear(&tab->id_map, e->rte.id); + rte_free(e, tab); + limit--; + } + + n->routes = NULL; + } + else + rescan: + for (struct rte_storage *e=n->routes; e; e=e->next) + { + struct rt_import_hook *s = e->rte.sender; + + if ((s->import_state == TIS_FLUSHING) || + (e->rte.stale_cycle < s->stale_valid) || + (e->rte.stale_cycle > s->stale_set)) { if (limit <= 0) { FIB_ITERATE_PUT(fit); - ev_schedule(tab->rt_event); + ev_send_loop(tab->loop, tab->prune_event); + ev_send_loop(tab->loop, tab->announce_event); + rt_unlock_table(tab); return; } - rte_modify(e); + rte_recalculate(tab, e->rte.sender, n, NULL, e->rte.src); limit--; goto rescan; } } - if (!n->routes) /* Orphaned FIB entry */ + if (!n->routes && !n->first) /* Orphaned FIB entry */ { FIB_ITERATE_PUT(fit); fib_delete(&tab->fib, n); @@ -2070,23 +2397,205 @@ again: tab->gc_time = current_time(); /* state change 2->0, 3->1 */ - tab->prune_state &= 1; - - if (tab->prune_state > 0) - ev_schedule(tab->rt_event); + if (tab->prune_state &= 1) + ev_send_loop(tab->loop, tab->prune_event); - /* FIXME: This should be handled in a better way */ - rt_prune_sources(); + uint flushed_channels = 0; /* Close flushed channels */ - WALK_LIST2_DELSAFE(c, n, x, tab->channels, table_node) - if (c->flush_active) + WALK_LIST2_DELSAFE(ih, n, x, tab->imports, n) + if (ih->import_state == TIS_FLUSHING) + { + ih->flush_seq = tab->next_export_seq; + rt_set_import_state(ih, TIS_WAITING); + flushed_channels++; + } + else if (ih->stale_pruning != ih->stale_pruned) + { + ih->stale_pruned = ih->stale_pruning; + + if (ih->req->trace_routes & D_STATES) + log(L_TRACE "%s: table prune after refresh end [%u]", ih->req->name, ih->stale_pruned); + } + + /* In some cases, we may want to directly proceed to export cleanup */ + if (EMPTY_LIST(tab->exports) && flushed_channels) + rt_export_cleanup(tab); + + ev_send_loop(tab->loop, tab->announce_event); + rt_unlock_table(tab); +} + +static void +rt_export_cleanup(void *data) +{ + rtable_private *tab = data; + ASSERT_DIE(birdloop_inside(tab->loop)); + + u64 min_seq = ~((u64) 0); + struct rt_pending_export *last_export_to_free = NULL; + struct rt_pending_export *first_export = tab->first_export; + + struct rt_export_hook *eh; + node *n; + WALK_LIST2(eh, n, tab->exports, n) + { + switch (atomic_load_explicit(&eh->export_state, memory_order_acquire)) + { + case TES_DOWN: + case TES_HUNGRY: + continue; + + case TES_READY: + { + struct rt_pending_export *last = atomic_load_explicit(&eh->last_export, memory_order_acquire); + if (!last) + /* No last export means that the channel has exported nothing since last cleanup */ + goto done; + + else if (min_seq > last->seq) + { + min_seq = last->seq; + last_export_to_free = last; + } + continue; + } + + default: + /* It's only safe to cleanup when the export state is idle or regular. No feeding or stopping allowed. */ + goto done; + } + } + + tab->first_export = last_export_to_free ? rt_next_export_fast(last_export_to_free) : NULL; + + if (config->table_debug) + log(L_TRACE "%s: Export cleanup, old first_export seq %lu, new %lu, min_seq %ld", + tab->name, + first_export ? first_export->seq : 0, + tab->first_export ? tab->first_export->seq : 0, + min_seq); + + WALK_LIST2(eh, n, tab->exports, n) + { + if (atomic_load_explicit(&eh->export_state, memory_order_acquire) != TES_READY) + continue; + + struct rt_pending_export *last = atomic_load_explicit(&eh->last_export, memory_order_acquire); + if (last == last_export_to_free) + { + /* This may fail when the channel managed to export more inbetween. This is OK. */ + atomic_compare_exchange_strong_explicit( + &eh->last_export, &last, NULL, + memory_order_release, + memory_order_relaxed); + + DBG("store hook=%p last_export=NULL\n", eh); + } + } + + while (first_export && (first_export->seq <= min_seq)) + { + ASSERT_DIE(first_export->new || first_export->old); + + const net_addr *n = first_export->new ? + first_export->new->rte.net : + first_export->old->rte.net; + net *net = SKIP_BACK(struct network, n.addr, (net_addr (*)[0]) n); + + ASSERT_DIE(net->first == first_export); + + if (first_export == net->last) + /* The only export here */ + net->last = net->first = NULL; + else + /* First is now the next one */ + net->first = atomic_load_explicit(&first_export->next, memory_order_relaxed); + + /* For now, the old route may be finally freed */ + if (first_export->old) + { + rt_rte_trace_in(D_ROUTES, first_export->old->rte.sender->req, &first_export->old->rte, "freed"); + hmap_clear(&tab->id_map, first_export->old->rte.id); + rte_free(first_export->old, tab); + } + +#ifdef LOCAL_DEBUG + memset(first_export, 0xbd, sizeof(struct rt_pending_export)); +#endif + + struct rt_export_block *reb = HEAD(tab->pending_exports); + ASSERT_DIE(reb == PAGE_HEAD(first_export)); + + u32 pos = (first_export - &reb->export[0]); + u32 end = atomic_load_explicit(&reb->end, memory_order_relaxed); + ASSERT_DIE(pos < end); + + struct rt_pending_export *next = NULL; + + if (++pos < end) + next = &reb->export[pos]; + else + { + rem_node(&reb->n); + +#ifdef LOCAL_DEBUG + memset(reb, 0xbe, page_size); +#endif + + free_page(reb); + + if (EMPTY_LIST(tab->pending_exports)) + { + if (config->table_debug) + log(L_TRACE "%s: Resetting export seq", tab->name); + + node *n; + WALK_LIST2(eh, n, tab->exports, n) + { + if (atomic_load_explicit(&eh->export_state, memory_order_acquire) != TES_READY) + continue; + + ASSERT_DIE(atomic_load_explicit(&eh->last_export, memory_order_acquire) == NULL); + bmap_reset(&eh->seq_map, 1024); + } + + tab->next_export_seq = 1; + } + else { - c->flush_active = 0; - channel_set_state(c, CS_DOWN); + reb = HEAD(tab->pending_exports); + next = &reb->export[0]; } + } - return; + first_export = next; + } + +done:; + struct rt_import_hook *ih; node *x; + WALK_LIST2_DELSAFE(ih, n, x, tab->imports, n) + if (ih->import_state == TIS_WAITING) + if (!first_export || (first_export->seq >= ih->flush_seq)) + { + ih->import_state = TIS_CLEARED; + ev_send(ih->req->list, ih->export_announce_event); + } + + if (EMPTY_LIST(tab->pending_exports) && ev_active(tab->announce_event)) + ev_postpone(tab->announce_event); + + /* If reduced to at most one export block pending */ + if (tab->cork_active && + ((!tab->first_export) || (tab->first_export->seq + 128 > tab->next_export_seq))) + { + tab->cork_active = 0; + ev_uncork(&rt_cork); + if (config->table_debug) + log(L_TRACE "%s: cork released", tab->name); + } + + rt_fast_prune_check(tab); } void @@ -2120,7 +2629,7 @@ rta_next_hop_outdated(rta *a) } void -rta_apply_hostentry(rta *a, struct hostentry *he, mpls_label_stack *mls) +rta_apply_hostentry(rta *a, struct hostentry *he, mpls_label_stack *mls, linpool *lp) { a->hostentry = he; a->dest = he->dest; @@ -2155,7 +2664,7 @@ no_nexthop: else { nhr = nhp; - nhp = (nhp ? (nhp->next = lp_alloc(rte_update_pool, NEXTHOP_MAX_SIZE)) : &(a->nh)); + nhp = (nhp ? (nhp->next = lp_alloc(lp, NEXTHOP_MAX_SIZE)) : &(a->nh)); } memset(nhp, 0, NEXTHOP_MAX_SIZE); @@ -2210,8 +2719,8 @@ no_nexthop: } } -static inline rte * -rt_next_hop_update_rte(rtable *tab UNUSED, rte *old) +static inline struct rte_storage * +rt_next_hop_update_rte(rtable_private *tab, net *n, rte *old) { rta *a = alloca(RTA_MAX_SIZE); memcpy(a, old->attrs, rta_size(old->attrs)); @@ -2219,58 +2728,71 @@ rt_next_hop_update_rte(rtable *tab UNUSED, rte *old) mpls_label_stack mls = { .len = a->nh.labels_orig }; memcpy(mls.stack, &a->nh.label[a->nh.labels - mls.len], mls.len * sizeof(u32)); - rta_apply_hostentry(a, old->attrs->hostentry, &mls); - a->aflags = 0; + rta_apply_hostentry(a, old->attrs->hostentry, &mls, tab->nhu_lp); + a->cached = 0; - rte *e = sl_alloc(rte_slab); - memcpy(e, old, sizeof(rte)); - e->attrs = rta_lookup(a); + rte e0 = *old; + e0.attrs = rta_lookup(a); - return e; + return rte_store(&e0, n, tab); } static inline int -rt_next_hop_update_net(rtable *tab, net *n) +rt_next_hop_update_net(rtable_private *tab, net *n) { - rte **k, *e, *new, *old_best, **new_best; + struct rte_storage *new; int count = 0; - int free_old_best = 0; - old_best = n->routes; + struct rte_storage *old_best = n->routes; if (!old_best) return 0; - for (k = &n->routes; e = *k; k = &e->next) - if (rta_next_hop_outdated(e->attrs)) - { - new = rt_next_hop_update_rte(tab, e); - *k = new; + for (struct rte_storage *e, **k = &n->routes; e = *k; k = &e->next) + if (rta_next_hop_outdated(e->rte.attrs)) + count++; + + if (!count) + return 0; - rte_trace_in(D_ROUTES, new->sender, new, "updated"); - rte_announce_i(tab, RA_ANY, n, new, e, NULL, NULL); + struct rte_multiupdate { + struct rte_storage *old, *new; + } *updates = alloca(sizeof(struct rte_multiupdate) * count); + + int pos = 0; + for (struct rte_storage *e, **k = &n->routes; e = *k; k = &e->next) + if (rta_next_hop_outdated(e->rte.attrs)) + { + struct rte_storage *new = rt_next_hop_update_rte(tab, n, &e->rte); /* Call a pre-comparison hook */ /* Not really an efficient way to compute this */ - if (e->attrs->src->proto->rte_recalculate) - e->attrs->src->proto->rte_recalculate(tab, n, new, e, NULL); + if (e->rte.src->owner->rte_recalculate) + e->rte.src->owner->rte_recalculate(tab, n, &new->rte, &e->rte, &old_best->rte); - if (e != old_best) - rte_free_quick(e); - else /* Freeing of the old best rte is postponed */ - free_old_best = 1; + updates[pos++] = (struct rte_multiupdate) { + .old = e, + .new = new, + }; - e = new; - count++; + /* Replace the route in the list */ + new->next = e->next; + *k = e = new; + + /* Get a new ID for the route */ + new->rte.lastmod = current_time(); + new->rte.id = hmap_first_zero(&tab->id_map); + hmap_set(&tab->id_map, new->rte.id); + + lp_flush(tab->nhu_lp); } - if (!count) - return 0; + ASSERT_DIE(pos == count); /* Find the new best route */ - new_best = NULL; - for (k = &n->routes; e = *k; k = &e->next) + struct rte_storage **new_best = NULL; + for (struct rte_storage *e, **k = &n->routes; e = *k; k = &e->next) { - if (!new_best || rte_better(e, *new_best)) + if (!new_best || rte_better(&e->rte, &(*new_best)->rte)) new_best = k; } @@ -2283,32 +2805,39 @@ rt_next_hop_update_net(rtable *tab, net *n) n->routes = new; } - /* Announce the new best route */ - if (new != old_best) - rte_trace_in(D_ROUTES, new->sender, new, "updated [best]"); - - /* Propagate changes */ - rte_announce_i(tab, RA_UNDEF, n, NULL, NULL, n->routes, old_best); - - if (free_old_best) - rte_free_quick(old_best); + /* Announce the changes */ + for (int i=0; i<count; i++) + { + _Bool nb = (new == updates[i].new), ob = (old_best == updates[i].old); + const char *best_indicator[2][2] = { + { "autoupdated", "autoupdated [-best]" }, + { "autoupdated [+best]", "autoupdated [best]" } + }; + rt_rte_trace_in(D_ROUTES, updates[i].new->rte.sender->req, &updates[i].new->rte, best_indicator[nb][ob]); + rte_announce(tab, n, updates[i].new, updates[i].old, new, old_best); + } return count; } static void -rt_next_hop_update(rtable *tab) +rt_next_hop_update(void *data) { + rtable_private *tab = data; + ASSERT_DIE(birdloop_inside(tab->loop)); + struct fib_iterator *fit = &tab->nhu_fit; int max_feed = 32; - if (tab->nhu_state == NHU_CLEAN) + if (atomic_load_explicit(&tab->nhu_state, memory_order_acquire) == NHU_CLEAN) return; - if (tab->nhu_state == NHU_SCHEDULED) + rt_lock_table(tab); + + if (atomic_load_explicit(&tab->nhu_state, memory_order_acquire) == NHU_SCHEDULED) { FIB_ITERATE_INIT(fit, &tab->fib); - tab->nhu_state = NHU_RUNNING; + ASSERT_DIE(atomic_exchange_explicit(&tab->nhu_state, NHU_RUNNING, memory_order_acq_rel) == NHU_SCHEDULED); } FIB_ITERATE_START(&tab->fib, fit, net, n) @@ -2316,7 +2845,8 @@ rt_next_hop_update(rtable *tab) if (max_feed <= 0) { FIB_ITERATE_PUT(fit); - ev_schedule(tab->rt_event); + ev_send_loop(tab->loop, tab->nhu_event); + rt_unlock_table(tab); return; } max_feed -= rt_next_hop_update_net(tab, n); @@ -2327,10 +2857,12 @@ rt_next_hop_update(rtable *tab) * NHU_DIRTY -> NHU_SCHEDULED * NHU_RUNNING -> NHU_CLEAN */ - tab->nhu_state &= 1; + if (atomic_fetch_and_explicit(&tab->nhu_state, NHU_SCHEDULED, memory_order_acq_rel) != NHU_RUNNING) + ev_send_loop(tab->loop, tab->nhu_event); + + ev_send_loop(tab->loop, tab->announce_event); - if (tab->nhu_state != NHU_CLEAN) - ev_schedule(tab->rt_event); + rt_unlock_table(tab); } @@ -2352,6 +2884,10 @@ rt_new_table(struct symbol *s, uint addr_type) c->gc_min_time = 5; c->min_settle_time = 1 S; c->max_settle_time = 20 S; + c->min_rr_settle_time = 30 S; + c->max_rr_settle_time = 90 S; + c->cork_limit = 4 * page_size / sizeof(struct rt_pending_export); + c->owner = new_config; add_tail(&new_config->tables, &c->n); @@ -2371,7 +2907,7 @@ rt_new_table(struct symbol *s, uint addr_type) * configuration. */ void -rt_lock_table(rtable *r) +rt_lock_table(rtable_private *r) { r->use_count++; } @@ -2385,16 +2921,11 @@ rt_lock_table(rtable *r) * for deletion by configuration changes. */ void -rt_unlock_table(rtable *r) +rt_unlock_table(rtable_private *r) { - if (!--r->use_count && r->deleted) - { - struct config *conf = r->deleted; - - /* Delete the routing table by freeing its pool */ - rt_shutdown(r); - config_del_obstacle(conf); - } + if (!--r->use_count && r->delete && + !r->prune_state && !atomic_load_explicit(&r->nhu_state, memory_order_acquire)) + birdloop_stop_self(r->loop, r->delete, r); } static struct rtable_config * @@ -2404,6 +2935,21 @@ rt_find_table_config(struct config *cf, char *name) return (sym && (sym->class == SYM_TABLE)) ? sym->table : NULL; } +static void +rt_done(void *data) +{ + RT_LOCKED((rtable *) data, t) + { + struct rtable_config *tc = t->config; + struct config *c = tc->owner; + + tc->table = NULL; + rem_node(&t->n); + + config_del_obstacle(c); + } +} + /** * rt_commit - commit new routing table configuration * @new: new configuration @@ -2426,14 +2972,15 @@ rt_commit(struct config *new, struct config *old) { WALK_LIST(o, old->tables) { - rtable *ot = o->table; - if (!ot->deleted) + RT_LOCK(o->table); + rtable_private *ot = RT_PRIV(o->table); + if (!ot->delete) { r = rt_find_table_config(new, o->name); if (r && (r->addr_type == o->addr_type) && !new->shutdown) { DBG("\t%s: same\n", o->name); - r->table = ot; + r->table = (rtable *) ot; ot->name = r->name; ot->config = r; if (o->sorted != r->sorted) @@ -2442,12 +2989,13 @@ rt_commit(struct config *new, struct config *old) else { DBG("\t%s: deleted\n", o->name); - ot->deleted = old; - config_add_obstacle(old); rt_lock_table(ot); + ot->delete = rt_done; + config_add_obstacle(old); rt_unlock_table(ot); } } + RT_UNLOCK(o->table); } } @@ -2461,19 +3009,6 @@ rt_commit(struct config *new, struct config *old) DBG("\tdone\n"); } -static inline void -do_feed_channel(struct channel *c, net *n, rte *e) -{ - rte_update_lock(); - if (c->ra_mode == RA_ACCEPTED) - rt_notify_accepted(c, n, NULL, NULL, c->refeeding); - else if (c->ra_mode == RA_MERGED) - rt_notify_merged(c, n, NULL, NULL, e, e, c->refeeding); - else /* RA_BASIC */ - rt_notify_basic(c, n, e, e, c->refeeding); - rte_update_unlock(); -} - /** * rt_feed_channel - advertise all routes to a channel * @c: channel to be fed @@ -2483,370 +3018,120 @@ do_feed_channel(struct channel *c, net *n, rte *e) * has something to do. (We avoid transferring all the routes in single pass in * order not to monopolize CPU time.) */ -int -rt_feed_channel(struct channel *c) +static void +rt_feed_channel(void *data) { + struct rt_export_hook *c = data; + struct fib_iterator *fit = &c->feed_fit; int max_feed = 256; - ASSERT(c->export_state == ES_FEEDING); - - if (!c->feed_active) - { - FIB_ITERATE_INIT(fit, &c->table->fib); - c->feed_active = 1; - } - - FIB_ITERATE_START(&c->table->fib, fit, net, n) - { - rte *e = n->routes; - if (max_feed <= 0) - { - FIB_ITERATE_PUT(fit); - return 0; - } - - if ((c->ra_mode == RA_OPTIMAL) || - (c->ra_mode == RA_ACCEPTED) || - (c->ra_mode == RA_MERGED)) - if (rte_is_valid(e)) - { - /* In the meantime, the protocol may fell down */ - if (c->export_state != ES_FEEDING) - goto done; - - do_feed_channel(c, n, e); - max_feed--; - } - - if (c->ra_mode == RA_ANY) - for(e = n->routes; e; e = e->next) - { - /* In the meantime, the protocol may fell down */ - if (c->export_state != ES_FEEDING) - goto done; - - if (!rte_is_valid(e)) - continue; - - do_feed_channel(c, n, e); - max_feed--; - } - } - FIB_ITERATE_END; - -done: - c->feed_active = 0; - return 1; -} - -/** - * rt_feed_baby_abort - abort protocol feeding - * @c: channel - * - * This function is called by the protocol code when the protocol stops or - * ceases to exist during the feeding. - */ -void -rt_feed_channel_abort(struct channel *c) -{ - if (c->feed_active) - { - /* Unlink the iterator */ - fit_get(&c->table->fib, &c->feed_fit); - c->feed_active = 0; - } -} - - -/* - * Import table - */ - -int -rte_update_in(struct channel *c, const net_addr *n, rte *new, struct rte_src *src) -{ - struct rtable *tab = c->in_table; - rte *old, **pos; - net *net; - - if (new) + rtable_private *tab; + if (c->export_state == TES_HUNGRY) { - net = net_get(tab, n); + rt_set_export_state(c, TES_FEEDING); - if (!new->pref) - new->pref = c->preference; + tab = RT_LOCK(c->table); - if (!rta_is_cached(new->attrs)) - new->attrs = rta_lookup(new->attrs); + struct rt_pending_export *rpe = rt_last_export(tab); + DBG("store hook=%p last_export=%p seq=%lu\n", c, rpe, rpe ? rpe->seq : 0); + atomic_store_explicit(&c->last_export, rpe, memory_order_relaxed); + + FIB_ITERATE_INIT(&c->feed_fit, &tab->fib); } else - { - net = net_find(tab, n); + tab = RT_LOCK(c->table); - if (!net) - goto drop_withdraw; - } + ASSERT_DIE(c->export_state == TES_FEEDING); - /* Find the old rte */ - for (pos = &net->routes; old = *pos; pos = &old->next) - if (old->attrs->src == src) +redo: + FIB_ITERATE_START(&tab->fib, fit, net, n) { - if (new && rte_same(old, new)) - { - /* Refresh the old rte, continue with update to main rtable */ - if (old->flags & (REF_STALE | REF_DISCARD | REF_MODIFY)) + if (max_feed <= 0) { - old->flags &= ~(REF_STALE | REF_DISCARD | REF_MODIFY); - return 1; - } - - goto drop_update; - } - - /* Move iterator if needed */ - if (old == c->reload_next_rte) - c->reload_next_rte = old->next; - - /* Remove the old rte */ - *pos = old->next; - rte_free_quick(old); - tab->rt_count--; - - break; - } - - if (!new) - { - if (!old) - goto drop_withdraw; - - if (!net->routes) - fib_delete(&tab->fib, net); - - return 1; - } - - struct channel_limit *l = &c->rx_limit; - if (l->action && !old) - { - if (tab->rt_count >= l->limit) - channel_notify_limit(c, l, PLD_RX, tab->rt_count); - - if (l->state == PLS_BLOCKED) - { - /* Required by rte_trace_in() */ - new->net = net; - - rte_trace_in(D_FILTERS, c, new, "ignored [limit]"); - goto drop_update; - } - } - - /* Insert the new rte */ - rte *e = rte_do_cow(new); - e->flags |= REF_COW; - e->net = net; - e->sender = c; - e->lastmod = current_time(); - e->next = *pos; - *pos = e; - tab->rt_count++; - return 1; - -drop_update: - c->stats.imp_updates_received++; - c->stats.imp_updates_ignored++; - rte_free(new); - - if (!net->routes) - fib_delete(&tab->fib, net); - - return 0; - -drop_withdraw: - c->stats.imp_withdraws_received++; - c->stats.imp_withdraws_ignored++; - return 0; -} - -int -rt_reload_channel(struct channel *c) -{ - struct rtable *tab = c->in_table; - struct fib_iterator *fit = &c->reload_fit; - int max_feed = 64; - - ASSERT(c->channel_state == CS_UP); + FIB_ITERATE_PUT(fit); + rt_send_export_event(c); - if (!c->reload_active) - { - FIB_ITERATE_INIT(fit, &tab->fib); - c->reload_active = 1; - } + RT_UNLOCK(c->table); + return; + } - do { - for (rte *e = c->reload_next_rte; e; e = e->next) - { - if (max_feed-- <= 0) + if (atomic_load_explicit(&c->export_state, memory_order_acquire) != TES_FEEDING) { - c->reload_next_rte = e; - debug("%s channel reload burst split (max_feed=%d)", c->proto->name, max_feed); - return 0; + RT_UNLOCK(c->table); + return; } - rte_update2(c, e->net->n.addr, rte_do_cow(e), e->attrs->src); - } - - c->reload_next_rte = NULL; - - FIB_ITERATE_START(&tab->fib, fit, net, n) - { - if (c->reload_next_rte = n->routes) + if (!n->routes || !rte_is_valid(&n->routes->rte)) + ; /* if no route, do nothing */ + else if (c->req->export_bulk) { - FIB_ITERATE_PUT_NEXT(fit, &tab->fib); - break; - } - } - FIB_ITERATE_END; - } - while (c->reload_next_rte); - - c->reload_active = 0; - return 1; -} - -void -rt_reload_channel_abort(struct channel *c) -{ - if (c->reload_active) - { - /* Unlink the iterator */ - fit_get(&c->in_table->fib, &c->reload_fit); - c->reload_next_rte = NULL; - c->reload_active = 0; - } -} - -void -rt_prune_sync(rtable *t, int all) -{ - struct fib_iterator fit; - - FIB_ITERATE_INIT(&fit, &t->fib); + uint count = rte_feed_count(n); + if (count) + { + rte **feed = alloca(count * sizeof(rte *)); + rte_feed_obtain(n, feed, count); -again: - FIB_ITERATE_START(&t->fib, &fit, net, n) - { - rte *e, **ee = &n->routes; + struct rt_pending_export *rpe_last, *rpe_first = n->first; + for (struct rt_pending_export *rpe = rpe_first; rpe; rpe = rpe_next(rpe, NULL)) + rpe_last = rpe; - while (e = *ee) - { - if (all || (e->flags & (REF_STALE | REF_DISCARD))) - { - *ee = e->next; - rte_free_quick(e); - t->rt_count--; - } - else - ee = &e->next; - } + FIB_ITERATE_PUT_NEXT(fit, &tab->fib); + RT_UNLOCK(c->table); - if (all || !n->routes) - { - FIB_ITERATE_PUT(&fit); - fib_delete(&t->fib, n); - goto again; - } - } - FIB_ITERATE_END; -} + c->req->export_bulk(c->req, n->n.addr, NULL, feed, count); + RT_LOCK(c->table); -/* - * Export table - */ + for (struct rt_pending_export *rpe = rpe_first; rpe; rpe = rpe_next(rpe, NULL)) + { + rpe_mark_seen(c, rpe); + if (rpe == rpe_last) + break; + ASSERT_DIE(rpe->seq < rpe_last->seq); + } -int -rte_update_out(struct channel *c, const net_addr *n, rte *new, rte *old0, int refeed) -{ - struct rtable *tab = c->out_table; - struct rte_src *src; - rte *old, **pos; - net *net; + max_feed -= count; - if (new) - { - net = net_get(tab, n); - src = new->attrs->src; + goto redo; + } + } + else if (c->req->export_one) + { + struct rt_pending_export rpe = { .new = n->routes, .new_best = n->routes }; - rte_store_tmp_attrs(new, rte_update_pool, NULL); + struct rt_pending_export *rpe_last, *rpe_first = n->first; + for (struct rt_pending_export *rpe = rpe_first; rpe; rpe = rpe_next(rpe, NULL)) + rpe_last = rpe; - if (!rta_is_cached(new->attrs)) - new->attrs = rta_lookup(new->attrs); - } - else - { - net = net_find(tab, n); - src = old0->attrs->src; + FIB_ITERATE_PUT_NEXT(fit, &tab->fib); + RT_UNLOCK(c->table); - if (!net) - goto drop_withdraw; - } + c->req->export_one(c->req, n->n.addr, &rpe); - /* Find the old rte */ - for (pos = &net->routes; old = *pos; pos = &old->next) - if ((c->ra_mode != RA_ANY) || (old->attrs->src == src)) - { - if (new && rte_same(old, new)) - { - /* REF_STALE / REF_DISCARD not used in export table */ - /* - if (old->flags & (REF_STALE | REF_DISCARD | REF_MODIFY)) + RT_LOCK(c->table); + for (struct rt_pending_export *rpe = rpe_first; rpe; rpe = rpe_next(rpe, NULL)) { - old->flags &= ~(REF_STALE | REF_DISCARD | REF_MODIFY); - return 1; + rpe_mark_seen(c, rpe); + if (rpe == rpe_last) + break; + ASSERT_DIE(rpe->seq < rpe_last->seq); } - */ - goto drop_update; + max_feed--; + goto redo; } - - /* Remove the old rte */ - *pos = old->next; - rte_free_quick(old); - tab->rt_count--; - - break; + else + bug("Export request must always provide an export method"); } + FIB_ITERATE_END; - if (!new) - { - if (!old) - goto drop_withdraw; - - if (!net->routes) - fib_delete(&tab->fib, net); - - return 1; - } - - /* Insert the new rte */ - rte *e = rte_do_cow(new); - e->flags |= REF_COW; - e->net = net; - e->sender = c; - e->lastmod = current_time(); - e->next = *pos; - *pos = e; - tab->rt_count++; - return 1; + c->event->hook = rt_export_hook; + rt_send_export_event(c); -drop_update: - return refeed; + RT_UNLOCK(c->table); -drop_withdraw: - return 0; + rt_set_export_state(c, TES_READY); } @@ -2953,7 +3238,7 @@ hc_delete_hostentry(struct hostcache *hc, pool *p, struct hostentry *he) } static void -rt_init_hostcache(rtable *tab) +rt_init_hostcache(rtable_private *tab) { struct hostcache *hc = mb_allocz(tab->rp, sizeof(struct hostcache)); init_list(&hc->hostentries); @@ -2969,7 +3254,7 @@ rt_init_hostcache(rtable *tab) } static void -rt_free_hostcache(rtable *tab) +rt_free_hostcache(rtable_private *tab) { struct hostcache *hc = tab->hostcache; @@ -2992,13 +3277,13 @@ rt_free_hostcache(rtable *tab) } static void -rt_notify_hostcache(rtable *tab, net *net) +rt_notify_hostcache(rtable_private *tab, net *net) { - if (tab->hcu_scheduled) + if (ev_active(tab->hcu_event)) return; if (trie_match_net(tab->hostcache->trie, net->n.addr)) - rt_schedule_hcu(tab); + ev_send_loop(tab->loop, tab->hcu_event); } static int @@ -3021,41 +3306,17 @@ rt_get_igp_metric(rte *rt) if (ea) return ea->u.data; - rta *a = rt->attrs; - -#ifdef CONFIG_OSPF - if ((a->source == RTS_OSPF) || - (a->source == RTS_OSPF_IA) || - (a->source == RTS_OSPF_EXT1)) - return rt->u.ospf.metric1; -#endif - -#ifdef CONFIG_RIP - if (a->source == RTS_RIP) - return rt->u.rip.metric; -#endif - -#ifdef CONFIG_BGP - if (a->source == RTS_BGP) - { - u64 metric = bgp_total_aigp_metric(rt); - return (u32) MIN(metric, (u64) IGP_METRIC_UNKNOWN); - } -#endif - -#ifdef CONFIG_BABEL - if (a->source == RTS_BABEL) - return rt->u.babel.metric; -#endif - - if (a->source == RTS_DEVICE) + if (rt->attrs->source == RTS_DEVICE) return 0; + if (rt->src->owner->class->rte_igp_metric) + return rt->src->owner->class->rte_igp_metric(rt); + return IGP_METRIC_UNKNOWN; } static int -rt_update_hostentry(rtable *tab, struct hostentry *he) +rt_update_hostentry(rtable_private *tab, struct hostentry *he) { rta *old_src = he->src; int direct = 0; @@ -3072,11 +3333,12 @@ rt_update_hostentry(rtable *tab, struct hostentry *he) net *n = net_route(tab, &he_addr); if (n) { - rte *e = n->routes; - rta *a = e->attrs; - pxlen = n->n.addr->pxlen; + struct rte_storage *e = n->routes; + rta *a = e->rte.attrs; + word pref = a->pref; - if (a->hostentry) + for (struct rte_storage *ee = n->routes; ee; ee = ee->next) + if ((ee->rte.attrs->pref >= pref) && ee->rte.attrs->hostentry) { /* Recursive route should not depend on another recursive route */ log(L_WARN "Next hop address %I resolvable through recursive route for %N", @@ -3084,6 +3346,8 @@ rt_update_hostentry(rtable *tab, struct hostentry *he) goto done; } + pxlen = n->n.addr->pxlen; + if (a->dest == RTD_UNICAST) { for (struct nexthop *nh = &(a->nh); nh; nh = nh->next) @@ -3104,7 +3368,7 @@ rt_update_hostentry(rtable *tab, struct hostentry *he) he->src = rta_clone(a); he->dest = a->dest; he->nexthop_linkable = !direct; - he->igp_metric = rt_get_igp_metric(e); + he->igp_metric = rt_get_igp_metric(&e->rte); } done: @@ -3116,8 +3380,11 @@ done: } static void -rt_update_hostcache(rtable *tab) +rt_update_hostcache(void *data) { + rtable_private *tab = data; + ASSERT_DIE(birdloop_inside(tab->loop)); + struct hostcache *hc = tab->hostcache; struct hostentry *he; node *n, *x; @@ -3138,15 +3405,15 @@ rt_update_hostcache(rtable *tab) if (rt_update_hostentry(tab, he)) rt_schedule_nhu(he->tab); } - - tab->hcu_scheduled = 0; } struct hostentry * -rt_get_hostentry(rtable *tab, ip_addr a, ip_addr ll, rtable *dep) +rt_get_hostentry(rtable *t, ip_addr a, ip_addr ll, rtable *dep) { struct hostentry *he; + rtable_private *tab = RT_LOCK(t); + if (!tab->hostcache) rt_init_hostcache(tab); @@ -3154,10 +3421,13 @@ rt_get_hostentry(rtable *tab, ip_addr a, ip_addr ll, rtable *dep) struct hostcache *hc = tab->hostcache; for (he = hc->hash_table[k >> hc->hash_shift]; he != NULL; he = he->next) if (ipa_equal(he->addr, a) && (he->tab == dep)) - return he; + goto done; he = hc_new_hostentry(hc, tab->rp, a, ipa_zero(ll) ? a : ll, dep, k); rt_update_hostentry(tab, he); + +done: + RT_UNLOCK(t); return he; } diff --git a/proto/babel/babel.c b/proto/babel/babel.c index 1e87212c..6d2a593e 100644 --- a/proto/babel/babel.c +++ b/proto/babel/babel.c @@ -641,13 +641,36 @@ babel_announce_rte(struct babel_proto *p, struct babel_entry *e) if (r) { rta a0 = { - .src = p->p.main_source, .source = RTS_BABEL, .scope = SCOPE_UNIVERSE, .dest = RTD_UNICAST, + .pref = c->preference, .from = r->neigh->addr, .nh.gw = r->next_hop, .nh.iface = r->neigh->ifa->iface, + .eattrs = alloca(sizeof(ea_list) + 3*sizeof(eattr)), + }; + + *a0.eattrs = (ea_list) { .count = 3 }; + a0.eattrs->attrs[0] = (eattr) { + .id = EA_BABEL_METRIC, + .type = EAF_TYPE_INT, + .u.data = r->metric, + }; + + struct adata *ad = alloca(sizeof(struct adata) + sizeof(u64)); + ad->length = sizeof(u64); + memcpy(ad->data, &(r->router_id), sizeof(u64)); + a0.eattrs->attrs[1] = (eattr) { + .id = EA_BABEL_ROUTER_ID, + .type = EAF_TYPE_OPAQUE, + .u.ptr = ad, + }; + + a0.eattrs->attrs[2] = (eattr) { + .id = EA_BABEL_SEQNO, + .type = EAF_TYPE_INT, + .u.data = r->seqno, }; /* @@ -658,40 +681,37 @@ babel_announce_rte(struct babel_proto *p, struct babel_entry *e) if (!neigh_find(&p->p, r->next_hop, r->neigh->ifa->iface, 0)) a0.nh.flags = RNF_ONLINK; - rta *a = rta_lookup(&a0); - rte *rte = rte_get_temp(a); - rte->u.babel.seqno = r->seqno; - rte->u.babel.metric = r->metric; - rte->u.babel.router_id = r->router_id; - rte->pflags = EA_ID_FLAG(EA_BABEL_METRIC) | EA_ID_FLAG(EA_BABEL_ROUTER_ID); + rte e0 = { + .attrs = &a0, + .src = p->p.main_source, + }; e->unreachable = 0; - rte_update2(c, e->n.addr, rte, p->p.main_source); + rte_update(c, e->n.addr, &e0, p->p.main_source); } else if (e->valid && (e->router_id != p->router_id)) { /* Unreachable */ rta a0 = { - .src = p->p.main_source, .source = RTS_BABEL, .scope = SCOPE_UNIVERSE, .dest = RTD_UNREACHABLE, + .pref = 1, }; - rta *a = rta_lookup(&a0); - rte *rte = rte_get_temp(a); - memset(&rte->u.babel, 0, sizeof(rte->u.babel)); - rte->pflags = 0; - rte->pref = 1; + rte e0 = { + .attrs = &a0, + .src = p->p.main_source, + }; e->unreachable = 1; - rte_update2(c, e->n.addr, rte, p->p.main_source); + rte_update(c, e->n.addr, &e0, p->p.main_source); } else { /* Retraction */ e->unreachable = 0; - rte_update2(c, e->n.addr, NULL, p->p.main_source); + rte_update(c, e->n.addr, NULL, p->p.main_source); } } @@ -701,7 +721,7 @@ babel_announce_retraction(struct babel_proto *p, struct babel_entry *e) { struct channel *c = (e->n.addr->type == NET_IP4) ? p->ip4_channel : p->ip6_channel; e->unreachable = 0; - rte_update2(c, e->n.addr, NULL, p->p.main_source); + rte_update(c, e->n.addr, NULL, p->p.main_source); } @@ -1727,7 +1747,7 @@ babel_add_iface(struct babel_proto *p, struct iface *new, struct babel_iface_con TRACE(D_EVENTS, "Adding interface %s", new->name); - pool *pool = rp_new(p->p.pool, new->name); + pool *pool = rp_new(p->p.pool, p->p.loop, new->name); ifa = mb_allocz(pool, sizeof(struct babel_iface)); ifa->proto = p; @@ -1779,7 +1799,7 @@ babel_remove_iface(struct babel_proto *p, struct babel_iface *ifa) rem_node(NODE ifa); - rfree(ifa->pool); /* contains ifa itself, locks, socket, etc */ + rp_free(ifa->pool, p->p.pool); /* contains ifa itself, locks, socket, etc */ } static int @@ -1887,7 +1907,8 @@ babel_reconfigure_ifaces(struct babel_proto *p, struct babel_config *cf) { struct iface *iface; - WALK_LIST(iface, iface_list) + IFACE_LEGACY_ACCESS; + WALK_LIST(iface, global_iface_list) { if (!(iface->flags & IF_UP)) continue; @@ -2010,7 +2031,13 @@ babel_dump(struct proto *P) static void babel_get_route_info(rte *rte, byte *buf) { - buf += bsprintf(buf, " (%d/%d) [%lR]", rte->pref, rte->u.babel.metric, rte->u.babel.router_id); + u64 rid = 0; + eattr *e = ea_find(rte->attrs->eattrs, EA_BABEL_ROUTER_ID); + if (e) + memcpy(&rid, e->u.ptr->data, sizeof(u64)); + + buf += bsprintf(buf, " (%d/%d) [%lR]", rte->attrs->pref, + ea_get_int(rte->attrs->eattrs, EA_BABEL_METRIC, BABEL_INFINITY), rid); } static int @@ -2018,6 +2045,9 @@ babel_get_attr(const eattr *a, byte *buf, int buflen UNUSED) { switch (a->id) { + case EA_BABEL_SEQNO: + return GA_FULL; + case EA_BABEL_METRIC: bsprintf(buf, "metric: %d", a->u.data); return GA_FULL; @@ -2231,45 +2261,23 @@ babel_kick_timer(struct babel_proto *p) static int -babel_preexport(struct proto *P, struct rte **new, struct linpool *pool UNUSED) +babel_preexport(struct channel *c, struct rte *new) { - struct rta *a = (*new)->attrs; - + struct rta *a = new->attrs; /* Reject our own unreachable routes */ - if ((a->dest == RTD_UNREACHABLE) && (a->src->proto == P)) + if ((a->dest == RTD_UNREACHABLE) && (new->src->owner == &c->proto->sources)) return -1; return 0; } -static void -babel_make_tmp_attrs(struct rte *rt, struct linpool *pool) -{ - struct adata *id = lp_alloc_adata(pool, sizeof(u64)); - memcpy(id->data, &rt->u.babel.router_id, sizeof(u64)); - - rte_init_tmp_attrs(rt, pool, 2); - rte_make_tmp_attr(rt, EA_BABEL_METRIC, EAF_TYPE_INT, rt->u.babel.metric); - rte_make_tmp_attr(rt, EA_BABEL_ROUTER_ID, EAF_TYPE_OPAQUE, (uintptr_t) id); -} - -static void -babel_store_tmp_attrs(struct rte *rt, struct linpool *pool) -{ - rte_init_tmp_attrs(rt, pool, 2); - rt->u.babel.metric = rte_store_tmp_attr(rt, EA_BABEL_METRIC); - - /* EA_BABEL_ROUTER_ID is read-only, we do not really save the value */ - rte_store_tmp_attr(rt, EA_BABEL_ROUTER_ID); -} - /* * babel_rt_notify - core tells us about new route (possibly our own), * so store it into our data structures. */ static void -babel_rt_notify(struct proto *P, struct channel *c UNUSED, struct network *net, - struct rte *new, struct rte *old UNUSED) +babel_rt_notify(struct proto *P, struct channel *c UNUSED, const net_addr *net, + struct rte *new, const struct rte *old UNUSED) { struct babel_proto *p = (void *) P; struct babel_entry *e; @@ -2277,19 +2285,31 @@ babel_rt_notify(struct proto *P, struct channel *c UNUSED, struct network *net, if (new) { /* Update */ - uint internal = (new->attrs->src->proto == P); - uint rt_seqno = internal ? new->u.babel.seqno : p->update_seqno; + uint rt_seqno; uint rt_metric = ea_get_int(new->attrs->eattrs, EA_BABEL_METRIC, 0); - u64 rt_router_id = internal ? new->u.babel.router_id : p->router_id; + u64 rt_router_id = 0; + + if (new->src->owner == &P->sources) + { + rt_seqno = ea_find(new->attrs->eattrs, EA_BABEL_SEQNO)->u.data; + eattr *e = ea_find(new->attrs->eattrs, EA_BABEL_ROUTER_ID); + if (e) + memcpy(&rt_router_id, e->u.ptr->data, sizeof(u64)); + } + else + { + rt_seqno = p->update_seqno; + rt_router_id = p->router_id; + } if (rt_metric > BABEL_INFINITY) { log(L_WARN "%s: Invalid babel_metric value %u for route %N", - p->p.name, rt_metric, net->n.addr); + p->p.name, rt_metric, net); rt_metric = BABEL_INFINITY; } - e = babel_get_entry(p, net->n.addr); + e = babel_get_entry(p, net); /* Activate triggered updates */ if ((e->valid != BABEL_ENTRY_VALID) || @@ -2307,7 +2327,7 @@ babel_rt_notify(struct proto *P, struct channel *c UNUSED, struct network *net, else { /* Withdraw */ - e = babel_find_entry(p, net->n.addr); + e = babel_find_entry(p, net); if (!e || e->valid != BABEL_ENTRY_VALID) return; @@ -2323,15 +2343,16 @@ babel_rt_notify(struct proto *P, struct channel *c UNUSED, struct network *net, static int babel_rte_better(struct rte *new, struct rte *old) { - return new->u.babel.metric < old->u.babel.metric; + uint new_metric = ea_find(new->attrs->eattrs, EA_BABEL_SEQNO)->u.data; + uint old_metric = ea_find(old->attrs->eattrs, EA_BABEL_SEQNO)->u.data; + + return new_metric < old_metric; } -static int -babel_rte_same(struct rte *new, struct rte *old) +static u32 +babel_rte_igp_metric(struct rte *rt) { - return ((new->u.babel.seqno == old->u.babel.seqno) && - (new->u.babel.metric == old->u.babel.metric) && - (new->u.babel.router_id == old->u.babel.router_id)); + return ea_get_int(rt->attrs->eattrs, EA_BABEL_METRIC, BABEL_INFINITY); } @@ -2352,6 +2373,12 @@ babel_postconfig(struct proto_config *CF) cf->ip6_channel = ip6 ?: ip6_sadr; } +static struct rte_owner_class babel_rte_owner_class = { + .get_route_info = babel_get_route_info, + .rte_better = babel_rte_better, + .rte_igp_metric = babel_rte_igp_metric, +}; + static struct proto * babel_init(struct proto_config *CF) { @@ -2365,10 +2392,8 @@ babel_init(struct proto_config *CF) P->if_notify = babel_if_notify; P->rt_notify = babel_rt_notify; P->preexport = babel_preexport; - P->make_tmp_attrs = babel_make_tmp_attrs; - P->store_tmp_attrs = babel_store_tmp_attrs; - P->rte_better = babel_rte_better; - P->rte_same = babel_rte_same; + + P->sources.class = &babel_rte_owner_class; return P; } @@ -2461,7 +2486,6 @@ babel_reconfigure(struct proto *P, struct proto_config *CF) return 1; } - struct protocol proto_babel = { .name = "Babel", .template = "babel%d", @@ -2476,6 +2500,5 @@ struct protocol proto_babel = { .start = babel_start, .shutdown = babel_shutdown, .reconfigure = babel_reconfigure, - .get_route_info = babel_get_route_info, .get_attr = babel_get_attr }; diff --git a/proto/babel/babel.h b/proto/babel/babel.h index 84feb085..8b6da3c8 100644 --- a/proto/babel/babel.h +++ b/proto/babel/babel.h @@ -28,6 +28,7 @@ #define EA_BABEL_METRIC EA_CODE(PROTOCOL_BABEL, 0) #define EA_BABEL_ROUTER_ID EA_CODE(PROTOCOL_BABEL, 1) +#define EA_BABEL_SEQNO EA_CODE(PROTOCOL_BABEL, 2) #define BABEL_MAGIC 42 #define BABEL_VERSION 2 diff --git a/proto/bfd/Makefile b/proto/bfd/Makefile index 402122fc..267dff98 100644 --- a/proto/bfd/Makefile +++ b/proto/bfd/Makefile @@ -1,6 +1,6 @@ -src := bfd.c io.c packets.c +src := bfd.c packets.c obj := $(src-o-files) $(all-daemon) $(cf-local) -tests_objs := $(tests_objs) $(src-o-files)
\ No newline at end of file +tests_objs := $(tests_objs) $(src-o-files) diff --git a/proto/bfd/bfd.c b/proto/bfd/bfd.c index dac184c5..63e0deff 100644 --- a/proto/bfd/bfd.c +++ b/proto/bfd/bfd.c @@ -113,15 +113,26 @@ #define HASH_IP_EQ(a1,n1,a2,n2) ipa_equal(a1, a2) && n1 == n2 #define HASH_IP_FN(a,n) ipa_hash(a) ^ u32_hash(n) -static list bfd_proto_list; -static list bfd_wait_list; +DEFINE_DOMAIN(rtable); +#define BFD_LOCK LOCK_DOMAIN(rtable, bfd_global.lock) +#define BFD_UNLOCK UNLOCK_DOMAIN(rtable, bfd_global.lock) +#define BFD_ASSERT_LOCKED ASSERT_DIE(DOMAIN_IS_LOCKED(rtable, bfd_global.lock)) + +static struct { + DOMAIN(rtable) lock; + list wait_list; + list proto_list; +} bfd_global; + +static struct bfd_session bfd_admin_down = { .loc = { .state = BFD_STATE_ADMIN_DOWN }, }; const char *bfd_state_names[] = { "AdminDown", "Down", "Init", "Up" }; static void bfd_session_set_min_tx(struct bfd_session *s, u32 val); static struct bfd_iface *bfd_get_iface(struct bfd_proto *p, ip_addr local, struct iface *iface); static void bfd_free_iface(struct bfd_iface *ifa); -static inline void bfd_notify_kick(struct bfd_proto *p); +static void bfd_remove_session(struct bfd_proto *p, struct bfd_session *s); +static void bfd_reconfigure_session_hook(void *vsession); /* @@ -140,37 +151,57 @@ bfd_merge_options(const struct bfd_iface_config *cf, const struct bfd_options *o }; } -static void +static int bfd_session_update_state(struct bfd_session *s, uint state, uint diag) { struct bfd_proto *p = s->ifa->bfd; - uint old_state = s->loc_state; - int notify; + uint old_state = BFD_LOC_STATE(s).state; if (state == old_state) - return; + { + if (current_time() > s->last_reqlist_check + 5 S) + { + BFD_LOCK; + if (EMPTY_LIST(s->request_list)) + { + bfd_remove_session(p, s); + BFD_UNLOCK; + return 1; + } + + s->last_reqlist_check = current_time(); + BFD_UNLOCK; + } + return 0; + } TRACE(D_EVENTS, "Session to %I changed state from %s to %s", s->addr, bfd_state_names[old_state], bfd_state_names[state]); - bfd_lock_sessions(p); - s->loc_state = state; - s->loc_diag = diag; + atomic_store_explicit(&s->loc, ((struct bfd_session_state) { .state = state, .diag = diag }), memory_order_release); s->last_state_change = current_time(); - notify = !NODE_VALID(&s->n); - if (notify) - add_tail(&p->notify_list, &s->n); - bfd_unlock_sessions(p); - if (state == BFD_STATE_UP) bfd_session_set_min_tx(s, s->cf.min_tx_int); if (old_state == BFD_STATE_UP) bfd_session_set_min_tx(s, s->cf.idle_tx_int); - if (notify) - bfd_notify_kick(p); + BFD_LOCK; + if (EMPTY_LIST(s->request_list)) + { + bfd_remove_session(p, s); + BFD_UNLOCK; + return 1; + } + + struct bfd_request *req; + node *nn; + WALK_LIST2(req, nn, s->request_list, n) + ev_send_self(&req->event); + + BFD_UNLOCK; + return 0; } static void @@ -188,7 +219,7 @@ bfd_session_update_tx_interval(struct bfd_session *s) return; /* Set timer relative to last tx_timer event */ - tm_set(s->tx_timer, s->last_tx + tx_int_l); + tm_set_in(s->tx_timer, s->last_tx + tx_int_l, s->ifa->bfd->p.loop); } static void @@ -202,7 +233,7 @@ bfd_session_update_detection_time(struct bfd_session *s, int kick) if (!s->last_rx) return; - tm_set(s->hold_timer, s->last_rx + timeout); + tm_set_in(s->hold_timer, s->last_rx + timeout, s->ifa->bfd->p.loop); } static void @@ -215,8 +246,8 @@ bfd_session_control_tx_timer(struct bfd_session *s, int reset) if (s->rem_demand_mode && !s->poll_active && - (s->loc_state == BFD_STATE_UP) && - (s->rem_state == BFD_STATE_UP)) + (BFD_LOC_STATE(s).state == BFD_STATE_UP) && + (s->rem.state == BFD_STATE_UP)) goto stop; if (s->rem_min_rx_int == 0) @@ -226,7 +257,7 @@ bfd_session_control_tx_timer(struct bfd_session *s, int reset) if (reset || !tm_active(s->tx_timer)) { s->last_tx = 0; - tm_start(s->tx_timer, 0); + tm_start_in(s->tx_timer, 0, s->ifa->bfd->p.loop); } return; @@ -286,28 +317,29 @@ bfd_session_process_ctl(struct bfd_session *s, u8 flags, u32 old_tx_int, u32 old int next_state = 0; int diag = BFD_DIAG_NOTHING; - switch (s->loc_state) + switch (BFD_LOC_STATE(s).state) { case BFD_STATE_ADMIN_DOWN: return; case BFD_STATE_DOWN: - if (s->rem_state == BFD_STATE_DOWN) next_state = BFD_STATE_INIT; - else if (s->rem_state == BFD_STATE_INIT) next_state = BFD_STATE_UP; + if (s->rem.state == BFD_STATE_DOWN) next_state = BFD_STATE_INIT; + else if (s->rem.state == BFD_STATE_INIT) next_state = BFD_STATE_UP; break; case BFD_STATE_INIT: - if (s->rem_state == BFD_STATE_ADMIN_DOWN) next_state = BFD_STATE_DOWN, diag = BFD_DIAG_NEIGHBOR_DOWN; - else if (s->rem_state >= BFD_STATE_INIT) next_state = BFD_STATE_UP; + if (s->rem.state == BFD_STATE_ADMIN_DOWN) next_state = BFD_STATE_DOWN, diag = BFD_DIAG_NEIGHBOR_DOWN; + else if (s->rem.state >= BFD_STATE_INIT) next_state = BFD_STATE_UP; break; case BFD_STATE_UP: - if (s->rem_state <= BFD_STATE_DOWN) next_state = BFD_STATE_DOWN, diag = BFD_DIAG_NEIGHBOR_DOWN; + if (s->rem.state <= BFD_STATE_DOWN) next_state = BFD_STATE_DOWN, diag = BFD_DIAG_NEIGHBOR_DOWN; break; } if (next_state) - bfd_session_update_state(s, next_state, diag); + if (bfd_session_update_state(s, next_state, diag)) + return; bfd_session_control_tx_timer(s, 0); @@ -322,7 +354,7 @@ bfd_session_timeout(struct bfd_session *s) TRACE(D_EVENTS, "Session to %I expired", s->addr); - s->rem_state = BFD_STATE_DOWN; + s->rem.state = BFD_STATE_DOWN; s->rem_id = 0; s->rem_min_tx_int = 0; s->rem_min_rx_int = 1; @@ -333,7 +365,8 @@ bfd_session_timeout(struct bfd_session *s) s->poll_active = 0; s->poll_scheduled = 0; - bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_TIMEOUT); + if (bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_TIMEOUT)) + return; bfd_session_control_tx_timer(s, 1); } @@ -349,7 +382,7 @@ bfd_session_set_min_tx(struct bfd_session *s, u32 val) s->des_min_tx_new = val; /* Postpone timer update if des_min_tx_int increases and the session is up */ - if ((s->loc_state != BFD_STATE_UP) || (val < s->des_min_tx_int)) + if ((BFD_LOC_STATE(s).state != BFD_STATE_UP) || (val < s->des_min_tx_int)) { s->des_min_tx_int = val; bfd_session_update_tx_interval(s); @@ -369,7 +402,7 @@ bfd_session_set_min_rx(struct bfd_session *s, u32 val) s->req_min_rx_new = val; /* Postpone timer update if req_min_rx_int decreases and the session is up */ - if ((s->loc_state != BFD_STATE_UP) || (val > s->req_min_rx_int)) + if ((BFD_LOC_STATE(s).state != BFD_STATE_UP) || (val > s->req_min_rx_int)) { s->req_min_rx_int = val; bfd_session_update_detection_time(s, 0); @@ -381,12 +414,14 @@ bfd_session_set_min_rx(struct bfd_session *s, u32 val) struct bfd_session * bfd_find_session_by_id(struct bfd_proto *p, u32 id) { + ASSERT_DIE(birdloop_inside(p->p.loop)); return HASH_FIND(p->session_hash_id, HASH_ID, id); } struct bfd_session * bfd_find_session_by_addr(struct bfd_proto *p, ip_addr addr, uint ifindex) { + ASSERT_DIE(birdloop_inside(p->p.loop)); return HASH_FIND(p->session_hash_ip, HASH_IP, addr, ifindex); } @@ -419,7 +454,8 @@ bfd_get_free_id(struct bfd_proto *p) static struct bfd_session * bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *iface, struct bfd_options *opts) { - birdloop_enter(p->loop); + ASSERT_DIE(birdloop_inside(p->p.loop)); + BFD_ASSERT_LOCKED; struct bfd_iface *ifa = bfd_get_iface(p, local, iface); @@ -433,10 +469,15 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface * HASH_INSERT(p->session_hash_ip, HASH_IP, s); s->cf = bfd_merge_options(ifa->cf, opts); + s->update_event = (event) { + .hook = bfd_reconfigure_session_hook, + .data = s, + .list = birdloop_event_list(p->p.loop), + }; /* Initialization of state variables - see RFC 5880 6.8.1 */ - s->loc_state = BFD_STATE_DOWN; - s->rem_state = BFD_STATE_DOWN; + atomic_store_explicit(&s->loc, ((struct bfd_session_state) { .state = BFD_STATE_DOWN }), memory_order_relaxed); + s->rem.state = BFD_STATE_DOWN; s->des_min_tx_int = s->des_min_tx_new = s->cf.idle_tx_int; s->req_min_rx_int = s->req_min_rx_new = s->cf.min_rx_int; s->rem_min_rx_int = 1; @@ -444,8 +485,8 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface * s->passive = s->cf.passive; s->tx_csn = random_u32(); - s->tx_timer = tm_new_init(p->tpool, bfd_tx_timer_hook, s, 0, 0); - s->hold_timer = tm_new_init(p->tpool, bfd_hold_timer_hook, s, 0, 0); + s->tx_timer = tm_new_init(p->p.pool, bfd_tx_timer_hook, s, 0, 0); + s->hold_timer = tm_new_init(p->p.pool, bfd_hold_timer_hook, s, 0, 0); bfd_session_update_tx_interval(s); bfd_session_control_tx_timer(s, 1); @@ -454,51 +495,15 @@ bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface * TRACE(D_EVENTS, "Session to %I added", s->addr); - birdloop_leave(p->loop); - return s; } -/* -static void -bfd_open_session(struct bfd_proto *p, struct bfd_session *s, ip_addr local, struct iface *ifa) -{ - birdloop_enter(p->loop); - - s->opened = 1; - - bfd_session_control_tx_timer(s); - - birdloop_leave(p->loop); -} - -static void -bfd_close_session(struct bfd_proto *p, struct bfd_session *s) -{ - birdloop_enter(p->loop); - - s->opened = 0; - - bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_PATH_DOWN); - bfd_session_control_tx_timer(s); - - birdloop_leave(p->loop); -} -*/ - static void bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) { - ip_addr ip = s->addr; - - /* Caller should ensure that request list is empty */ - - birdloop_enter(p->loop); - - /* Remove session from notify list if scheduled for notification */ - /* No need for bfd_lock_sessions(), we are already protected by birdloop_enter() */ - if (NODE_VALID(&s->n)) - rem_node(&s->n); + ASSERT_DIE(birdloop_inside(p->p.loop)); + BFD_ASSERT_LOCKED; + ASSERT_DIE(EMPTY_LIST(s->request_list)); bfd_free_iface(s->ifa); @@ -508,25 +513,27 @@ bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) HASH_REMOVE(p->session_hash_id, HASH_ID, s); HASH_REMOVE(p->session_hash_ip, HASH_IP, s); - sl_free(p->session_slab, s); - - TRACE(D_EVENTS, "Session to %I removed", ip); + TRACE(D_EVENTS, "Session to %I removed", s->addr); - birdloop_leave(p->loop); + sl_free(p->session_slab, s); } static void bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s) { + ASSERT_DIE(birdloop_inside(p->p.loop)); + BFD_LOCK; if (EMPTY_LIST(s->request_list)) + { + bfd_remove_session(p, s); + BFD_UNLOCK; return; - - birdloop_enter(p->loop); + } struct bfd_request *req = SKIP_BACK(struct bfd_request, n, HEAD(s->request_list)); s->cf = bfd_merge_options(s->ifa->cf, &req->opts); - u32 tx = (s->loc_state == BFD_STATE_UP) ? s->cf.min_tx_int : s->cf.idle_tx_int; + u32 tx = (BFD_LOC_STATE(s).state == BFD_STATE_UP) ? s->cf.min_tx_int : s->cf.idle_tx_int; bfd_session_set_min_tx(s, tx); bfd_session_set_min_rx(s, s->cf.min_rx_int); s->detect_mult = s->cf.multiplier; @@ -534,9 +541,15 @@ bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s) bfd_session_control_tx_timer(s, 0); - birdloop_leave(p->loop); - TRACE(D_EVENTS, "Session to %I reconfigured", s->addr); + BFD_UNLOCK; +} + +static void +bfd_reconfigure_session_hook(void *data) +{ + struct bfd_session *s = data; + return bfd_reconfigure_session(s->ifa->bfd, s); } @@ -573,7 +586,7 @@ bfd_get_iface(struct bfd_proto *p, ip_addr local, struct iface *iface) struct bfd_config *cf = (struct bfd_config *) (p->p.cf); struct bfd_iface_config *ic = bfd_find_iface_config(cf, iface); - ifa = mb_allocz(p->tpool, sizeof(struct bfd_iface)); + ifa = mb_allocz(p->p.pool, sizeof(struct bfd_iface)); ifa->local = local; ifa->iface = iface; ifa->cf = ic; @@ -604,7 +617,7 @@ bfd_free_iface(struct bfd_iface *ifa) } static void -bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_config *nc) +bfd_reconfigure_iface(struct bfd_proto *p UNUSED, struct bfd_iface *ifa, struct bfd_config *nc) { struct bfd_iface_config *new = bfd_find_iface_config(nc, ifa->iface); struct bfd_iface_config *old = ifa->cf; @@ -618,9 +631,7 @@ bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_con (new->passive != old->passive); /* This should be probably changed to not access ifa->cf from the BFD thread */ - birdloop_enter(p->loop); ifa->cf = new; - birdloop_leave(p->loop); } @@ -629,17 +640,22 @@ bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_con */ static void -bfd_request_notify(struct bfd_request *req, u8 state, u8 diag) +bfd_request_notify(void *data) { - u8 old_state = req->state; + struct bfd_request *req = data; + struct bfd_session_state old = req->old_state; - if (state == old_state) + BFD_LOCK; /* Needed to safely access req->session */ + struct bfd_session_state new = atomic_load_explicit(&req->session->loc, memory_order_acquire); + BFD_UNLOCK; + + if (new.state == old.state) return; - req->state = state; - req->diag = diag; - req->old_state = old_state; - req->down = (old_state == BFD_STATE_UP) && (state == BFD_STATE_DOWN); + req->state = new.state; + req->diag = new.diag; + req->old_state = new; + req->down = (old.state == BFD_STATE_UP) && (new.state == BFD_STATE_DOWN); if (req->hook) req->hook(req); @@ -648,9 +664,12 @@ bfd_request_notify(struct bfd_request *req, u8 state, u8 diag) static int bfd_add_request(struct bfd_proto *p, struct bfd_request *req) { + BFD_ASSERT_LOCKED; + ASSERT_DIE(req->session == &bfd_admin_down); + struct bfd_config *cf = (struct bfd_config *) (p->p.cf); - if (p->p.vrf_set && (p->p.vrf != req->vrf)) + if (p->p.vrf && (p->p.vrf != req->vrf)) return 0; if (ipa_is_ip4(req->addr) ? !cf->accept_ipv4 : !cf->accept_ipv6) @@ -661,7 +680,6 @@ bfd_add_request(struct bfd_proto *p, struct bfd_request *req) uint ifindex = req->iface ? req->iface->index : 0; struct bfd_session *s = bfd_find_session_by_addr(p, req->addr, ifindex); - u8 state, diag; if (!s) s = bfd_add_session(p, req->addr, req->local, req->iface, &req->opts); @@ -670,52 +688,67 @@ bfd_add_request(struct bfd_proto *p, struct bfd_request *req) add_tail(&s->request_list, &req->n); req->session = s; - bfd_lock_sessions(p); - state = s->loc_state; - diag = s->loc_diag; - bfd_unlock_sessions(p); - - bfd_request_notify(req, state, diag); + ev_send_self(&req->event); return 1; } static void -bfd_submit_request(struct bfd_request *req) +bfd_pickup_requests(void *_data UNUSED) { - node *n; + struct bfd_proto *p; + node *nn; + WALK_LIST2(p, nn, bfd_global.proto_list, bfd_node) + { + birdloop_enter(p->p.loop); + BFD_LOCK; - WALK_LIST(n, bfd_proto_list) - if (bfd_add_request(SKIP_BACK(struct bfd_proto, bfd_node, n), req)) - return; + struct bfd_request *req; + node *rn, *rnxt; + WALK_LIST2_DELSAFE(req, rn, rnxt, bfd_global.wait_list, n) + bfd_add_request(p, req); - rem_node(&req->n); - add_tail(&bfd_wait_list, &req->n); - req->session = NULL; - bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0); + BFD_UNLOCK; + birdloop_ping(p->p.loop); + birdloop_leave(p->p.loop); + } } +static event bfd_pickup_event = { .hook = bfd_pickup_requests }; +#define bfd_schedule_pickup() ev_send(&global_event_list, &bfd_pickup_event) + static void bfd_take_requests(struct bfd_proto *p) { + struct bfd_request *req; node *n, *nn; - - WALK_LIST_DELSAFE(n, nn, bfd_wait_list) - bfd_add_request(p, SKIP_BACK(struct bfd_request, n, n)); + BFD_LOCK; + WALK_LIST2_DELSAFE(req, n, nn, bfd_global.wait_list, n) + bfd_add_request(p, req); + BFD_UNLOCK; } static void bfd_drop_requests(struct bfd_proto *p) { node *n; - - HASH_WALK(p->session_hash_id, next_id, s) + BFD_LOCK; + HASH_WALK_DELSAFE(p->session_hash_id, next_id, s) { - /* We assume that p is not in bfd_proto_list */ WALK_LIST_FIRST(n, s->request_list) - bfd_submit_request(SKIP_BACK(struct bfd_request, n, n)); + { + struct bfd_request *req = SKIP_BACK(struct bfd_request, n, n); + rem_node(&req->n); + add_tail(&bfd_global.wait_list, &req->n); + req->session = &bfd_admin_down; + ev_send_self(&req->event); + } + + bfd_schedule_pickup(); + bfd_remove_session(p, s); } HASH_WALK_END; + BFD_UNLOCK; } static struct resclass bfd_request_class; @@ -724,13 +757,11 @@ struct bfd_request * bfd_request_session(pool *p, ip_addr addr, ip_addr local, struct iface *iface, struct iface *vrf, void (*hook)(struct bfd_request *), void *data, + struct event_list *list, const struct bfd_options *opts) { struct bfd_request *req = ralloc(p, &bfd_request_class); - /* Hack: self-link req->n, we will call rem_node() on it */ - req->n.prev = req->n.next = &req->n; - req->addr = addr; req->local = local; req->iface = iface; @@ -739,10 +770,19 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local, if (opts) req->opts = *opts; - bfd_submit_request(req); - req->hook = hook; req->data = data; + req->event = (event) { + .hook = bfd_request_notify, + .data = req, + .list = list, + }; + + BFD_LOCK; + req->session = &bfd_admin_down; + add_tail(&bfd_global.wait_list, &req->n); + bfd_schedule_pickup(); + BFD_UNLOCK; return req; } @@ -750,30 +790,29 @@ bfd_request_session(pool *p, ip_addr addr, ip_addr local, void bfd_update_request(struct bfd_request *req, const struct bfd_options *opts) { - struct bfd_session *s = req->session; - if (!memcmp(opts, &req->opts, sizeof(const struct bfd_options))) return; + BFD_LOCK; req->opts = *opts; - if (s) - bfd_reconfigure_session(s->ifa->bfd, s); + struct bfd_session *s = req->session; + if (s != &bfd_admin_down) + ev_send_self(&s->update_event); + + BFD_UNLOCK; } static void bfd_request_free(resource *r) { struct bfd_request *req = (struct bfd_request *) r; - struct bfd_session *s = req->session; + BFD_LOCK; rem_node(&req->n); + BFD_UNLOCK; - /* Remove the session if there is no request for it. Skip that if - inside notify hooks, will be handled by bfd_notify_hook() itself */ - - if (s && EMPTY_LIST(s->request_list) && !s->notify_running) - bfd_remove_session(s->ifa->bfd, s); + ev_postpone(&req->event); } static void @@ -810,7 +849,7 @@ bfd_neigh_notify(struct neighbor *nb) if ((nb->scope > 0) && !n->req) { ip_addr local = ipa_nonzero(n->local) ? n->local : nb->ifa->ip; - n->req = bfd_request_session(p->p.pool, n->addr, local, nb->iface, p->p.vrf, NULL, NULL, NULL); + n->req = bfd_request_session(p->p.pool, n->addr, local, nb->iface, p->p.vrf, NULL, NULL, birdloop_event_list(p->p.loop), NULL); } if ((nb->scope <= 0) && n->req) @@ -827,7 +866,7 @@ bfd_start_neighbor(struct bfd_proto *p, struct bfd_neighbor *n) if (n->multihop) { - n->req = bfd_request_session(p->p.pool, n->addr, n->local, NULL, p->p.vrf, NULL, NULL, NULL); + n->req = bfd_request_session(p->p.pool, n->addr, n->local, NULL, p->p.vrf, NULL, NULL, birdloop_event_list(p->p.loop), NULL); return; } @@ -902,107 +941,15 @@ bfd_reconfigure_neighbors(struct bfd_proto *p, struct bfd_config *new) /* - * BFD notify socket - */ - -/* This core notify code should be replaced after main loop transition to birdloop */ - -int pipe(int pipefd[2]); -void pipe_drain(int fd); -void pipe_kick(int fd); - -static int -bfd_notify_hook(sock *sk, uint len UNUSED) -{ - struct bfd_proto *p = sk->data; - struct bfd_session *s; - list tmp_list; - u8 state, diag; - node *n, *nn; - - pipe_drain(sk->fd); - - bfd_lock_sessions(p); - init_list(&tmp_list); - add_tail_list(&tmp_list, &p->notify_list); - init_list(&p->notify_list); - bfd_unlock_sessions(p); - - WALK_LIST_FIRST(s, tmp_list) - { - bfd_lock_sessions(p); - rem_node(&s->n); - state = s->loc_state; - diag = s->loc_diag; - bfd_unlock_sessions(p); - - s->notify_running = 1; - WALK_LIST_DELSAFE(n, nn, s->request_list) - bfd_request_notify(SKIP_BACK(struct bfd_request, n, n), state, diag); - s->notify_running = 0; - - /* Remove the session if all requests were removed in notify hooks */ - if (EMPTY_LIST(s->request_list)) - bfd_remove_session(p, s); - } - - return 0; -} - -static inline void -bfd_notify_kick(struct bfd_proto *p) -{ - pipe_kick(p->notify_ws->fd); -} - -static void -bfd_noterr_hook(sock *sk, int err) -{ - struct bfd_proto *p = sk->data; - log(L_ERR "%s: Notify socket error: %m", p->p.name, err); -} - -static void -bfd_notify_init(struct bfd_proto *p) -{ - int pfds[2]; - sock *sk; - - int rv = pipe(pfds); - if (rv < 0) - die("pipe: %m"); - - sk = sk_new(p->p.pool); - sk->type = SK_MAGIC; - sk->rx_hook = bfd_notify_hook; - sk->err_hook = bfd_noterr_hook; - sk->fd = pfds[0]; - sk->data = p; - if (sk_open(sk) < 0) - die("bfd: sk_open failed"); - p->notify_rs = sk; - - /* The write sock is not added to any event loop */ - sk = sk_new(p->p.pool); - sk->type = SK_MAGIC; - sk->fd = pfds[1]; - sk->data = p; - sk->flags = SKF_THREAD; - if (sk_open(sk) < 0) - die("bfd: sk_open failed"); - p->notify_ws = sk; -} - - -/* * BFD protocol glue */ void bfd_init_all(void) { - init_list(&bfd_proto_list); - init_list(&bfd_wait_list); + bfd_global.lock = DOMAIN_NEW(rtable, "BFD Global"); + init_list(&bfd_global.wait_list); + init_list(&bfd_global.proto_list); } static struct proto * @@ -1021,22 +968,13 @@ bfd_start(struct proto *P) struct bfd_proto *p = (struct bfd_proto *) P; struct bfd_config *cf = (struct bfd_config *) (P->cf); - p->loop = birdloop_new(); - p->tpool = rp_new(NULL, "BFD thread root"); - pthread_spin_init(&p->lock, PTHREAD_PROCESS_PRIVATE); - p->session_slab = sl_new(P->pool, sizeof(struct bfd_session)); HASH_INIT(p->session_hash_id, P->pool, 8); HASH_INIT(p->session_hash_ip, P->pool, 8); init_list(&p->iface_list); - init_list(&p->notify_list); - bfd_notify_init(p); - - add_tail(&bfd_proto_list, &p->bfd_node); - - birdloop_enter(p->loop); + add_tail(&bfd_global.proto_list, &p->bfd_node); if (cf->accept_ipv4 && cf->accept_direct) p->rx4_1 = bfd_open_rx_sk(p, 0, SK_IPV4); @@ -1050,42 +988,33 @@ bfd_start(struct proto *P) if (cf->accept_ipv6 && cf->accept_multihop) p->rx6_m = bfd_open_rx_sk(p, 1, SK_IPV6); - birdloop_leave(p->loop); - bfd_take_requests(p); struct bfd_neighbor *n; WALK_LIST(n, cf->neigh_list) bfd_start_neighbor(p, n); - birdloop_start(p->loop); - return PS_UP; } - static int bfd_shutdown(struct proto *P) { struct bfd_proto *p = (struct bfd_proto *) P; - struct bfd_config *cf = (struct bfd_config *) (P->cf); + struct bfd_config *cf = (struct bfd_config *) (p->p.cf); rem_node(&p->bfd_node); - birdloop_stop(p->loop); - - struct bfd_neighbor *n; - WALK_LIST(n, cf->neigh_list) - bfd_stop_neighbor(p, n); + struct bfd_neighbor *bn; + WALK_LIST(bn, cf->neigh_list) + bfd_stop_neighbor(p, bn); bfd_drop_requests(p); - /* FIXME: This is hack */ - birdloop_enter(p->loop); - rfree(p->tpool); - birdloop_leave(p->loop); - - birdloop_free(p->loop); + if (p->rx4_1) sk_stop(p->rx4_1); + if (p->rx4_m) sk_stop(p->rx4_m); + if (p->rx6_1) sk_stop(p->rx6_1); + if (p->rx6_m) sk_stop(p->rx6_m); return PS_DOWN; } @@ -1098,6 +1027,8 @@ bfd_reconfigure(struct proto *P, struct proto_config *c) struct bfd_config *new = (struct bfd_config *) c; struct bfd_iface *ifa; + ASSERT_DIE(birdloop_inside(P->loop)); + /* TODO: Improve accept reconfiguration */ if ((new->accept_ipv4 != old->accept_ipv4) || (new->accept_ipv6 != old->accept_ipv6) || @@ -1105,21 +1036,21 @@ bfd_reconfigure(struct proto *P, struct proto_config *c) (new->accept_multihop != old->accept_multihop)) return 0; - birdloop_mask_wakeups(p->loop); + birdloop_mask_wakeups(p->p.loop); WALK_LIST(ifa, p->iface_list) bfd_reconfigure_iface(p, ifa, new); - HASH_WALK(p->session_hash_id, next_id, s) + HASH_WALK_DELSAFE(p->session_hash_id, next_id, s) { if (s->ifa->changed) bfd_reconfigure_session(p, s); } - HASH_WALK_END; + HASH_WALK_DELSAFE_END; bfd_reconfigure_neighbors(p, new); - birdloop_unmask_wakeups(p->loop); + birdloop_unmask_wakeups(p->p.loop); return 1; } @@ -1140,13 +1071,14 @@ bfd_show_sessions(struct proto *P) { byte tbuf[TM_DATETIME_BUFFER_SIZE]; struct bfd_proto *p = (struct bfd_proto *) P; - uint state, diag UNUSED; btime tx_int, timeout; const char *ifname; + birdloop_enter(P->loop); if (p->p.proto_state != PS_UP) { cli_msg(-1020, "%s: is not up", p->p.name); + birdloop_leave(P->loop); return; } @@ -1154,12 +1086,9 @@ bfd_show_sessions(struct proto *P) cli_msg(-1020, "%-25s %-10s %-10s %-12s %8s %8s", "IP address", "Interface", "State", "Since", "Interval", "Timeout"); - HASH_WALK(p->session_hash_id, next_id, s) { - /* FIXME: this is thread-unsafe, but perhaps harmless */ - state = s->loc_state; - diag = s->loc_diag; + uint state = BFD_LOC_STATE(s).state; ifname = (s->ifa && s->ifa->iface) ? s->ifa->iface->name : "---"; tx_int = s->last_tx ? MAX(s->des_min_tx_int, s->rem_min_rx_int) : 0; timeout = (btime) MAX(s->req_min_rx_int, s->rem_min_tx_int) * s->rem_detect_mult; @@ -1171,6 +1100,8 @@ bfd_show_sessions(struct proto *P) s->addr, ifname, bfd_state_names[state], tbuf, tx_int, timeout); } HASH_WALK_END; + + birdloop_leave(P->loop); } diff --git a/proto/bfd/bfd.h b/proto/bfd/bfd.h index 91fdaa60..b3266857 100644 --- a/proto/bfd/bfd.h +++ b/proto/bfd/bfd.h @@ -17,12 +17,12 @@ #include "nest/password.h" #include "conf/conf.h" #include "lib/hash.h" +#include "lib/io-loop.h" #include "lib/resource.h" #include "lib/socket.h" #include "lib/string.h" #include "nest/bfd.h" -#include "io.h" #define BFD_CONTROL_PORT 3784 @@ -87,19 +87,13 @@ struct bfd_neighbor struct bfd_proto { struct proto p; - struct birdloop *loop; - pool *tpool; - pthread_spinlock_t lock; + node bfd_node; slab *session_slab; HASH(struct bfd_session) session_hash_id; HASH(struct bfd_session) session_hash_ip; - sock *notify_rs; - sock *notify_ws; - list notify_list; - sock *rx4_1; sock *rx6_1; sock *rx4_m; @@ -122,7 +116,6 @@ struct bfd_iface struct bfd_session { - node n; ip_addr addr; /* Address of session */ struct bfd_iface *ifa; /* Iface associated with session */ struct bfd_session *next_id; /* Next in bfd.session_hash_id */ @@ -133,14 +126,15 @@ struct bfd_session u8 poll_active; u8 poll_scheduled; - u8 loc_state; - u8 rem_state; - u8 loc_diag; - u8 rem_diag; + _Atomic struct bfd_session_state loc; + struct bfd_session_state rem; +#define BFD_LOC_STATE(s) atomic_load_explicit(&(s)->loc, memory_order_relaxed) + u32 loc_id; /* Local session ID (local discriminator) */ u32 rem_id; /* Remote session ID (remote discriminator) */ - struct bfd_session_config cf; /* Static configuration parameters */ + struct bfd_session_config cf; /* Static configuration parameers */ + event update_event; /* Reconfiguration requested */ u32 des_min_tx_int; /* Desired min rx interval, local option */ u32 des_min_tx_new; /* Used for des_min_tx_int change */ @@ -162,6 +156,7 @@ struct bfd_session list request_list; /* List of client requests (struct bfd_request) */ btime last_state_change; /* Time of last state change */ + btime last_reqlist_check; /* Time of last check whether the request list is not empty */ u8 notify_running; /* 1 if notify hooks are running */ u8 rx_csn_known; /* Received crypto sequence number is known */ @@ -208,10 +203,6 @@ extern const char *bfd_state_names[]; extern const u8 bfd_auth_type_to_hash_alg[]; - -static inline void bfd_lock_sessions(struct bfd_proto *p) { pthread_spin_lock(&p->lock); } -static inline void bfd_unlock_sessions(struct bfd_proto *p) { pthread_spin_unlock(&p->lock); } - /* bfd.c */ struct bfd_session * bfd_find_session_by_id(struct bfd_proto *p, u32 id); struct bfd_session * bfd_find_session_by_addr(struct bfd_proto *p, ip_addr addr, uint ifindex); diff --git a/proto/bfd/config.Y b/proto/bfd/config.Y index df1cba42..ed5479fb 100644 --- a/proto/bfd/config.Y +++ b/proto/bfd/config.Y @@ -36,6 +36,7 @@ proto: bfd_proto ; bfd_proto_start: proto_start BFD { this_proto = proto_config_new(&proto_bfd, $1); + this_proto->loop_order = DOMAIN_ORDER(proto); init_list(&BFD_CFG->patt_list); init_list(&BFD_CFG->neigh_list); BFD_CFG->accept_ipv4 = BFD_CFG->accept_ipv6 = 1; diff --git a/proto/bfd/io.c b/proto/bfd/io.c deleted file mode 100644 index 1cd9365a..00000000 --- a/proto/bfd/io.c +++ /dev/null @@ -1,535 +0,0 @@ -/* - * BIRD -- I/O and event loop - * - * Can be freely distributed and used under the terms of the GNU GPL. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <errno.h> -#include <fcntl.h> -#include <poll.h> -#include <pthread.h> -#include <time.h> -#include <sys/time.h> - -#include "nest/bird.h" -#include "proto/bfd/io.h" - -#include "lib/buffer.h" -#include "lib/lists.h" -#include "lib/resource.h" -#include "lib/event.h" -#include "lib/timer.h" -#include "lib/socket.h" - - -struct birdloop -{ - pool *pool; - pthread_t thread; - pthread_mutex_t mutex; - - u8 stop_called; - u8 poll_active; - u8 wakeup_masked; - int wakeup_fds[2]; - - struct timeloop time; - list event_list; - list sock_list; - uint sock_num; - - BUFFER(sock *) poll_sk; - BUFFER(struct pollfd) poll_fd; - u8 poll_changed; - u8 close_scheduled; -}; - - -/* - * Current thread context - */ - -static pthread_key_t current_loop_key; -extern pthread_key_t current_time_key; - -static inline struct birdloop * -birdloop_current(void) -{ - return pthread_getspecific(current_loop_key); -} - -static inline void -birdloop_set_current(struct birdloop *loop) -{ - pthread_setspecific(current_loop_key, loop); - pthread_setspecific(current_time_key, loop ? &loop->time : &main_timeloop); -} - -static inline void -birdloop_init_current(void) -{ - pthread_key_create(¤t_loop_key, NULL); -} - - -/* - * Wakeup code for birdloop - */ - -static void -pipe_new(int *pfds) -{ - int rv = pipe(pfds); - if (rv < 0) - die("pipe: %m"); - - if (fcntl(pfds[0], F_SETFL, O_NONBLOCK) < 0) - die("fcntl(O_NONBLOCK): %m"); - - if (fcntl(pfds[1], F_SETFL, O_NONBLOCK) < 0) - die("fcntl(O_NONBLOCK): %m"); -} - -void -pipe_drain(int fd) -{ - char buf[64]; - int rv; - - try: - rv = read(fd, buf, 64); - if (rv < 0) - { - if (errno == EINTR) - goto try; - if (errno == EAGAIN) - return; - die("wakeup read: %m"); - } - if (rv == 64) - goto try; -} - -void -pipe_kick(int fd) -{ - u64 v = 1; - int rv; - - try: - rv = write(fd, &v, sizeof(u64)); - if (rv < 0) - { - if (errno == EINTR) - goto try; - if (errno == EAGAIN) - return; - die("wakeup write: %m"); - } -} - -static inline void -wakeup_init(struct birdloop *loop) -{ - pipe_new(loop->wakeup_fds); -} - -static inline void -wakeup_drain(struct birdloop *loop) -{ - pipe_drain(loop->wakeup_fds[0]); -} - -static inline void -wakeup_do_kick(struct birdloop *loop) -{ - pipe_kick(loop->wakeup_fds[1]); -} - -static inline void -wakeup_kick(struct birdloop *loop) -{ - if (!loop->wakeup_masked) - wakeup_do_kick(loop); - else - loop->wakeup_masked = 2; -} - -/* For notifications from outside */ -void -wakeup_kick_current(void) -{ - struct birdloop *loop = birdloop_current(); - - if (loop && loop->poll_active) - wakeup_kick(loop); -} - - -/* - * Events - */ - -static inline uint -events_waiting(struct birdloop *loop) -{ - return !EMPTY_LIST(loop->event_list); -} - -static inline void -events_init(struct birdloop *loop) -{ - init_list(&loop->event_list); -} - -static void -events_fire(struct birdloop *loop) -{ - times_update(&loop->time); - ev_run_list(&loop->event_list); -} - -void -ev2_schedule(event *e) -{ - struct birdloop *loop = birdloop_current(); - - if (loop->poll_active && EMPTY_LIST(loop->event_list)) - wakeup_kick(loop); - - if (e->n.next) - rem_node(&e->n); - - add_tail(&loop->event_list, &e->n); -} - - -/* - * Sockets - */ - -static void -sockets_init(struct birdloop *loop) -{ - init_list(&loop->sock_list); - loop->sock_num = 0; - - BUFFER_INIT(loop->poll_sk, loop->pool, 4); - BUFFER_INIT(loop->poll_fd, loop->pool, 4); - loop->poll_changed = 1; /* add wakeup fd */ -} - -static void -sockets_add(struct birdloop *loop, sock *s) -{ - add_tail(&loop->sock_list, &s->n); - loop->sock_num++; - - s->index = -1; - loop->poll_changed = 1; - - if (loop->poll_active) - wakeup_kick(loop); -} - -void -sk_start(sock *s) -{ - struct birdloop *loop = birdloop_current(); - - sockets_add(loop, s); -} - -static void -sockets_remove(struct birdloop *loop, sock *s) -{ - rem_node(&s->n); - loop->sock_num--; - - if (s->index >= 0) - loop->poll_sk.data[s->index] = NULL; - - s->index = -1; - loop->poll_changed = 1; - - /* Wakeup moved to sk_stop() */ -} - -void -sk_stop(sock *s) -{ - struct birdloop *loop = birdloop_current(); - - sockets_remove(loop, s); - - if (loop->poll_active) - { - loop->close_scheduled = 1; - wakeup_kick(loop); - } - else - close(s->fd); - - s->fd = -1; -} - -static inline uint sk_want_events(sock *s) -{ return (s->rx_hook ? POLLIN : 0) | ((s->ttx != s->tpos) ? POLLOUT : 0); } - -/* -FIXME: this should be called from sock code - -static void -sockets_update(struct birdloop *loop, sock *s) -{ - if (s->index >= 0) - loop->poll_fd.data[s->index].events = sk_want_events(s); -} -*/ - -static void -sockets_prepare(struct birdloop *loop) -{ - BUFFER_SET(loop->poll_sk, loop->sock_num + 1); - BUFFER_SET(loop->poll_fd, loop->sock_num + 1); - - struct pollfd *pfd = loop->poll_fd.data; - sock **psk = loop->poll_sk.data; - uint i = 0; - node *n; - - WALK_LIST(n, loop->sock_list) - { - sock *s = SKIP_BACK(sock, n, n); - - ASSERT(i < loop->sock_num); - - s->index = i; - *psk = s; - pfd->fd = s->fd; - pfd->events = sk_want_events(s); - pfd->revents = 0; - - pfd++; - psk++; - i++; - } - - ASSERT(i == loop->sock_num); - - /* Add internal wakeup fd */ - *psk = NULL; - pfd->fd = loop->wakeup_fds[0]; - pfd->events = POLLIN; - pfd->revents = 0; - - loop->poll_changed = 0; -} - -static void -sockets_close_fds(struct birdloop *loop) -{ - struct pollfd *pfd = loop->poll_fd.data; - sock **psk = loop->poll_sk.data; - int poll_num = loop->poll_fd.used - 1; - - int i; - for (i = 0; i < poll_num; i++) - if (psk[i] == NULL) - close(pfd[i].fd); - - loop->close_scheduled = 0; -} - -int sk_read(sock *s, int revents); -int sk_write(sock *s); - -static void -sockets_fire(struct birdloop *loop) -{ - struct pollfd *pfd = loop->poll_fd.data; - sock **psk = loop->poll_sk.data; - int poll_num = loop->poll_fd.used - 1; - - times_update(&loop->time); - - /* Last fd is internal wakeup fd */ - if (pfd[poll_num].revents & POLLIN) - wakeup_drain(loop); - - int i; - for (i = 0; i < poll_num; pfd++, psk++, i++) - { - int e = 1; - - if (! pfd->revents) - continue; - - if (pfd->revents & POLLNVAL) - die("poll: invalid fd %d", pfd->fd); - - if (pfd->revents & POLLIN) - while (e && *psk && (*psk)->rx_hook) - e = sk_read(*psk, 0); - - e = 1; - if (pfd->revents & POLLOUT) - while (e && *psk) - e = sk_write(*psk); - } -} - - -/* - * Birdloop - */ - -static void * birdloop_main(void *arg); - -struct birdloop * -birdloop_new(void) -{ - /* FIXME: this init should be elsewhere and thread-safe */ - static int init = 0; - if (!init) - { birdloop_init_current(); init = 1; } - - pool *p = rp_new(NULL, "Birdloop root"); - struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop)); - loop->pool = p; - pthread_mutex_init(&loop->mutex, NULL); - - wakeup_init(loop); - - events_init(loop); - timers_init(&loop->time, p); - sockets_init(loop); - - return loop; -} - -void -birdloop_start(struct birdloop *loop) -{ - int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop); - if (rv) - die("pthread_create(): %M", rv); -} - -void -birdloop_stop(struct birdloop *loop) -{ - pthread_mutex_lock(&loop->mutex); - loop->stop_called = 1; - wakeup_do_kick(loop); - pthread_mutex_unlock(&loop->mutex); - - int rv = pthread_join(loop->thread, NULL); - if (rv) - die("pthread_join(): %M", rv); -} - -void -birdloop_free(struct birdloop *loop) -{ - rfree(loop->pool); -} - - -void -birdloop_enter(struct birdloop *loop) -{ - /* TODO: these functions could save and restore old context */ - pthread_mutex_lock(&loop->mutex); - birdloop_set_current(loop); -} - -void -birdloop_leave(struct birdloop *loop) -{ - /* TODO: these functions could save and restore old context */ - birdloop_set_current(NULL); - pthread_mutex_unlock(&loop->mutex); -} - -void -birdloop_mask_wakeups(struct birdloop *loop) -{ - pthread_mutex_lock(&loop->mutex); - loop->wakeup_masked = 1; - pthread_mutex_unlock(&loop->mutex); -} - -void -birdloop_unmask_wakeups(struct birdloop *loop) -{ - pthread_mutex_lock(&loop->mutex); - if (loop->wakeup_masked == 2) - wakeup_do_kick(loop); - loop->wakeup_masked = 0; - pthread_mutex_unlock(&loop->mutex); -} - -static void * -birdloop_main(void *arg) -{ - struct birdloop *loop = arg; - timer *t; - int rv, timeout; - - birdloop_set_current(loop); - - pthread_mutex_lock(&loop->mutex); - while (1) - { - events_fire(loop); - timers_fire(&loop->time); - - times_update(&loop->time); - if (events_waiting(loop)) - timeout = 0; - else if (t = timers_first(&loop->time)) - timeout = (tm_remains(t) TO_MS) + 1; - else - timeout = -1; - - if (loop->poll_changed) - sockets_prepare(loop); - - loop->poll_active = 1; - pthread_mutex_unlock(&loop->mutex); - - try: - rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout); - if (rv < 0) - { - if (errno == EINTR || errno == EAGAIN) - goto try; - die("poll: %m"); - } - - pthread_mutex_lock(&loop->mutex); - loop->poll_active = 0; - - if (loop->close_scheduled) - sockets_close_fds(loop); - - if (loop->stop_called) - break; - - if (rv) - sockets_fire(loop); - - timers_fire(&loop->time); - } - - loop->stop_called = 0; - pthread_mutex_unlock(&loop->mutex); - - return NULL; -} - - diff --git a/proto/bfd/io.h b/proto/bfd/io.h deleted file mode 100644 index ec706e9a..00000000 --- a/proto/bfd/io.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * BIRD -- I/O and event loop - * - * Can be freely distributed and used under the terms of the GNU GPL. - */ - -#ifndef _BIRD_BFD_IO_H_ -#define _BIRD_BFD_IO_H_ - -#include "nest/bird.h" -#include "lib/lists.h" -#include "lib/resource.h" -#include "lib/event.h" -#include "lib/timer.h" -#include "lib/socket.h" - - -void ev2_schedule(event *e); - -void sk_start(sock *s); -void sk_stop(sock *s); - -struct birdloop *birdloop_new(void); -void birdloop_start(struct birdloop *loop); -void birdloop_stop(struct birdloop *loop); -void birdloop_free(struct birdloop *loop); - -void birdloop_enter(struct birdloop *loop); -void birdloop_leave(struct birdloop *loop); -void birdloop_mask_wakeups(struct birdloop *loop); -void birdloop_unmask_wakeups(struct birdloop *loop); - - -#endif /* _BIRD_BFD_IO_H_ */ diff --git a/proto/bfd/packets.c b/proto/bfd/packets.c index 7618e20f..893d582d 100644 --- a/proto/bfd/packets.c +++ b/proto/bfd/packets.c @@ -290,9 +290,11 @@ bfd_send_ctl(struct bfd_proto *p, struct bfd_session *s, int final) if (!sk) return; + struct bfd_session_state loc = BFD_LOC_STATE(s); + pkt = (struct bfd_ctl_packet *) sk->tbuf; - pkt->vdiag = bfd_pack_vdiag(1, s->loc_diag); - pkt->flags = bfd_pack_flags(s->loc_state, 0); + pkt->vdiag = bfd_pack_vdiag(1, loc.diag); + pkt->flags = bfd_pack_flags(loc.state, 0); pkt->detect_mult = s->detect_mult; pkt->length = BFD_BASE_LEN; pkt->snd_id = htonl(s->loc_id); @@ -313,7 +315,7 @@ bfd_send_ctl(struct bfd_proto *p, struct bfd_session *s, int final) log(L_WARN "%s: Old packet overwritten in TX buffer", p->p.name); TRACE(D_PACKETS, "Sending CTL to %I [%s%s]", s->addr, - bfd_state_names[s->loc_state], bfd_format_flags(pkt->flags, fb)); + bfd_state_names[loc.state], bfd_format_flags(pkt->flags, fb)); sk_send_to(sk, pkt->length, s->addr, sk->dport); } @@ -382,16 +384,17 @@ bfd_rx_hook(sock *sk, uint len) u32 old_rx_int = s->rem_min_rx_int; s->rem_id= ntohl(pkt->snd_id); - s->rem_state = bfd_pkt_get_state(pkt); - s->rem_diag = bfd_pkt_get_diag(pkt); + s->rem.state = bfd_pkt_get_state(pkt); + s->rem.diag = bfd_pkt_get_diag(pkt); s->rem_demand_mode = pkt->flags & BFD_FLAG_DEMAND; s->rem_min_tx_int = ntohl(pkt->des_min_tx_int); s->rem_min_rx_int = ntohl(pkt->req_min_rx_int); s->rem_detect_mult = pkt->detect_mult; TRACE(D_PACKETS, "CTL received from %I [%s%s]", sk->faddr, - bfd_state_names[s->rem_state], bfd_format_flags(pkt->flags, fb)); + bfd_state_names[s->rem.state], bfd_format_flags(pkt->flags, fb)); + /* This call may drop the session, must be called in tail position */ bfd_session_process_ctl(s, pkt->flags, old_tx_int, old_rx_int); return 1; @@ -410,7 +413,7 @@ bfd_err_hook(sock *sk, int err) sock * bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af) { - sock *sk = sk_new(p->tpool); + sock *sk = sk_new(p->p.pool); sk->type = SK_UDP; sk->subtype = af; sk->sport = !multihop ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; @@ -425,6 +428,7 @@ bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af) sk->tos = IP_PREC_INTERNET_CONTROL; sk->priority = sk_priority_control; sk->flags = SKF_THREAD | SKF_LADDR_RX | (!multihop ? SKF_TTL_RX : 0); + sk->loop = p->p.loop; if (sk_open(sk) < 0) goto err; @@ -441,7 +445,7 @@ bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af) sock * bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa) { - sock *sk = sk_new(p->tpool); + sock *sk = sk_new(p->p.pool); sk->type = SK_UDP; sk->saddr = local; sk->dport = ifa ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; @@ -457,6 +461,7 @@ bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa) sk->priority = sk_priority_control; sk->ttl = ifa ? 255 : -1; sk->flags = SKF_THREAD | SKF_BIND | SKF_HIGH_PORT; + sk->loop = p->p.loop; if (sk_open(sk) < 0) goto err; diff --git a/proto/bgp/attrs.c b/proto/bgp/attrs.c index 24ba00ba..02b07410 100644 --- a/proto/bgp/attrs.c +++ b/proto/bgp/attrs.c @@ -333,26 +333,26 @@ bgp_aigp_set_metric(struct linpool *pool, const struct adata *ad, u64 metric) } int -bgp_total_aigp_metric_(rte *e, u64 *metric, const struct adata **ad) +bgp_total_aigp_metric_(struct rta *a, u64 *metric, const struct adata **ad) { - eattr *a = ea_find(e->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_AIGP)); - if (!a) + eattr *ea = ea_find(a->eattrs, EA_CODE(PROTOCOL_BGP, BA_AIGP)); + if (!ea) return 0; - const byte *b = bgp_aigp_get_tlv(a->u.ptr, BGP_AIGP_METRIC); + const byte *b = bgp_aigp_get_tlv(ea->u.ptr, BGP_AIGP_METRIC); if (!b) return 0; u64 aigp = get_u64(b + 3); - u64 step = e->attrs->igp_metric; + u64 step = a->igp_metric; - if (!rte_resolvable(e) || (step >= IGP_METRIC_UNKNOWN)) + if (!rta_resolvable(a) || (step >= IGP_METRIC_UNKNOWN)) step = BGP_AIGP_MAX; if (!step) step = 1; - *ad = a->u.ptr; + *ad = ea->u.ptr; *metric = aigp + step; if (*metric < aigp) *metric = BGP_AIGP_MAX; @@ -371,6 +371,13 @@ bgp_init_aigp_metric(rte *e, u64 *metric, const struct adata **ad) return *metric < IGP_METRIC_UNKNOWN; } +u32 +bgp_rte_igp_metric(struct rte *rt) +{ + u64 metric = bgp_total_aigp_metric(rt->attrs); + return (u32) MIN(metric, (u64) IGP_METRIC_UNKNOWN); +} + /* * Attribute hooks @@ -896,7 +903,7 @@ bgp_decode_large_community(struct bgp_parse_state *s, uint code UNUSED, uint fla static void bgp_export_mpls_label_stack(struct bgp_export_state *s, eattr *a) { - net_addr *n = s->route->net->n.addr; + const net_addr *n = s->route->net; u32 *labels = (u32 *) a->u.ptr->data; uint lnum = a->u.ptr->length / 4; @@ -1617,7 +1624,7 @@ bgp_free_prefix_table(struct bgp_channel *c) } static struct bgp_prefix * -bgp_get_prefix(struct bgp_channel *c, net_addr *net, u32 path_id) +bgp_get_prefix(struct bgp_channel *c, const net_addr *net, u32 path_id) { u32 hash = net_hash(net) ^ u32_hash(path_id); struct bgp_prefix *px = HASH_FIND(c->prefix_hash, PXH, net, path_id, hash); @@ -1661,12 +1668,10 @@ bgp_free_prefix(struct bgp_channel *c, struct bgp_prefix *px) */ int -bgp_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +bgp_preexport(struct channel *c, rte *e) { - rte *e = *new; - struct proto *SRC = e->attrs->src->proto; - struct bgp_proto *p = (struct bgp_proto *) P; - struct bgp_proto *src = (SRC->proto == &proto_bgp) ? (struct bgp_proto *) SRC : NULL; + struct bgp_proto *p = (struct bgp_proto *) (c->proto); + struct bgp_proto *src = bgp_rte_proto(e); /* Reject our routes */ if (src == p) @@ -1690,11 +1695,11 @@ bgp_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) } /* Handle well-known communities, RFC 1997 */ - struct eattr *c; + struct eattr *com; if (p->cf->interpret_communities && - (c = ea_find(e->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_COMMUNITY)))) + (com = ea_find(e->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_COMMUNITY)))) { - const struct adata *d = c->u.ptr; + const struct adata *d = com->u.ptr; /* Do not export anywhere */ if (int_set_contains(d, BGP_COMM_NO_ADVERTISE)) @@ -1719,8 +1724,7 @@ bgp_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) static ea_list * bgp_update_attrs(struct bgp_proto *p, struct bgp_channel *c, rte *e, ea_list *attrs0, struct linpool *pool) { - struct proto *SRC = e->attrs->src->proto; - struct bgp_proto *src = (SRC->proto == &proto_bgp) ? (void *) SRC : NULL; + struct bgp_proto *src = bgp_rte_proto(e); struct bgp_export_state s = { .proto = p, .channel = c, .pool = pool, .src = src, .route = e, .mpls = c->desc->mpls }; ea_list *attrs = attrs0; eattr *a; @@ -1774,7 +1778,7 @@ bgp_update_attrs(struct bgp_proto *p, struct bgp_channel *c, rte *e, ea_list *at /* AIGP attribute - accumulate local metric or originate new one */ u64 metric; if (s.local_next_hop && - (bgp_total_aigp_metric_(e, &metric, &ad) || + (bgp_total_aigp_metric_(e->attrs, &metric, &ad) || (c->cf->aigp_originate && bgp_init_aigp_metric(e, &metric, &ad)))) { ad = bgp_aigp_set_metric(pool, ad, metric); @@ -1833,7 +1837,7 @@ bgp_update_attrs(struct bgp_proto *p, struct bgp_channel *c, rte *e, ea_list *at } void -bgp_rt_notify(struct proto *P, struct channel *C, net *n, rte *new, rte *old) +bgp_rt_notify(struct proto *P, struct channel *C, const net_addr *n, rte *new, const rte *old) { struct bgp_proto *p = (void *) P; struct bgp_channel *c = (void *) C; @@ -1843,21 +1847,19 @@ bgp_rt_notify(struct proto *P, struct channel *C, net *n, rte *new, rte *old) if (new) { - struct ea_list *attrs = bgp_update_attrs(p, c, new, new->attrs->eattrs, bgp_linpool2); + struct ea_list *attrs = bgp_update_attrs(p, c, new, new->attrs->eattrs, C->rte_update_pool); /* If attributes are invalid, we fail back to withdraw */ buck = attrs ? bgp_get_bucket(c, attrs) : bgp_get_withdraw_bucket(c); - path = new->attrs->src->global_id; - - lp_flush(bgp_linpool2); + path = new->src->global_id; } else { buck = bgp_get_withdraw_bucket(c); - path = old->attrs->src->global_id; + path = old->src->global_id; } - px = bgp_get_prefix(c, n->n.addr, c->add_path_tx ? path : 0); + px = bgp_get_prefix(c, n, c->add_path_tx ? path : 0); add_tail(&buck->prefixes, &px->buck_node); bgp_schedule_packet(p->conn, c, PKT_UPDATE); @@ -1874,42 +1876,52 @@ bgp_get_neighbor(rte *r) return as; /* If AS_PATH is not defined, we treat rte as locally originated */ - struct bgp_proto *p = (void *) r->attrs->src->proto; + struct bgp_proto *p = bgp_rte_proto(r); return p->cf->confederation ?: p->local_as; } static inline int rte_stale(rte *r) { - if (r->u.bgp.stale < 0) + if (r->pflags & BGP_REF_STALE) + return 1; + + if (r->pflags & BGP_REF_NOT_STALE) + return 0; + + /* If staleness is unknown, compute and cache it */ + eattr *a = ea_find(r->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_COMMUNITY)); + if (a && int_set_contains(a->u.ptr, BGP_COMM_LLGR_STALE)) { - /* If staleness is unknown, compute and cache it */ - eattr *a = ea_find(r->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_COMMUNITY)); - r->u.bgp.stale = a && int_set_contains(a->u.ptr, BGP_COMM_LLGR_STALE); + r->pflags |= BGP_REF_STALE; + return 1; + } + else + { + r->pflags |= BGP_REF_NOT_STALE; + return 0; } - - return r->u.bgp.stale; } int bgp_rte_better(rte *new, rte *old) { - struct bgp_proto *new_bgp = (struct bgp_proto *) new->attrs->src->proto; - struct bgp_proto *old_bgp = (struct bgp_proto *) old->attrs->src->proto; + struct bgp_proto *new_bgp = bgp_rte_proto(new); + struct bgp_proto *old_bgp = bgp_rte_proto(old); eattr *x, *y; u32 n, o; /* Skip suppressed routes (see bgp_rte_recalculate()) */ - n = new->u.bgp.suppressed; - o = old->u.bgp.suppressed; + n = new->pflags & BGP_REF_SUPPRESSED; + o = old->pflags & BGP_REF_SUPPRESSED; if (n > o) return 0; if (n < o) return 1; /* RFC 4271 9.1.2.1. Route resolvability test */ - n = rte_resolvable(new); - o = rte_resolvable(old); + n = rta_resolvable(new->attrs); + o = rta_resolvable(old->attrs); if (n > o) return 1; if (n < o) @@ -1934,8 +1946,8 @@ bgp_rte_better(rte *new, rte *old) return 0; /* RFC 7311 4.1 - Apply AIGP metric */ - u64 n2 = bgp_total_aigp_metric(new); - u64 o2 = bgp_total_aigp_metric(old); + u64 n2 = bgp_total_aigp_metric(new->attrs); + u64 o2 = bgp_total_aigp_metric(old->attrs); if (n2 < o2) return 1; if (n2 > o2) @@ -2039,21 +2051,18 @@ bgp_rte_better(rte *new, rte *old) int bgp_rte_mergable(rte *pri, rte *sec) { - struct bgp_proto *pri_bgp = (struct bgp_proto *) pri->attrs->src->proto; - struct bgp_proto *sec_bgp = (struct bgp_proto *) sec->attrs->src->proto; + struct bgp_proto *pri_bgp = bgp_rte_proto(pri); + struct bgp_proto *sec_bgp = bgp_rte_proto(sec); eattr *x, *y; u32 p, s; /* Skip suppressed routes (see bgp_rte_recalculate()) */ - if (pri->u.bgp.suppressed != sec->u.bgp.suppressed) + /* LLGR draft - depreference stale routes */ + if (pri->pflags != sec->pflags) return 0; /* RFC 4271 9.1.2.1. Route resolvability test */ - if (rte_resolvable(pri) != rte_resolvable(sec)) - return 0; - - /* LLGR draft - depreference stale routes */ - if (rte_stale(pri) != rte_stale(sec)) + if (rta_resolvable(pri->attrs) != rta_resolvable(sec->attrs)) return 0; /* Start with local preferences */ @@ -2118,24 +2127,23 @@ bgp_rte_mergable(rte *pri, rte *sec) static inline int same_group(rte *r, u32 lpref, u32 lasn) { - return (r->pref == lpref) && (bgp_get_neighbor(r) == lasn); + return (r->attrs->pref == lpref) && (bgp_get_neighbor(r) == lasn); } static inline int -use_deterministic_med(rte *r) +use_deterministic_med(struct rte_storage *r) { - struct proto *P = r->attrs->src->proto; - return (P->proto == &proto_bgp) && ((struct bgp_proto *) P)->cf->deterministic_med; + struct bgp_proto *p = bgp_rte_proto(&r->rte); + return p && p->cf->deterministic_med; } int -bgp_rte_recalculate(rtable *table, net *net, rte *new, rte *old, rte *old_best) +bgp_rte_recalculate(rtable_private *table, net *net, rte *new, rte *old, rte *old_best) { - rte *r, *s; rte *key = new ? new : old; - u32 lpref = key->pref; + u32 lpref = key->attrs->pref; u32 lasn = bgp_get_neighbor(key); - int old_suppressed = old ? old->u.bgp.suppressed : 0; + int old_suppressed = old ? !!(old->pflags & BGP_REF_SUPPRESSED) : 0; /* * Proper RFC 4271 path selection is a bit complicated, it cannot be @@ -2187,11 +2195,11 @@ bgp_rte_recalculate(rtable *table, net *net, rte *new, rte *old, rte *old_best) */ if (new) - new->u.bgp.suppressed = 1; + new->pflags |= BGP_REF_SUPPRESSED; if (old) { - old->u.bgp.suppressed = 1; + old->pflags |= BGP_REF_SUPPRESSED; /* The fast case - replace not best with worse (or remove not best) */ if (old_suppressed && !(new && bgp_rte_better(new, old))) @@ -2199,13 +2207,13 @@ bgp_rte_recalculate(rtable *table, net *net, rte *new, rte *old, rte *old_best) } /* The default case - find a new best-in-group route */ - r = new; /* new may not be in the list */ - for (s=net->routes; rte_is_valid(s); s=s->next) - if (use_deterministic_med(s) && same_group(s, lpref, lasn)) + rte *r = new; /* new may not be in the list */ + for (struct rte_storage *s = net->routes; rte_is_valid(&s->rte); s = s->next) + if (use_deterministic_med(s) && same_group(&s->rte, lpref, lasn)) { - s->u.bgp.suppressed = 1; - if (!r || bgp_rte_better(s, r)) - r = s; + s->rte.pflags |= BGP_REF_SUPPRESSED; + if (!r || bgp_rte_better(&s->rte, r)) + r = &s->rte; } /* Simple case - the last route in group disappears */ @@ -2214,16 +2222,16 @@ bgp_rte_recalculate(rtable *table, net *net, rte *new, rte *old, rte *old_best) /* Found if new is mergable with best-in-group */ if (new && (new != r) && bgp_rte_mergable(r, new)) - new->u.bgp.suppressed = 0; + new->pflags &= ~BGP_REF_SUPPRESSED; /* Found all existing routes mergable with best-in-group */ - for (s=net->routes; rte_is_valid(s); s=s->next) - if (use_deterministic_med(s) && same_group(s, lpref, lasn)) - if ((s != r) && bgp_rte_mergable(r, s)) - s->u.bgp.suppressed = 0; + for (struct rte_storage *s = net->routes; rte_is_valid(&s->rte); s = s->next) + if (use_deterministic_med(s) && same_group(&s->rte, lpref, lasn)) + if ((&s->rte != r) && bgp_rte_mergable(r, &s->rte)) + s->rte.pflags &= ~BGP_REF_SUPPRESSED; /* Found best-in-group */ - r->u.bgp.suppressed = 0; + r->pflags &= ~BGP_REF_SUPPRESSED; /* * There are generally two reasons why we have to force @@ -2255,25 +2263,44 @@ bgp_rte_recalculate(rtable *table, net *net, rte *new, rte *old, rte *old_best) return !old_suppressed; } -struct rte * -bgp_rte_modify_stale(struct rte *r, struct linpool *pool) +void +bgp_rte_modify_stale(struct rt_export_request *req, const net_addr *n, struct rt_pending_export *rpe UNUSED, rte **feed, uint count) { - eattr *a = ea_find(r->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_COMMUNITY)); - const struct adata *ad = a ? a->u.ptr : NULL; - uint flags = a ? a->flags : BAF_PARTIAL; + struct bgp_channel *c = SKIP_BACK(struct bgp_channel, stale_feed, req); - if (ad && int_set_contains(ad, BGP_COMM_NO_LLGR)) - return NULL; + do { + rte *r = feed[--count]; + if (r->sender != c->c.in_req.hook) + continue; + + /* A new route, do not mark as stale */ + if (r->stale_cycle == c->c.in_req.hook->stale_set) + continue; + + eattr *ea = ea_find(r->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_COMMUNITY)); + const struct adata *ad = ea ? ea->u.ptr : NULL; + uint flags = ea ? ea->flags : BAF_PARTIAL; + + rte e0 = *r; + e0.flags |= REF_USE_STALE; - if (ad && int_set_contains(ad, BGP_COMM_LLGR_STALE)) - return r; + if (ad && int_set_contains(ad, BGP_COMM_NO_LLGR)) + rte_import(&c->c.in_req, n, NULL, r->src); - r = rte_cow_rta(r, pool); - bgp_set_attr_ptr(&(r->attrs->eattrs), pool, BA_COMMUNITY, flags, - int_set_add(pool, ad, BGP_COMM_LLGR_STALE)); - r->u.bgp.stale = 1; + else if (ad && int_set_contains(ad, BGP_COMM_LLGR_STALE)) + rte_import(&c->c.in_req, n, &e0, r->src); - return r; + else { + rta *a = e0.attrs = rta_do_cow(r->attrs, c->c.rte_update_pool); + + bgp_set_attr_ptr(&(a->eattrs), c->c.rte_update_pool, BA_COMMUNITY, flags, + int_set_add(c->c.rte_update_pool, ad, BGP_COMM_LLGR_STALE)); + e0.pflags |= BGP_REF_STALE; + + rte_import(&c->c.in_req, n, &e0, r->src); + lp_flush(c->c.rte_update_pool); + } + } while (count); } @@ -2356,22 +2383,22 @@ bgp_get_route_info(rte *e, byte *buf) eattr *o = ea_find(e->attrs->eattrs, EA_CODE(PROTOCOL_BGP, BA_ORIGIN)); u32 origas; - buf += bsprintf(buf, " (%d", e->pref); + buf += bsprintf(buf, " (%d", e->attrs->pref); - if (e->u.bgp.suppressed) + if (e->pflags & BGP_REF_SUPPRESSED) buf += bsprintf(buf, "-"); if (rte_stale(e)) buf += bsprintf(buf, "s"); - u64 metric = bgp_total_aigp_metric(e); + u64 metric = bgp_total_aigp_metric(e->attrs); if (metric < BGP_AIGP_MAX) { buf += bsprintf(buf, "/%lu", metric); } else if (e->attrs->igp_metric) { - if (!rte_resolvable(e)) + if (!rta_resolvable(e->attrs)) buf += bsprintf(buf, "/-"); else if (e->attrs->igp_metric >= IGP_METRIC_UNKNOWN) buf += bsprintf(buf, "/?"); diff --git a/proto/bgp/bgp.c b/proto/bgp/bgp.c index e4d754b1..65cc3a40 100644 --- a/proto/bgp/bgp.c +++ b/proto/bgp/bgp.c @@ -124,11 +124,8 @@ #include "bgp.h" - -struct linpool *bgp_linpool; /* Global temporary pool */ -struct linpool *bgp_linpool2; /* Global temporary pool for bgp_rt_notify() */ -static list bgp_sockets; /* Global list of listening sockets */ - +/* Global list of listening sockets */ +static list STATIC_LIST_INIT(bgp_sockets); static void bgp_connect(struct bgp_proto *p); static void bgp_active(struct bgp_proto *p); @@ -140,6 +137,15 @@ static void bgp_update_bfd(struct bgp_proto *p, const struct bfd_options *bfd); static int bgp_incoming_connection(sock *sk, uint dummy UNUSED); static void bgp_listen_sock_err(sock *sk UNUSED, int err); +static void bgp_graceful_restart_feed(struct bgp_channel *c); +static inline void channel_refresh_end_reload(struct channel *c) +{ + channel_refresh_end(c); + + if (c->in_table) + channel_request_reload(c); +} + /** * bgp_open - open a BGP instance * @p: BGP instance @@ -152,16 +158,14 @@ static void bgp_listen_sock_err(sock *sk UNUSED, int err); static int bgp_open(struct bgp_proto *p) { + ASSERT_DIE(birdloop_inside(&main_birdloop)); + struct bgp_socket *bs = NULL; struct iface *ifa = p->cf->strict_bind ? p->cf->iface : NULL; ip_addr addr = p->cf->strict_bind ? p->cf->local_ip : (p->ipv4 ? IPA_NONE4 : IPA_NONE6); uint port = p->cf->local_port; - /* FIXME: Add some global init? */ - if (!bgp_linpool) - init_list(&bgp_sockets); - /* We assume that cf->iface is defined iff cf->local_ip is link-local */ WALK_LIST(bs, bgp_sockets) @@ -180,7 +184,7 @@ bgp_open(struct bgp_proto *p) sk->sport = port; sk->iface = ifa; sk->vrf = p->p.vrf; - sk->flags = 0; + sk->flags = SKF_PASSIVE_THREAD; sk->tos = IP_PREC_INTERNET_CONTROL; sk->rbsize = BGP_RX_BUFFER_SIZE; sk->tbsize = BGP_TX_BUFFER_SIZE; @@ -198,12 +202,6 @@ bgp_open(struct bgp_proto *p) add_tail(&bgp_sockets, &bs->n); - if (!bgp_linpool) - { - bgp_linpool = lp_new_default(proto_pool); - bgp_linpool2 = lp_new_default(proto_pool); - } - return 0; err: @@ -222,6 +220,7 @@ err: static void bgp_close(struct bgp_proto *p) { + ASSERT_DIE(birdloop_inside(&main_birdloop)); struct bgp_socket *bs = p->sock; ASSERT(bs && bs->uc); @@ -232,15 +231,6 @@ bgp_close(struct bgp_proto *p) rfree(bs->sk); rem_node(&bs->n); mb_free(bs); - - if (!EMPTY_LIST(bgp_sockets)) - return; - - rfree(bgp_linpool); - bgp_linpool = NULL; - - rfree(bgp_linpool2); - bgp_linpool2 = NULL; } static inline int @@ -325,7 +315,7 @@ bgp_initiate(struct bgp_proto *p) { p->start_state = BSS_DELAY; BGP_TRACE(D_EVENTS, "Startup delayed by %d seconds due to errors", p->startup_delay); - bgp_start_timer(p->startup_timer, p->startup_delay); + bgp_start_timer(p, p->startup_timer, p->startup_delay); } else bgp_startup(p); @@ -346,6 +336,7 @@ err1: /** * bgp_start_timer - start a BGP timer + * @p: bgp_proto which the timer belongs to * @t: timer * @value: time (in seconds) to fire (0 to disable the timer) * @@ -354,14 +345,16 @@ err1: * timers. */ void -bgp_start_timer(timer *t, uint value) +bgp_start_timer(struct bgp_proto *p, timer *t, uint value) { + BGP_ASSERT_INSIDE(p); + if (value) { /* The randomization procedure is specified in RFC 4271 section 10 */ btime time = value S; btime randomize = random() % ((time / 4) + 1); - tm_start(t, time - randomize); + tm_start_in(t, time - randomize, p->p.loop); } else tm_stop(t); @@ -377,7 +370,7 @@ bgp_start_timer(timer *t, uint value) void bgp_close_conn(struct bgp_conn *conn) { - // struct bgp_proto *p = conn->bgp; + BGP_ASSERT_INSIDE(conn->bgp); DBG("BGP: Closing connection\n"); conn->packets_to_send = 0; @@ -469,6 +462,8 @@ bgp_graceful_close_conn(struct bgp_conn *conn, int subcode, byte *data, uint len static void bgp_down(struct bgp_proto *p) { + bgp_start_timer(p, p->startup_timer, 0); + if (p->start_state > BSS_PREPARE) { bgp_setup_auth(p, 0); @@ -482,21 +477,34 @@ bgp_down(struct bgp_proto *p) } static void -bgp_decision(void *vp) +bgp_active_event(void *vp) { struct bgp_proto *p = vp; - DBG("BGP: Decision start\n"); + BGP_ASSERT_INSIDE(p); + + DBG("%s: Decision start\n", p->p.name); if ((p->p.proto_state == PS_START) && (p->outgoing_conn.state == BS_IDLE) && (p->incoming_conn.state != BS_OPENCONFIRM) && !p->passive) bgp_active(p); +} + +static void +bgp_down_event(void *vp) +{ + struct bgp_proto *p = vp; + BGP_ENTER(p); + + DBG("%s: Down event\n", p->p.name); if ((p->p.proto_state == PS_STOP) && (p->outgoing_conn.state == BS_IDLE) && (p->incoming_conn.state == BS_IDLE)) bgp_down(p); + + BGP_LEAVE(p); } static struct bgp_proto * @@ -528,7 +536,7 @@ bgp_stop(struct bgp_proto *p, int subcode, byte *data, uint len) proto_notify_state(&p->p, PS_STOP); bgp_graceful_close_conn(&p->outgoing_conn, subcode, data, len); bgp_graceful_close_conn(&p->incoming_conn, subcode, data, len); - ev_schedule(p->event); + ev_send_loop(&main_birdloop, p->down_event); } static inline void @@ -575,6 +583,7 @@ bgp_conn_enter_established_state(struct bgp_conn *conn) p->link_addr = p->neigh->iface->llv6->ip; conn->sk->fast_rx = 0; + conn->sk->cork = &rt_cork; p->conn = conn; p->last_error_class = 0; @@ -719,7 +728,7 @@ bgp_conn_enter_close_state(struct bgp_conn *conn) conn->sk->rx_hook = NULL; /* Timeout for CLOSE state, if we cannot send notification soon then we just hangup */ - bgp_start_timer(conn->hold_timer, 10); + bgp_start_timer(p, conn->hold_timer, 10); if (os == BS_ESTABLISHED) bgp_conn_leave_established_state(p); @@ -733,7 +742,8 @@ bgp_conn_enter_idle_state(struct bgp_conn *conn) bgp_close_conn(conn); bgp_conn_set_state(conn, BS_IDLE); - ev_schedule(p->event); + ev_send_loop(p->p.loop, p->active_event); + ev_send_loop(&main_birdloop, p->down_event); if (os == BS_ESTABLISHED) bgp_conn_leave_established_state(p); @@ -775,25 +785,25 @@ bgp_handle_graceful_restart(struct bgp_proto *p) { case BGP_GRS_NONE: c->gr_active = BGP_GRS_ACTIVE; - rt_refresh_begin(c->c.table, &c->c); + channel_refresh_begin(&c->c); break; case BGP_GRS_ACTIVE: - rt_refresh_end(c->c.table, &c->c); - rt_refresh_begin(c->c.table, &c->c); + channel_refresh_end(&c->c); + channel_refresh_begin(&c->c); break; case BGP_GRS_LLGR: - rt_refresh_begin(c->c.table, &c->c); - rt_modify_stale(c->c.table, &c->c); + channel_refresh_begin(&c->c); + bgp_graceful_restart_feed(c); break; } } else { /* Just flush the routes */ - rt_refresh_begin(c->c.table, &c->c); - rt_refresh_end(c->c.table, &c->c); + channel_refresh_begin(&c->c); + channel_refresh_end(&c->c); } /* Reset bucket and prefix tables */ @@ -808,9 +818,54 @@ bgp_handle_graceful_restart(struct bgp_proto *p) ASSERT(p->gr_active_num > 0); proto_notify_state(&p->p, PS_START); - tm_start(p->gr_timer, p->conn->remote_caps->gr_time S); + tm_start_in(p->gr_timer, p->conn->remote_caps->gr_time S, p->p.loop); +} + +static void +bgp_graceful_restart_feed_done(struct rt_export_request *req) +{ + req->hook = NULL; } +static void +bgp_graceful_restart_feed_dump_req(struct rt_export_request *req) +{ + struct bgp_channel *c = SKIP_BACK(struct bgp_channel, stale_feed, req); + debug(" BGP-GR %s.%s export request %p\n", c->c.proto->name, c->c.name, req); +} + +static void +bgp_graceful_restart_feed_log_state_change(struct rt_export_request *req, u8 state) +{ + struct bgp_channel *c = SKIP_BACK(struct bgp_channel, stale_feed, req); + struct bgp_proto *p = (void *) c->c.proto; + BGP_TRACE(D_EVENTS, "Long-lived graceful restart export state changed to %s", rt_export_state_name(state)); + + if (state == TES_READY) + rt_stop_export(req, bgp_graceful_restart_feed_done); +} + +static void +bgp_graceful_restart_drop_export(struct rt_export_request *req UNUSED, const net_addr *n UNUSED, struct rt_pending_export *rpe UNUSED) +{ /* Nothing to do */ } + +static void +bgp_graceful_restart_feed(struct bgp_channel *c) +{ + c->stale_feed = (struct rt_export_request) { + .name = "BGP-GR", + .list = &global_work_list, + .trace_routes = c->c.debug | c->c.proto->debug, + .dump_req = bgp_graceful_restart_feed_dump_req, + .log_state_change = bgp_graceful_restart_feed_log_state_change, + .export_bulk = bgp_rte_modify_stale, + .export_one = bgp_graceful_restart_drop_export, + }; + + rt_request_export(c->c.table, &c->stale_feed); +} + + /** * bgp_graceful_restart_done - finish active BGP graceful restart * @c: BGP channel @@ -833,8 +888,11 @@ bgp_graceful_restart_done(struct bgp_channel *c) if (!p->gr_active_num) BGP_TRACE(D_EVENTS, "Neighbor graceful restart done"); + if (c->stale_feed.hook) + rt_stop_export(&c->stale_feed, bgp_graceful_restart_feed_done); + tm_stop(c->stale_timer); - rt_refresh_end(c->c.table, &c->c); + channel_refresh_end_reload(&c->c); } /** @@ -875,8 +933,8 @@ bgp_graceful_restart_timeout(timer *t) /* Channel is in GR, and supports LLGR -> start LLGR */ c->gr_active = BGP_GRS_LLGR; - tm_start(c->stale_timer, c->stale_time S); - rt_modify_stale(c->c.table, &c->c); + tm_start_in(c->stale_timer, c->stale_time S, p->p.loop); + bgp_graceful_restart_feed(c); } } else @@ -913,11 +971,11 @@ bgp_refresh_begin(struct bgp_channel *c) if (c->load_state == BFS_LOADING) { log(L_WARN "%s: BEGIN-OF-RR received before END-OF-RIB, ignoring", p->p.name); return; } - c->load_state = BFS_REFRESHING; - rt_refresh_begin(c->c.table, &c->c); + if (c->load_state == BFS_REFRESHING) + channel_refresh_end(&c->c); - if (c->c.in_table) - rt_refresh_begin(c->c.in_table, &c->c); + c->load_state = BFS_REFRESHING; + channel_refresh_begin(&c->c); } /** @@ -938,10 +996,7 @@ bgp_refresh_end(struct bgp_channel *c) { log(L_WARN "%s: END-OF-RR received without prior BEGIN-OF-RR, ignoring", p->p.name); return; } c->load_state = BFS_NONE; - rt_refresh_end(c->c.table, &c->c); - - if (c->c.in_table) - rt_prune_sync(c->c.in_table, 0); + channel_refresh_end_reload(&c->c); } @@ -955,7 +1010,7 @@ bgp_send_open(struct bgp_conn *conn) bgp_prepare_capabilities(conn); bgp_schedule_packet(conn, NULL, PKT_OPEN); bgp_conn_set_state(conn, BS_OPENSENT); - bgp_start_timer(conn->hold_timer, conn->bgp->cf->initial_hold_time); + bgp_start_timer(conn->bgp, conn->hold_timer, conn->bgp->cf->initial_hold_time); } static void @@ -1032,7 +1087,7 @@ bgp_hold_timeout(timer *t) and perhaps just not processed BGP packets in time. */ if (sk_rx_ready(conn->sk) > 0) - bgp_start_timer(conn->hold_timer, 10); + bgp_start_timer(p, conn->hold_timer, 10); else if ((conn->state == BS_ESTABLISHED) && p->llgr_ready) { BGP_TRACE(D_EVENTS, "Hold timer expired"); @@ -1077,10 +1132,12 @@ bgp_setup_conn(struct bgp_proto *p, struct bgp_conn *conn) static void bgp_setup_sk(struct bgp_conn *conn, sock *s) { + ASSERT_DIE(s->flags & SKF_THREAD); s->data = conn; s->err_hook = bgp_sock_err; s->fast_rx = 1; conn->sk = s; + sk_start(s); } static void @@ -1089,10 +1146,12 @@ bgp_active(struct bgp_proto *p) int delay = MAX(1, p->cf->connect_delay_time); struct bgp_conn *conn = &p->outgoing_conn; + BGP_ASSERT_INSIDE(p); + BGP_TRACE(D_EVENTS, "Connect delayed by %d seconds", delay); bgp_setup_conn(p, conn); bgp_conn_set_state(conn, BS_ACTIVE); - bgp_start_timer(conn->connect_timer, delay); + bgp_start_timer(p, conn->connect_timer, delay); } /** @@ -1109,9 +1168,12 @@ bgp_connect(struct bgp_proto *p) /* Enter Connect state and start establishing c struct bgp_conn *conn = &p->outgoing_conn; int hops = p->cf->multihop ? : 1; + BGP_ASSERT_INSIDE(p); + DBG("BGP: Connecting\n"); sock *s = sk_new(p->p.pool); s->type = SK_TCP_ACTIVE; + s->flags |= SKF_THREAD; s->saddr = p->local_ip; s->daddr = p->remote_ip; s->dport = p->cf->remote_port; @@ -1139,7 +1201,7 @@ bgp_connect(struct bgp_proto *p) /* Enter Connect state and start establishing c goto err; DBG("BGP: Waiting for connect success\n"); - bgp_start_timer(conn->connect_timer, p->cf->connect_retry_time); + bgp_start_timer(p, conn->connect_timer, p->cf->connect_retry_time); return; err: @@ -1211,6 +1273,18 @@ bgp_incoming_connection(sock *sk, uint dummy UNUSED) return 0; } + if (p->p.loop == &main_birdloop) + { + /* Protocol is down for whatever reason. No need for locking. */ + BGP_TRACE(D_EVENTS, "Incoming connection from %I%J (port %d) rejected (protocol is down)", + sk->daddr, ipa_is_link_local(sk->daddr) ? sk->iface : NULL, + sk->dport); + rfree(sk); + return 0; + } + + BGP_ENTER(p); + /* * BIRD should keep multiple incoming connections in OpenSent state (for * details RFC 4271 8.2.1 par 3), but it keeps just one. Duplicate incoming @@ -1240,6 +1314,7 @@ bgp_incoming_connection(sock *sk, uint dummy UNUSED) if (!acc) { rfree(sk); + BGP_LEAVE(p); return 0; } @@ -1265,6 +1340,7 @@ bgp_incoming_connection(sock *sk, uint dummy UNUSED) p = bgp_spawn(p, sk->daddr); p->postponed_sk = sk; rmove(sk, p->p.pool); + BGP_LEAVE(p); return 0; } @@ -1272,12 +1348,14 @@ bgp_incoming_connection(sock *sk, uint dummy UNUSED) bgp_setup_conn(p, &p->incoming_conn); bgp_setup_sk(&p->incoming_conn, sk); bgp_send_open(&p->incoming_conn); + BGP_LEAVE(p); return 0; err: sk_log_error(sk, p->p.name); log(L_ERR "%s: Incoming connection aborted", p->p.name); rfree(sk); + BGP_LEAVE(p); return 0; } @@ -1312,10 +1390,9 @@ bgp_neigh_notify(neighbor *n) struct bgp_proto *p = (struct bgp_proto *) n->proto; int ps = p->p.proto_state; - if (n != p->neigh) - return; + BGP_ASSERT_INSIDE(p); - if ((ps == PS_DOWN) || (ps == PS_STOP)) + if ((n != p->neigh) || (ps == PS_DOWN) || (ps == PS_STOP)) return; int prepare = (ps == PS_START) && (p->start_state == BSS_PREPARE); @@ -1393,7 +1470,7 @@ bgp_update_bfd(struct bgp_proto *p, const struct bfd_options *bfd) if (bfd && !p->bfd_req && !bgp_is_dynamic(p)) p->bfd_req = bfd_request_session(p->p.pool, p->remote_ip, p->local_ip, p->cf->multihop ? NULL : p->neigh->iface, - p->p.vrf, bgp_bfd_notify, p, bfd); + p->p.vrf, bgp_bfd_notify, p, birdloop_event_list(p->p.loop), bfd); if (!bfd && p->bfd_req) { @@ -1408,12 +1485,9 @@ bgp_reload_routes(struct channel *C) struct bgp_proto *p = (void *) C->proto; struct bgp_channel *c = (void *) C; - ASSERT(p->conn && (p->route_refresh || c->c.in_table)); + ASSERT(p->conn && (p->route_refresh)); - if (c->c.in_table) - channel_schedule_reload(C); - else - bgp_schedule_packet(p->conn, c, PKT_ROUTE_REFRESH); + bgp_schedule_packet(p->conn, c, PKT_ROUTE_REFRESH); } static void @@ -1474,9 +1548,12 @@ bgp_start_locked(struct object_lock *lock) struct bgp_proto *p = lock->data; const struct bgp_config *cf = p->cf; + BGP_ENTER(p); + if (p->p.proto_state != PS_START) { DBG("BGP: Got lock in different state %d\n", p->p.proto_state); + BGP_LEAVE(p); return; } @@ -1486,10 +1563,11 @@ bgp_start_locked(struct object_lock *lock) { /* Multi-hop sessions do not use neighbor entries */ bgp_initiate(p); + BGP_LEAVE(p); return; } - neighbor *n = neigh_find(&p->p, p->remote_ip, cf->iface, NEF_STICKY); + neighbor *n = neigh_find(&p->p, p->remote_ip, cf->iface, NEF_STICKY | NEF_NOTIFY_MAIN); if (!n) { log(L_ERR "%s: Invalid remote address %I%J", p->p.name, p->remote_ip, cf->iface); @@ -1497,6 +1575,7 @@ bgp_start_locked(struct object_lock *lock) p->p.disabled = 1; bgp_store_error(p, NULL, BE_MISC, BEM_INVALID_NEXT_HOP); proto_notify_state(&p->p, PS_DOWN); + BGP_LEAVE(p); return; } @@ -1508,6 +1587,8 @@ bgp_start_locked(struct object_lock *lock) BGP_TRACE(D_EVENTS, "Waiting for link on %s", n->iface->name); else bgp_start_neighbor(p); + + BGP_LEAVE(p); } static int @@ -1546,10 +1627,13 @@ bgp_start(struct proto *P) p->stats.rx_bytes = p->stats.tx_bytes = 0; p->last_rx_update = 0; - p->event = ev_new_init(p->p.pool, bgp_decision, p); + p->active_event = ev_new_init(p->p.pool, bgp_active_event, p); + p->down_event = ev_new_init(p->p.pool, bgp_down_event, p); p->startup_timer = tm_new_init(p->p.pool, bgp_startup_timeout, p, 0, 0); p->gr_timer = tm_new_init(p->p.pool, bgp_graceful_restart_timeout, p, 0, 0); + p->rx_lp = lp_new_default(p->p.pool); + p->local_id = proto_get_router_id(P->cf); if (p->rr_client) p->rr_cluster_id = p->cf->rr_cluster_id ? p->cf->rr_cluster_id : p->local_id; @@ -1677,6 +1761,13 @@ done: return p->p.proto_state; } +struct rte_owner_class bgp_rte_owner_class = { + .get_route_info = bgp_get_route_info, + .rte_better = bgp_rte_better, + .rte_mergable = bgp_rte_mergable, + .rte_igp_metric = bgp_rte_igp_metric, +}; + static struct proto * bgp_init(struct proto_config *CF) { @@ -1690,10 +1781,9 @@ bgp_init(struct proto_config *CF) P->reload_routes = bgp_reload_routes; P->feed_begin = bgp_feed_begin; P->feed_end = bgp_feed_end; - P->rte_better = bgp_rte_better; - P->rte_mergable = bgp_rte_mergable; - P->rte_recalculate = cf->deterministic_med ? bgp_rte_recalculate : NULL; - P->rte_modify = bgp_rte_modify_stale; + + P->sources.class = &bgp_rte_owner_class; + P->sources.rte_recalculate = cf->deterministic_med ? bgp_rte_recalculate : NULL; p->cf = cf; p->is_internal = (cf->local_as == cf->remote_as); @@ -1745,17 +1835,19 @@ bgp_channel_start(struct channel *C) ip_addr src = p->local_ip; if (c->igp_table_ip4) - rt_lock_table(c->igp_table_ip4); + RT_LOCKED(c->igp_table_ip4, t) + rt_lock_table(t); if (c->igp_table_ip6) - rt_lock_table(c->igp_table_ip6); + RT_LOCKED(c->igp_table_ip6, t) + rt_lock_table(t); c->pool = p->p.pool; // XXXX bgp_init_bucket_table(c); bgp_init_prefix_table(c); if (c->cf->import_table) - channel_setup_in_table(C); + channel_setup_in_table(C, 0); if (c->cf->export_table) channel_setup_out_table(C); @@ -1829,10 +1921,12 @@ bgp_channel_cleanup(struct channel *C) struct bgp_channel *c = (void *) C; if (c->igp_table_ip4) - rt_unlock_table(c->igp_table_ip4); + RT_LOCKED(c->igp_table_ip4, t) + rt_unlock_table(t); if (c->igp_table_ip6) - rt_unlock_table(c->igp_table_ip6); + RT_LOCKED(c->igp_table_ip6, t) + rt_unlock_table(t); c->index = 0; @@ -2430,6 +2524,9 @@ bgp_show_proto_info(struct proto *P) { struct bgp_proto *p = (struct bgp_proto *) P; + if (p->p.proto_state != PS_DOWN) + BGP_ASSERT_INSIDE(p); + cli_msg(-1006, " BGP state: %s", bgp_state_dsc(p)); if (bgp_is_dynamic(p) && p->cf->remote_range) @@ -2556,6 +2653,5 @@ struct protocol proto_bgp = { .copy_config = bgp_copy_config, .get_status = bgp_get_status, .get_attr = bgp_get_attr, - .get_route_info = bgp_get_route_info, .show_proto_info = bgp_show_proto_info }; diff --git a/proto/bgp/bgp.h b/proto/bgp/bgp.h index cca4b448..d5ac3bd9 100644 --- a/proto/bgp/bgp.h +++ b/proto/bgp/bgp.h @@ -200,6 +200,10 @@ struct bgp_channel_config { #define BGP_BFD_GRACEFUL 2 /* BFD down triggers graceful restart */ +/* rte->pflags */ +#define BGP_REF_SUPPRESSED 0x1 /* Used for deterministic MED comparison */ +#define BGP_REF_STALE 0x2 /* Route is LLGR_STATE */ +#define BGP_REF_NOT_STALE 0x4 /* Route is NOT LLGR_STATE */ struct bgp_af_caps { u32 afi; @@ -308,6 +312,7 @@ struct bgp_proto { struct bgp_conn *conn; /* Connection we have established */ struct bgp_conn outgoing_conn; /* Outgoing connection we're working with */ struct bgp_conn incoming_conn; /* Incoming connection we have neither accepted nor rejected yet */ + struct linpool *rx_lp; /* Linpool for parsing received updates */ struct object_lock *lock; /* Lock for neighbor connection */ struct neighbor *neigh; /* Neighbor entry corresponding to remote ip, NULL if multihop */ struct bgp_socket *sock; /* Shared listening socket */ @@ -317,7 +322,8 @@ struct bgp_proto { btime last_established; /* Last time of enter/leave of established state */ btime last_rx_update; /* Last time of RX update */ ip_addr link_addr; /* Link-local version of local_ip */ - event *event; /* Event for respawning and shutting process */ + event *active_event; /* Event for respawning */ + event *down_event; /* Event to shut down */ timer *startup_timer; /* Timer used to delay protocol startup due to previous errors (startup_delay) */ timer *gr_timer; /* Timer waiting for reestablishment after graceful restart */ int dynamic_name_counter; /* Counter for dynamic BGP names */ @@ -362,6 +368,7 @@ struct bgp_channel { timer *stale_timer; /* Long-lived stale timer for LLGR */ u32 stale_time; /* Stored LLGR stale time from last session */ + struct rt_export_request stale_feed; /* Feeder request for stale route modification */ u8 add_path_rx; /* Session expects receive of ADD-PATH extended NLRI */ u8 add_path_tx; /* Session expects transmit of ADD-PATH extended NLRI */ @@ -489,11 +496,8 @@ bgp_parse_error(struct bgp_parse_state *s, uint subcode) longjmp(s->err_jmpbuf, 1); } -extern struct linpool *bgp_linpool; -extern struct linpool *bgp_linpool2; - -void bgp_start_timer(timer *t, uint value); +void bgp_start_timer(struct bgp_proto *p, timer *t, uint value); void bgp_check_config(struct bgp_config *c); void bgp_error(struct bgp_conn *c, unsigned code, unsigned subcode, byte *data, int len); void bgp_close_conn(struct bgp_conn *c); @@ -513,11 +517,12 @@ struct rte_source *bgp_find_source(struct bgp_proto *p, u32 path_id); struct rte_source *bgp_get_source(struct bgp_proto *p, u32 path_id); static inline int -rte_resolvable(rte *rt) +rta_resolvable(rta *a) { - return rt->attrs->dest == RTD_UNICAST; + return a->dest == RTD_UNICAST; } +extern struct rte_owner_class bgp_rte_owner_class; #ifdef LOCAL_DEBUG #define BGP_FORCE_DEBUG 1 @@ -580,24 +585,31 @@ void bgp_free_prefix(struct bgp_channel *c, struct bgp_prefix *bp); int bgp_rte_better(struct rte *, struct rte *); int bgp_rte_mergable(rte *pri, rte *sec); -int bgp_rte_recalculate(rtable *table, net *net, rte *new, rte *old, rte *old_best); -struct rte *bgp_rte_modify_stale(struct rte *r, struct linpool *pool); -void bgp_rt_notify(struct proto *P, struct channel *C, net *n, rte *new, rte *old); -int bgp_preexport(struct proto *, struct rte **, struct linpool *); +int bgp_rte_recalculate(rtable_private *table, net *net, rte *new, rte *old, rte *old_best); +void bgp_rte_modify_stale(struct rt_export_request *, const net_addr *, struct rt_pending_export *, rte **, uint); +u32 bgp_rte_igp_metric(struct rte *); +void bgp_rt_notify(struct proto *P, struct channel *C, const net_addr *n, rte *new, const rte *old); +int bgp_preexport(struct channel *, struct rte *); int bgp_get_attr(const struct eattr *e, byte *buf, int buflen); -void bgp_get_route_info(struct rte *, byte *buf); -int bgp_total_aigp_metric_(rte *e, u64 *metric, const struct adata **ad); +void bgp_get_route_info(struct rte *, byte *); +int bgp_total_aigp_metric_(rta *a, u64 *metric, const struct adata **ad); + +static inline struct bgp_proto *bgp_rte_proto(struct rte *rte) +{ + return (rte->src->owner->class == &bgp_rte_owner_class) ? + SKIP_BACK(struct bgp_proto, p.sources, rte->src->owner) : NULL; +} #define BGP_AIGP_METRIC 1 #define BGP_AIGP_MAX U64(0xffffffffffffffff) static inline u64 -bgp_total_aigp_metric(rte *r) +bgp_total_aigp_metric(rta *a) { u64 metric = BGP_AIGP_MAX; const struct adata *ad; - bgp_total_aigp_metric_(r, &metric, &ad); + bgp_total_aigp_metric_(a, &metric, &ad); return metric; } @@ -749,5 +761,10 @@ void bgp_update_next_hop(struct bgp_export_state *s, eattr *a, ea_list **to); #define ORIGIN_EGP 1 #define ORIGIN_INCOMPLETE 2 +/* Loop */ + +#define BGP_ENTER(bgp) birdloop_enter(bgp->p.loop) +#define BGP_LEAVE(bgp) birdloop_leave(bgp->p.loop) +#define BGP_ASSERT_INSIDE(bgp) ASSERT_DIE((bgp->p.loop != &main_birdloop) && birdloop_inside(bgp->p.loop)) #endif diff --git a/proto/bgp/config.Y b/proto/bgp/config.Y index 2dfbdca9..8e42bbdb 100644 --- a/proto/bgp/config.Y +++ b/proto/bgp/config.Y @@ -48,6 +48,7 @@ proto: bgp_proto '}' ; bgp_proto_start: proto_start BGP { this_proto = proto_config_new(&proto_bgp, $1); + this_proto->loop_order = DOMAIN_ORDER(proto); BGP_CFG->local_port = BGP_PORT; BGP_CFG->remote_port = BGP_PORT; BGP_CFG->multihop = -1; /* undefined */ diff --git a/proto/bgp/packets.c b/proto/bgp/packets.c index 99b5d5b4..88b66040 100644 --- a/proto/bgp/packets.c +++ b/proto/bgp/packets.c @@ -914,7 +914,7 @@ bgp_rx_open(struct bgp_conn *conn, byte *pkt, uint len) conn->hold_time, conn->keepalive_time, p->remote_as, p->remote_id, conn->as4_session); bgp_schedule_packet(conn, NULL, PKT_KEEPALIVE); - bgp_start_timer(conn->hold_timer, conn->hold_time); + bgp_start_timer(p, conn->hold_timer, conn->hold_time); bgp_conn_enter_openconfirm_state(conn); } @@ -971,7 +971,7 @@ bgp_apply_next_hop(struct bgp_parse_state *s, rta *a, ip_addr gw, ip_addr ll) s->hostentry = rt_get_hostentry(tab, gw, ll, c->c.table); if (!s->mpls) - rta_apply_hostentry(a, s->hostentry, NULL); + rta_apply_hostentry(a, s->hostentry, NULL, s->pool); /* With MPLS, hostentry is applied later in bgp_apply_mpls_labels() */ } @@ -1005,7 +1005,7 @@ bgp_apply_mpls_labels(struct bgp_parse_state *s, rta *a, u32 *labels, uint lnum) ms.len = lnum; memcpy(ms.stack, labels, 4*lnum); - rta_apply_hostentry(a, s->hostentry, &ms); + rta_apply_hostentry(a, s->hostentry, &ms, s->pool); } } @@ -1339,6 +1339,8 @@ bgp_rte_update(struct bgp_parse_state *s, net_addr *n, u32 path_id, rta *a0) { if (path_id != s->last_id) { + rt_unlock_source(s->last_src); + s->last_src = rt_get_source(&s->proto->p, path_id); s->last_id = path_id; @@ -1349,28 +1351,25 @@ bgp_rte_update(struct bgp_parse_state *s, net_addr *n, u32 path_id, rta *a0) if (!a0) { /* Route withdraw */ - rte_update3(&s->channel->c, n, NULL, s->last_src); + rte_update(&s->channel->c, n, NULL, s->last_src); return; } /* Prepare cached route attributes */ if (s->cached_rta == NULL) { - a0->src = s->last_src; - /* Workaround for rta_lookup() breaking eattrs */ ea_list *ea = a0->eattrs; s->cached_rta = rta_lookup(a0); a0->eattrs = ea; } - rta *a = rta_clone(s->cached_rta); - rte *e = rte_get_temp(a); + rte e0 = { + .attrs = s->cached_rta, + .src = s->last_src, + }; - e->pflags = 0; - e->u.bgp.suppressed = 0; - e->u.bgp.stale = -1; - rte_update3(&s->channel->c, n, e, s->last_src); + rte_update(&s->channel->c, n, &e0, s->last_src); } static void @@ -2296,7 +2295,7 @@ again: ; struct bgp_write_state s = { .proto = p, .channel = c, - .pool = bgp_linpool, + .pool = c->c.rte_update_pool, .mp_reach = (c->afi != BGP_AF_IPV4) || c->ext_next_hop, .as4_session = p->as4_session, .add_path = c->add_path_tx, @@ -2424,6 +2423,7 @@ bgp_decode_nlri(struct bgp_parse_state *s, u32 afi, byte *nlri, uint len, ea_lis s->last_id = 0; s->last_src = s->proto->p.main_source; + rt_lock_source(s->last_src); /* * IPv4 BGP and MP-BGP may be used together in one update, therefore we do not @@ -2440,6 +2440,7 @@ bgp_decode_nlri(struct bgp_parse_state *s, u32 afi, byte *nlri, uint len, ea_lis a->scope = SCOPE_UNIVERSE; a->from = s->proto->remote_ip; a->eattrs = ea; + a->pref = c->c.preference; c->desc->decode_next_hop(s, nh, nh_len, a); bgp_finish_attrs(s, a); @@ -2453,6 +2454,8 @@ bgp_decode_nlri(struct bgp_parse_state *s, u32 afi, byte *nlri, uint len, ea_lis rta_free(s->cached_rta); s->cached_rta = NULL; + + rt_unlock_source(s->last_src); } static void @@ -2472,12 +2475,12 @@ bgp_rx_update(struct bgp_conn *conn, byte *pkt, uint len) if (conn->state != BS_ESTABLISHED) { bgp_error(conn, 5, fsm_err_subcode[conn->state], NULL, 0); return; } - bgp_start_timer(conn->hold_timer, conn->hold_time); + bgp_start_timer(p, conn->hold_timer, conn->hold_time); /* Initialize parse state */ struct bgp_parse_state s = { .proto = p, - .pool = bgp_linpool, + .pool = p->rx_lp, .as4_session = p->as4_session, }; @@ -2808,7 +2811,7 @@ bgp_fire_tx(struct bgp_conn *conn) { conn->packets_to_send &= ~(1 << PKT_KEEPALIVE); BGP_TRACE(D_PACKETS, "Sending KEEPALIVE"); - bgp_start_timer(conn->keepalive_timer, conn->keepalive_time); + bgp_start_timer(p, conn->keepalive_timer, conn->keepalive_time); return bgp_send(conn, PKT_KEEPALIVE, BGP_HEADER_LENGTH); } else while (conn->channels_to_send) @@ -2893,7 +2896,7 @@ bgp_schedule_packet(struct bgp_conn *conn, struct bgp_channel *c, int type) conn->packets_to_send |= 1 << type; if ((conn->sk->tpos == conn->sk->tbuf) && !ev_active(conn->tx_ev)) - ev_schedule(conn->tx_ev); + ev_send_loop(conn->bgp->p.loop, conn->tx_ev); } void bgp_kick_tx(void *vconn) @@ -2906,7 +2909,7 @@ bgp_kick_tx(void *vconn) ; if (!max && !ev_active(conn->tx_ev)) - ev_schedule(conn->tx_ev); + ev_send_loop(conn->bgp->p.loop, conn->tx_ev); } void @@ -2920,7 +2923,7 @@ bgp_tx(sock *sk) ; if (!max && !ev_active(conn->tx_ev)) - ev_schedule(conn->tx_ev); + ev_send_loop(conn->bgp->p.loop, conn->tx_ev); } @@ -3102,7 +3105,7 @@ bgp_rx_keepalive(struct bgp_conn *conn) struct bgp_proto *p = conn->bgp; BGP_TRACE(D_PACKETS, "Got KEEPALIVE"); - bgp_start_timer(conn->hold_timer, conn->hold_time); + bgp_start_timer(p, conn->hold_timer, conn->hold_time); if (conn->state == BS_OPENCONFIRM) { bgp_conn_enter_established_state(conn); return; } diff --git a/proto/mrt/mrt.c b/proto/mrt/mrt.c index 8d97c860..e12f7743 100644 --- a/proto/mrt/mrt.c +++ b/proto/mrt/mrt.c @@ -113,13 +113,13 @@ mrt_buffer_flush(buffer *b) } #define MRT_DEFINE_TYPE(S, T) \ - static inline void mrt_put_##S##_(buffer *b, T x) \ + UNUSED static inline void mrt_put_##S##_(buffer *b, T x) \ { \ put_##S(b->pos, x); \ b->pos += sizeof(T); \ } \ \ - static inline void mrt_put_##S(buffer *b, T x) \ + UNUSED static inline void mrt_put_##S(buffer *b, T x) \ { \ mrt_buffer_need(b, sizeof(T)); \ put_##S(b->pos, x); \ @@ -228,7 +228,7 @@ mrt_next_table_(rtable *tab, rtable *tab_ptr, const char *pattern) NODE_VALID(tn); tn = tn->next) { - tab = SKIP_BACK(struct rtable, n, tn); + tab = SKIP_BACK(rtable, n, tn); if (patmatch(pattern, tab->name) && ((tab->addr_type == NET_IP4) || (tab->addr_type == NET_IP6))) return tab; @@ -243,13 +243,21 @@ mrt_next_table(struct mrt_table_dump_state *s) rtable *tab = mrt_next_table_(s->table, s->table_ptr, s->table_expr); if (s->table) - rt_unlock_table(s->table); + { + RT_LOCK(s->table); + rt_unlock_table(RT_PRIV(s->table)); + RT_UNLOCK(s->table); + } s->table = tab; s->ipv4 = tab ? (tab->addr_type == NET_IP4) : 0; if (s->table) - rt_lock_table(s->table); + { + RT_LOCK(s->table); + rt_lock_table(RT_PRIV(s->table)); + RT_UNLOCK(s->table); + } return s->table; } @@ -460,7 +468,7 @@ mrt_rib_table_entry_bgp_attrs(struct mrt_table_dump_state *s, rte *r) return; fail: - mrt_log(s, "Attribute list too long for %N", r->net->n.addr); + mrt_log(s, "Attribute list too long for %N", r->net); } #endif @@ -472,9 +480,9 @@ mrt_rib_table_entry(struct mrt_table_dump_state *s, rte *r) #ifdef CONFIG_BGP /* Find peer index */ - if (r->attrs->src->proto->proto == &proto_bgp) + struct bgp_proto *p = bgp_rte_proto(r); + if (p) { - struct bgp_proto *p = (void *) r->attrs->src->proto; struct mrt_peer_entry *n = HASH_FIND(s->peer_hash, PEER, p->remote_id, p->remote_as, p->remote_ip); @@ -488,7 +496,7 @@ mrt_rib_table_entry(struct mrt_table_dump_state *s, rte *r) /* Path Identifier */ if (s->add_path) - mrt_put_u32(b, r->attrs->src->private_id); + mrt_put_u32(b, r->src->private_id); /* Route Attributes */ mrt_put_u16(b, 0); @@ -512,26 +520,21 @@ mrt_rib_table_dump(struct mrt_table_dump_state *s, net *n, int add_path) mrt_init_message(&s->buf, MRT_TABLE_DUMP_V2, subtype); mrt_rib_table_header(s, n->n.addr); - rte *rt, *rt0; - for (rt0 = n->routes; rt = rt0; rt0 = rt0->next) + for (struct rte_storage *rt, *rt0 = n->routes; rt = rt0; rt0 = rt0->next) { - if (rte_is_filtered(rt)) + if (rte_is_filtered(&rt->rte)) continue; /* Skip routes that should be reported in the other phase */ - if (!s->always_add_path && (!rt->attrs->src->private_id != !s->add_path)) + if (!s->always_add_path && (!rt->rte.src->private_id != !s->add_path)) { s->want_add_path = 1; continue; } - rte_make_tmp_attrs(&rt, s->linpool, NULL); - - if (f_run(s->filter, &rt, s->linpool, 0) <= F_ACCEPT) - mrt_rib_table_entry(s, rt); - - if (rt != rt0) - rte_free(rt); + rte e = rt->rte; + if (f_run(s->filter, &e, s->linpool, 0) <= F_ACCEPT) + mrt_rib_table_entry(s, &e); lp_flush(s->linpool); } @@ -558,10 +561,11 @@ mrt_rib_table_dump(struct mrt_table_dump_state *s, net *n, int add_path) static struct mrt_table_dump_state * mrt_table_dump_init(pool *pp) { - pool *pool = rp_new(pp, "MRT Table Dump"); + pool *pool = rp_new(pp, &main_birdloop, "MRT Table Dump"); struct mrt_table_dump_state *s = mb_allocz(pool, sizeof(struct mrt_table_dump_state)); s->pool = pool; + s->parent = pp; s->linpool = lp_new(pool, 4080); s->peer_lp = lp_new(pool, 4080); mrt_buffer_init(&s->buf, pool, 2 * MRT_ATTR_BUFFER_SIZE); @@ -578,18 +582,27 @@ mrt_table_dump_init(pool *pp) static void mrt_table_dump_free(struct mrt_table_dump_state *s) { - if (s->table_open) - FIB_ITERATE_UNLINK(&s->fit, &s->table->fib); - if (s->table) - rt_unlock_table(s->table); + { + RT_LOCK(s->table); + + if (s->table_open) + FIB_ITERATE_UNLINK(&s->fit, &RT_PRIV(s->table)->fib); + + rt_unlock_table(RT_PRIV(s->table)); + RT_UNLOCK(s->table); + } if (s->table_ptr) - rt_unlock_table(s->table_ptr); + { + RT_LOCK(s->table_ptr); + rt_unlock_table(RT_PRIV(s->table_ptr)); + RT_UNLOCK(s->table_ptr); + } config_del_obstacle(s->config); - rfree(s->pool); + rp_free(s->pool, s->parent); } @@ -601,8 +614,14 @@ mrt_table_dump_step(struct mrt_table_dump_state *s) s->max = 2048; s->bws = &bws; + rtable_private *tab; + if (s->table_open) + { + RT_LOCK(s->table); + tab = RT_PRIV(s->table); goto step; + } while (mrt_next_table(s)) { @@ -611,15 +630,18 @@ mrt_table_dump_step(struct mrt_table_dump_state *s) mrt_peer_table_dump(s); - FIB_ITERATE_INIT(&s->fit, &s->table->fib); + RT_LOCK(s->table); + tab = RT_PRIV(s->table); + FIB_ITERATE_INIT(&s->fit, &tab->fib); s->table_open = 1; step: - FIB_ITERATE_START(&s->table->fib, &s->fit, net, n) + FIB_ITERATE_START(&tab->fib, &s->fit, net, n) { if (s->max < 0) { FIB_ITERATE_PUT(&s->fit); + RT_UNLOCK(s->table); return 0; } @@ -639,6 +661,7 @@ mrt_table_dump_step(struct mrt_table_dump_state *s) mrt_peer_table_flush(s); } + RT_UNLOCK(s->table); return 1; } @@ -666,7 +689,11 @@ mrt_timer(timer *t) s->always_add_path = cf->always_add_path; if (s->table_ptr) - rt_lock_table(s->table_ptr); + { + RT_LOCK(s->table_ptr); + rt_lock_table(RT_PRIV(s->table_ptr)); + RT_UNLOCK(s->table_ptr); + } p->table_dump = s; ev_schedule(p->event); @@ -739,7 +766,11 @@ mrt_dump_cmd(struct mrt_dump_data *d) s->filename = d->filename; if (s->table_ptr) - rt_lock_table(s->table_ptr); + { + RT_LOCK(s->table_ptr); + rt_lock_table(RT_PRIV(s->table_ptr)); + RT_UNLOCK(s->table_ptr); + } this_cli->cont = mrt_dump_cont; this_cli->cleanup = mrt_dump_cleanup; diff --git a/proto/mrt/mrt.h b/proto/mrt/mrt.h index 4ff94c12..2e616f6f 100644 --- a/proto/mrt/mrt.h +++ b/proto/mrt/mrt.h @@ -40,7 +40,7 @@ struct mrt_proto { struct mrt_dump_data { const char *table_expr; - struct rtable *table_ptr; + rtable *table_ptr; const struct filter *filter; const char *filename; }; @@ -60,20 +60,21 @@ struct mrt_table_dump_state { /* Configuration information */ const char *table_expr; /* Wildcard for table name (or NULL) */ - struct rtable *table_ptr; /* Explicit table (or NULL) */ + rtable *table_ptr; /* Explicit table (or NULL) */ const struct filter *filter; /* Optional filter */ const char *filename; /* Filename pattern */ int always_add_path; /* Always use *_ADDPATH message subtypes */ /* Allocated by mrt_table_dump_init() */ pool *pool; /* Pool for table dump */ + pool *parent; /* Parent pool for cleanup */ linpool *linpool; /* Temporary linear pool */ linpool *peer_lp; /* Linear pool for peer entries in peer_hash */ buffer buf; /* Buffer for MRT messages */ HASH(struct mrt_peer_entry) peer_hash; /* Hash for peers to find the index */ - struct rtable *table; /* Processed table, NULL initially */ + rtable *table; /* Processed table, NULL initially */ struct fib_iterator fit; /* Iterator in processed table */ int table_open; /* Whether iterator is linked */ diff --git a/proto/ospf/iface.c b/proto/ospf/iface.c index f38b8210..049030ac 100644 --- a/proto/ospf/iface.c +++ b/proto/ospf/iface.c @@ -311,7 +311,7 @@ ospf_iface_remove(struct ospf_iface *ifa) ospf_iface_sm(ifa, ISM_DOWN); rem_node(NODE ifa); - rfree(ifa->pool); + rp_free(ifa->pool, p->p.pool); } void @@ -522,7 +522,10 @@ static inline void add_nbma_node(struct ospf_iface *ifa, struct nbma_node *src, int found) { struct nbma_node *n = mb_alloc(ifa->pool, sizeof(struct nbma_node)); + + n->n = (node) {}; add_tail(&ifa->nbma_list, NODE n); + n->ip = src->ip; n->eligible = src->eligible; n->found = found; @@ -564,7 +567,7 @@ ospf_iface_new(struct ospf_area *oa, struct ifa *addr, struct ospf_iface_patt *i OSPF_TRACE(D_EVENTS, "Adding interface %s (%N) to area %R", iface->name, &addr->prefix, oa->areaid); - pool = rp_new(p->p.pool, "OSPF Interface"); + pool = rp_new(p->p.pool, p->p.loop, "OSPF Interface"); ifa = mb_allocz(pool, sizeof(struct ospf_iface)); ifa->iface = iface; ifa->addr = addr; @@ -684,7 +687,7 @@ ospf_iface_new_vlink(struct ospf_proto *p, struct ospf_iface_patt *ip) /* Vlink ifname is stored just after the ospf_iface structure */ - pool = rp_new(p->p.pool, "OSPF Vlink"); + pool = rp_new(p->p.pool, p->p.loop, "OSPF Vlink"); ifa = mb_allocz(pool, sizeof(struct ospf_iface) + 16); ifa->oa = p->backbone; ifa->cf = ip; @@ -1222,7 +1225,8 @@ ospf_reconfigure_ifaces2(struct ospf_proto *p) struct iface *iface; struct ifa *a; - WALK_LIST(iface, iface_list) + IFACE_LEGACY_ACCESS; + WALK_LIST(iface, global_iface_list) { if (! (iface->flags & IF_UP)) continue; @@ -1268,7 +1272,8 @@ ospf_reconfigure_ifaces3(struct ospf_proto *p) struct iface *iface; struct ifa *a; - WALK_LIST(iface, iface_list) + IFACE_LEGACY_ACCESS; + WALK_LIST(iface, global_iface_list) { if (! (iface->flags & IF_UP)) continue; diff --git a/proto/ospf/neighbor.c b/proto/ospf/neighbor.c index ca369819..4ae0d3fa 100644 --- a/proto/ospf/neighbor.c +++ b/proto/ospf/neighbor.c @@ -80,7 +80,7 @@ struct ospf_neighbor * ospf_neighbor_new(struct ospf_iface *ifa) { struct ospf_proto *p = ifa->oa->po; - struct pool *pool = rp_new(p->p.pool, "OSPF Neighbor"); + struct pool *pool = rp_new(p->p.pool, p->p.loop, "OSPF Neighbor"); struct ospf_neighbor *n = mb_allocz(pool, sizeof(struct ospf_neighbor)); n->pool = pool; @@ -120,7 +120,7 @@ ospf_neigh_down(struct ospf_neighbor *n) s_get(&(n->dbsi)); release_lsrtl(p, n); rem_node(NODE n); - rfree(n->pool); + rp_free(n->pool, p->p.pool); OSPF_TRACE(D_EVENTS, "Neighbor %R on %s removed", rid, ifa->ifname); } @@ -777,7 +777,7 @@ ospf_neigh_update_bfd(struct ospf_neighbor *n, int use_bfd) if (use_bfd && !n->bfd_req) n->bfd_req = bfd_request_session(n->pool, n->ip, n->ifa->addr->ip, n->ifa->iface, p->p.vrf, - ospf_neigh_bfd_hook, n, NULL); + ospf_neigh_bfd_hook, n, birdloop_event_list(p->p.loop), NULL); if (!use_bfd && n->bfd_req) { diff --git a/proto/ospf/ospf.c b/proto/ospf/ospf.c index ba8c2e2b..16774df6 100644 --- a/proto/ospf/ospf.c +++ b/proto/ospf/ospf.c @@ -107,12 +107,10 @@ #include <stdlib.h> #include "ospf.h" -static int ospf_preexport(struct proto *P, rte **new, struct linpool *pool); -static void ospf_make_tmp_attrs(struct rte *rt, struct linpool *pool); -static void ospf_store_tmp_attrs(struct rte *rt, struct linpool *pool); +static int ospf_preexport(struct channel *C, rte *new); static void ospf_reload_routes(struct channel *C); static int ospf_rte_better(struct rte *new, struct rte *old); -static int ospf_rte_same(struct rte *new, struct rte *old); +static u32 ospf_rte_igp_metric(struct rte *rt); static void ospf_disp(timer *timer); @@ -378,10 +376,8 @@ ospf_init(struct proto_config *CF) P->reload_routes = ospf_reload_routes; P->feed_begin = ospf_feed_begin; P->feed_end = ospf_feed_end; - P->make_tmp_attrs = ospf_make_tmp_attrs; - P->store_tmp_attrs = ospf_store_tmp_attrs; - P->rte_better = ospf_rte_better; - P->rte_same = ospf_rte_same; + + P->sources.class = &ospf_rte_owner_class; return P; } @@ -390,7 +386,9 @@ ospf_init(struct proto_config *CF) static int ospf_rte_better(struct rte *new, struct rte *old) { - if (new->u.ospf.metric1 == LSINFINITY) + u32 new_metric1 = ea_get_int(new->attrs->eattrs, EA_OSPF_METRIC1, LSINFINITY); + + if (new_metric1 == LSINFINITY) return 0; if(new->attrs->source < old->attrs->source) return 1; @@ -398,27 +396,27 @@ ospf_rte_better(struct rte *new, struct rte *old) if(new->attrs->source == RTS_OSPF_EXT2) { - if(new->u.ospf.metric2 < old->u.ospf.metric2) return 1; - if(new->u.ospf.metric2 > old->u.ospf.metric2) return 0; + u32 old_metric2 = ea_get_int(old->attrs->eattrs, EA_OSPF_METRIC2, LSINFINITY); + u32 new_metric2 = ea_get_int(new->attrs->eattrs, EA_OSPF_METRIC2, LSINFINITY); + if(new_metric2 < old_metric2) return 1; + if(new_metric2 > old_metric2) return 0; } - if (new->u.ospf.metric1 < old->u.ospf.metric1) + u32 old_metric1 = ea_get_int(old->attrs->eattrs, EA_OSPF_METRIC1, LSINFINITY); + if (new_metric1 < old_metric1) return 1; return 0; /* Old is shorter or same */ } -static int -ospf_rte_same(struct rte *new, struct rte *old) +static u32 +ospf_rte_igp_metric(struct rte *rt) { - /* new->attrs == old->attrs always */ - return - new->u.ospf.metric1 == old->u.ospf.metric1 && - new->u.ospf.metric2 == old->u.ospf.metric2 && - new->u.ospf.tag == old->u.ospf.tag && - new->u.ospf.router_id == old->u.ospf.router_id; -} + if (rt->attrs->source == RTS_OSPF_EXT2) + return IGP_METRIC_UNKNOWN; + return ea_get_int(rt->attrs->eattrs, EA_OSPF_METRIC1, LSINFINITY); +} void ospf_schedule_rtcalc(struct ospf_proto *p) @@ -484,14 +482,13 @@ ospf_disp(timer * timer) * import to the filters. */ static int -ospf_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +ospf_preexport(struct channel *c, rte *e) { - struct ospf_proto *p = (struct ospf_proto *) P; + struct ospf_proto *p = (struct ospf_proto *) c->proto; struct ospf_area *oa = ospf_main_area(p); - rte *e = *new; /* Reject our own routes */ - if (e->attrs->src->proto == P) + if (e->sender == c->in_req.hook) return -1; /* Do not export routes to stub areas */ @@ -501,26 +498,6 @@ ospf_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) return 0; } -static void -ospf_make_tmp_attrs(struct rte *rt, struct linpool *pool) -{ - rte_init_tmp_attrs(rt, pool, 4); - rte_make_tmp_attr(rt, EA_OSPF_METRIC1, EAF_TYPE_INT, rt->u.ospf.metric1); - rte_make_tmp_attr(rt, EA_OSPF_METRIC2, EAF_TYPE_INT, rt->u.ospf.metric2); - rte_make_tmp_attr(rt, EA_OSPF_TAG, EAF_TYPE_INT, rt->u.ospf.tag); - rte_make_tmp_attr(rt, EA_OSPF_ROUTER_ID, EAF_TYPE_ROUTER_ID, rt->u.ospf.router_id); -} - -static void -ospf_store_tmp_attrs(struct rte *rt, struct linpool *pool) -{ - rte_init_tmp_attrs(rt, pool, 4); - rt->u.ospf.metric1 = rte_store_tmp_attr(rt, EA_OSPF_METRIC1); - rt->u.ospf.metric2 = rte_store_tmp_attr(rt, EA_OSPF_METRIC2); - rt->u.ospf.tag = rte_store_tmp_attr(rt, EA_OSPF_TAG); - rt->u.ospf.router_id = rte_store_tmp_attr(rt, EA_OSPF_ROUTER_ID); -} - /** * ospf_shutdown - Finish of OSPF instance * @P: OSPF protocol instance @@ -558,6 +535,9 @@ ospf_shutdown(struct proto *P) } FIB_WALK_END; + if (tm_active(p->disp_timer)) + tm_stop(p->disp_timer); + return PS_DOWN; } @@ -607,16 +587,20 @@ ospf_get_route_info(rte * rte, byte * buf) } buf += bsprintf(buf, " %s", type); - buf += bsprintf(buf, " (%d/%d", rte->pref, rte->u.ospf.metric1); + buf += bsprintf(buf, " (%d/%d", rte->attrs->pref, ea_get_int(rte->attrs->eattrs, EA_OSPF_METRIC1, LSINFINITY)); if (rte->attrs->source == RTS_OSPF_EXT2) - buf += bsprintf(buf, "/%d", rte->u.ospf.metric2); + buf += bsprintf(buf, "/%d", ea_get_int(rte->attrs->eattrs, EA_OSPF_METRIC2, LSINFINITY)); buf += bsprintf(buf, ")"); - if ((rte->attrs->source == RTS_OSPF_EXT1 || rte->attrs->source == RTS_OSPF_EXT2) && rte->u.ospf.tag) + if (rte->attrs->source == RTS_OSPF_EXT1 || rte->attrs->source == RTS_OSPF_EXT2) { - buf += bsprintf(buf, " [%x]", rte->u.ospf.tag); + eattr *ea = ea_find(rte->attrs->eattrs, EA_OSPF_TAG); + if (ea && (ea->u.data > 0)) + buf += bsprintf(buf, " [%x]", ea->u.data); } - if (rte->u.ospf.router_id) - buf += bsprintf(buf, " [%R]", rte->u.ospf.router_id); + + eattr *ea = ea_find(rte->attrs->eattrs, EA_OSPF_ROUTER_ID); + if (ea) + buf += bsprintf(buf, " [%R]", ea->u.data); } static int @@ -1533,6 +1517,12 @@ ospf_sh_lsadb(struct lsadb_show_data *ld) } +struct rte_owner_class ospf_rte_owner_class = { + .get_route_info = ospf_get_route_info, + .rte_better = ospf_rte_better, + .rte_igp_metric = ospf_rte_igp_metric, +}; + struct protocol proto_ospf = { .name = "OSPF", .template = "ospf%d", @@ -1548,5 +1538,4 @@ struct protocol proto_ospf = { .reconfigure = ospf_reconfigure, .get_status = ospf_get_status, .get_attr = ospf_get_attr, - .get_route_info = ospf_get_route_info }; diff --git a/proto/ospf/ospf.h b/proto/ospf/ospf.h index 3e704ae8..a5f83e79 100644 --- a/proto/ospf/ospf.h +++ b/proto/ospf/ospf.h @@ -1007,6 +1007,8 @@ void ospf_sh_state(struct proto *P, int verbose, int reachable); void ospf_sh_lsadb(struct lsadb_show_data *ld); +extern struct rte_owner_class ospf_rte_owner_class; + /* iface.c */ void ospf_iface_chstate(struct ospf_iface *ifa, u8 state); void ospf_iface_sm(struct ospf_iface *ifa, int event); diff --git a/proto/ospf/rt.c b/proto/ospf/rt.c index faee49dc..3e208023 100644 --- a/proto/ospf/rt.c +++ b/proto/ospf/rt.c @@ -144,7 +144,7 @@ orta_compare(const struct ospf_proto *p, const orta *new, const orta *old) { int r; - if (old->type == RTS_DUMMY) + if (!old->type) return 1; /* Prefer intra-area to inter-area to externals */ @@ -195,7 +195,7 @@ orta_compare_asbr(const struct ospf_proto *p, const orta *new, const orta *old) { int r; - if (old->type == RTS_DUMMY) + if (!old->type) return 1; if (!p->rfc1583) @@ -225,7 +225,7 @@ orta_compare_ext(const struct ospf_proto *p, const orta *new, const orta *old) { int r; - if (old->type == RTS_DUMMY) + if (!old->type) return 1; /* 16.4 (6a) - prefer routes with lower type */ @@ -2053,36 +2053,61 @@ again1: if (nf->n.type) /* Add the route */ { rta a0 = { - .src = p->p.main_source, .source = nf->n.type, .scope = SCOPE_UNIVERSE, .dest = RTD_UNICAST, .nh = *(nf->n.nhs), + .pref = p->p.main_channel->preference, }; if (reload || ort_changed(nf, &a0)) { - rta *a = rta_lookup(&a0); - rte *e = rte_get_temp(a); + a0.eattrs = alloca(sizeof(ea_list) + 4 * sizeof(eattr)); + memset(a0.eattrs, 0, sizeof(ea_list)); - rta_free(nf->old_rta); - nf->old_rta = rta_clone(a); - e->u.ospf.metric1 = nf->old_metric1 = nf->n.metric1; - e->u.ospf.metric2 = nf->old_metric2 = nf->n.metric2; - e->u.ospf.tag = nf->old_tag = nf->n.tag; - e->u.ospf.router_id = nf->old_rid = nf->n.rid; - e->pflags = EA_ID_FLAG(EA_OSPF_METRIC1) | EA_ID_FLAG(EA_OSPF_ROUTER_ID); + nf->old_metric1 = nf->n.metric1; + nf->old_metric2 = nf->n.metric2; + nf->old_tag = nf->n.tag; + nf->old_rid = nf->n.rid; + + a0.eattrs->attrs[a0.eattrs->count++] = (eattr) { + .id = EA_OSPF_METRIC1, + .type = EAF_TYPE_INT, + .u.data = nf->n.metric1, + }; if (nf->n.type == RTS_OSPF_EXT2) - e->pflags |= EA_ID_FLAG(EA_OSPF_METRIC2); + a0.eattrs->attrs[a0.eattrs->count++] = (eattr) { + .id = EA_OSPF_METRIC2, + .type = EAF_TYPE_INT, + .u.data = nf->n.metric2, + }; - /* Perhaps onfly if tag is non-zero? */ if ((nf->n.type == RTS_OSPF_EXT1) || (nf->n.type == RTS_OSPF_EXT2)) - e->pflags |= EA_ID_FLAG(EA_OSPF_TAG); + a0.eattrs->attrs[a0.eattrs->count++] = (eattr) { + .id = EA_OSPF_TAG, + .type = EAF_TYPE_INT, + .u.data = nf->n.tag, + }; + + a0.eattrs->attrs[a0.eattrs->count++] = (eattr) { + .id = EA_OSPF_ROUTER_ID, + .type = EAF_TYPE_ROUTER_ID, + .u.data = nf->n.rid, + }; + + rta_free(nf->old_rta); + nf->old_rta = rta_lookup(&a0); + + rte e0 = { + .attrs = nf->old_rta, + .src = p->p.main_source, + }; DBG("Mod rte type %d - %N via %I on iface %s, met %d\n", a0.source, nf->fn.addr, a0.gw, a0.iface ? a0.iface->name : "(none)", nf->n.metric1); - rte_update(&p->p, nf->fn.addr, e); + + rte_update(p->p.main_channel, nf->fn.addr, &e0, p->p.main_source); } } else if (nf->old_rta) @@ -2091,7 +2116,7 @@ again1: rta_free(nf->old_rta); nf->old_rta = NULL; - rte_update(&p->p, nf->fn.addr, NULL); + rte_update(p->p.main_channel, nf->fn.addr, NULL, p->p.main_source); } /* Remove unused rt entry, some special entries are persistent */ @@ -2107,7 +2132,6 @@ again1: } FIB_ITERATE_END; - WALK_LIST(oa, p->area_list) { /* Cleanup ASBR hash tables */ diff --git a/proto/ospf/topology.c b/proto/ospf/topology.c index 52c2a0ce..bb88d20a 100644 --- a/proto/ospf/topology.c +++ b/proto/ospf/topology.c @@ -1300,7 +1300,7 @@ find_surrogate_fwaddr(struct ospf_proto *p, struct ospf_area *oa) } void -ospf_rt_notify(struct proto *P, struct channel *ch UNUSED, net *n, rte *new, rte *old UNUSED) +ospf_rt_notify(struct proto *P, struct channel *ch UNUSED, const net_addr *n, rte *new, const rte *old UNUSED) { struct ospf_proto *p = (struct ospf_proto *) P; struct ospf_area *oa = NULL; /* non-NULL for NSSA-LSA */ @@ -1319,7 +1319,7 @@ ospf_rt_notify(struct proto *P, struct channel *ch UNUSED, net *n, rte *new, rte if (!new) { - nf = fib_find(&p->rtf, n->n.addr); + nf = fib_find(&p->rtf, n); if (!nf || !nf->external_rte) return; @@ -1346,14 +1346,14 @@ ospf_rt_notify(struct proto *P, struct channel *ch UNUSED, net *n, rte *new, rte if (m1 > LSINFINITY) { log(L_WARN "%s: Invalid ospf_metric1 value %u for route %N", - p->p.name, m1, n->n.addr); + p->p.name, m1, n); m1 = LSINFINITY; } if (m2 > LSINFINITY) { log(L_WARN "%s: Invalid ospf_metric2 value %u for route %N", - p->p.name, m2, n->n.addr); + p->p.name, m2, n); m2 = LSINFINITY; } @@ -1377,12 +1377,12 @@ ospf_rt_notify(struct proto *P, struct channel *ch UNUSED, net *n, rte *new, rte if (ipa_zero(fwd)) { log(L_ERR "%s: Cannot find forwarding address for NSSA-LSA %N", - p->p.name, n->n.addr); + p->p.name, n); return; } } - nf = fib_get(&p->rtf, n->n.addr); + nf = fib_get(&p->rtf, n); ospf_originate_ext_lsa(p, oa, nf, LSA_M_EXPORT, metric, ebit, fwd, tag, 1, p->vpn_pe); nf->external_rte = 1; } diff --git a/proto/ospf/topology.h b/proto/ospf/topology.h index 535d1f1b..c36d0b50 100644 --- a/proto/ospf/topology.h +++ b/proto/ospf/topology.h @@ -200,7 +200,7 @@ void ospf_originate_sum_rt_lsa(struct ospf_proto *p, struct ospf_area *oa, u32 d void ospf_originate_ext_lsa(struct ospf_proto *p, struct ospf_area *oa, ort *nf, u8 mode, u32 metric, u32 ebit, ip_addr fwaddr, u32 tag, int pbit, int dn); void ospf_originate_gr_lsa(struct ospf_proto *p, struct ospf_iface *ifa); -void ospf_rt_notify(struct proto *P, struct channel *ch, net *n, rte *new, rte *old); +void ospf_rt_notify(struct proto *P, struct channel *ch, const net_addr *n, rte *new, const rte *old); void ospf_update_topology(struct ospf_proto *p); struct top_hash_entry *ospf_hash_find(struct top_graph *, u32 domain, u32 lsa, u32 rtr, u32 type); diff --git a/proto/perf/perf.c b/proto/perf/perf.c index ba401a8a..aa688d88 100644 --- a/proto/perf/perf.c +++ b/proto/perf/perf.c @@ -143,10 +143,10 @@ perf_loop(void *data) if (!p->attrs_per_rte || !(i % p->attrs_per_rte)) { struct rta a0 = { - .src = p->p.main_source, .source = RTS_PERF, .scope = SCOPE_UNIVERSE, .dest = RTD_UNICAST, + .pref = p->p.main_channel->preference, .nh.iface = p->ifa->iface, .nh.gw = gw, .nh.weight = 1, @@ -160,18 +160,17 @@ perf_loop(void *data) clock_gettime(CLOCK_MONOTONIC, &ts_generated); - for (uint i=0; i<N; i++) { - rte *e = rte_get_temp(p->data[i].a); - e->pflags = 0; - - rte_update(P, &(p->data[i].net), e); + for (uint i=0; i<N; i++) + { + rte e0 = { .attrs = p->data[i].a, .src = P->main_source, }; + rte_update(P->main_channel, &(p->data[i].net), &e0, P->main_source); } clock_gettime(CLOCK_MONOTONIC, &ts_update); if (!p->keep) for (uint i=0; i<N; i++) - rte_update(P, &(p->data[i].net), NULL); + rte_update(P->main_channel, &(p->data[i].net), NULL, P->main_source); clock_gettime(CLOCK_MONOTONIC, &ts_withdraw); @@ -199,12 +198,14 @@ perf_loop(void *data) p->exp++; } - rt_schedule_prune(P->main_channel->table); + RT_LOCK(P->main_channel->table); + rt_schedule_prune(RT_PRIV(P->main_channel->table)); + RT_UNLOCK(P->main_channel->table); ev_schedule(p->loop); } static void -perf_rt_notify(struct proto *P, struct channel *c UNUSED, struct network *net UNUSED, struct rte *new UNUSED, struct rte *old UNUSED) +perf_rt_notify(struct proto *P, struct channel *c UNUSED, const net_addr *net UNUSED, struct rte *new UNUSED, const struct rte *old UNUSED) { struct perf_proto *p = (struct perf_proto *) P; p->exp++; diff --git a/proto/pipe/config.Y b/proto/pipe/config.Y index 1202c169..fc08445f 100644 --- a/proto/pipe/config.Y +++ b/proto/pipe/config.Y @@ -16,7 +16,7 @@ CF_DEFINES CF_DECLS -CF_KEYWORDS(PIPE, PEER, TABLE) +CF_KEYWORDS(PIPE, PEER, TABLE, MAX, GENERATION) CF_GRAMMAR @@ -25,6 +25,8 @@ proto: pipe_proto '}' { this_channel = NULL; } ; pipe_proto_start: proto_start PIPE { this_proto = proto_config_new(&proto_pipe, $1); + this_proto->loop_order = DOMAIN_ORDER(proto); + PIPE_CFG->max_generation = 16; } proto_name { @@ -41,6 +43,10 @@ pipe_proto: | pipe_proto proto_item ';' | pipe_proto channel_item_ ';' | pipe_proto PEER TABLE rtable ';' { PIPE_CFG->peer = $4; } + | pipe_proto MAX GENERATION expr ';' { + if (($4 < 1) || ($4 > 254)) cf_error("Max generation must be in range 1..254, got %u", $4); + PIPE_CFG->max_generation = $4; + } ; CF_CODE diff --git a/proto/pipe/pipe.c b/proto/pipe/pipe.c index 3532f114..270f7b92 100644 --- a/proto/pipe/pipe.c +++ b/proto/pipe/pipe.c @@ -43,67 +43,58 @@ #include "pipe.h" +#ifdef CONFIG_BGP +#include "proto/bgp/bgp.h" +#endif + static void -pipe_rt_notify(struct proto *P, struct channel *src_ch, net *n, rte *new, rte *old) +pipe_rt_notify(struct proto *P, struct channel *src_ch, const net_addr *n, rte *new, const rte *old) { struct pipe_proto *p = (void *) P; struct channel *dst = (src_ch == p->pri) ? p->sec : p->pri; - struct rte_src *src; - - rte *e; - rta *a; if (!new && !old) return; - if (dst->table->pipe_busy) - { - log(L_ERR "Pipe loop detected when sending %N to table %s", - n->n.addr, dst->table->name); - return; - } - if (new) { - a = alloca(rta_size(new->attrs)); + rta *a = alloca(rta_size(new->attrs)); memcpy(a, new->attrs, rta_size(new->attrs)); - a->aflags = 0; + a->cached = 0; a->hostentry = NULL; - e = rte_get_temp(a); - e->pflags = 0; - - /* Copy protocol specific embedded attributes. */ - memcpy(&(e->u), &(new->u), sizeof(e->u)); - e->pref = new->pref; - e->pflags = new->pflags; -#ifdef CONFIG_BGP - /* Hack to cleanup cached value */ - if (e->attrs->src->proto->proto == &proto_bgp) - e->u.bgp.stale = -1; -#endif + rte e0 = { + .attrs = a, + .src = new->src, + .generation = new->generation + 1, + }; - src = a->src; + rte_update(dst, n, &e0, new->src); } else - { - e = NULL; - src = old->attrs->src; - } - - src_ch->table->pipe_busy = 1; - rte_update2(dst, n->n.addr, e, src); - src_ch->table->pipe_busy = 0; + rte_update(dst, n, NULL, old->src); } static int -pipe_preexport(struct proto *P, rte **ee, struct linpool *p UNUSED) +pipe_preexport(struct channel *c, rte *e) { - struct proto *pp = (*ee)->sender->proto; + struct pipe_proto *p = (void *) c->proto; + + /* Avoid direct loopbacks */ + if (e->sender == c->in_req.hook) + return -1; - if (pp == P) - return -1; /* Avoid local loops automatically */ + /* Indirection check */ + uint max_generation = ((struct pipe_config *) p->p.cf)->max_generation; + if (e->generation >= max_generation) + { + log_rl(&p->rl_gen, L_ERR "Route overpiped (%u hops of %u configured in %s) in table %s: %N %s/%u:%u", + e->generation, max_generation, c->proto->name, + c->table->name, e->net, e->src->owner->name, e->src->private_id, e->src->global_id); + + return -1; + } return 0; } @@ -117,6 +108,23 @@ pipe_reload_routes(struct channel *C) channel_request_feeding((C == p->pri) ? p->sec : p->pri); } +static void +pipe_feed_begin(struct channel *C, int refeeding UNUSED) +{ + struct pipe_proto *p = (void *) C->proto; + struct channel *dst = (C == p->pri) ? p->sec : p->pri; + + channel_refresh_begin(dst); +} + +static void +pipe_feed_end(struct channel *C) +{ + struct pipe_proto *p = (void *) C->proto; + struct channel *dst = (C == p->pri) ? p->sec : p->pri; + + channel_refresh_end(dst); +} static void pipe_postconfig(struct proto_config *CF) @@ -187,6 +195,10 @@ pipe_init(struct proto_config *CF) P->rt_notify = pipe_rt_notify; P->preexport = pipe_preexport; P->reload_routes = pipe_reload_routes; + P->feed_begin = pipe_feed_begin; + P->feed_end = pipe_feed_end; + + p->rl_gen = (struct tbf) TBF_DEFAULT_LOG_LIMITS; pipe_configure_channels(p, cf); @@ -219,8 +231,18 @@ pipe_get_status(struct proto *P, byte *buf) static void pipe_show_stats(struct pipe_proto *p) { - struct proto_stats *s1 = &p->pri->stats; - struct proto_stats *s2 = &p->sec->stats; + struct channel_import_stats *s1i = &p->pri->import_stats; + struct channel_export_stats *s1e = &p->pri->export_stats; + struct channel_import_stats *s2i = &p->sec->import_stats; + struct channel_export_stats *s2e = &p->sec->export_stats; + + struct rt_import_stats *rs1i = p->pri->in_req.hook ? &p->pri->in_req.hook->stats : NULL; + struct rt_export_stats *rs1e = p->pri->out_req.hook ? &p->pri->out_req.hook->stats : NULL; + struct rt_import_stats *rs2i = p->sec->in_req.hook ? &p->sec->in_req.hook->stats : NULL; + struct rt_export_stats *rs2e = p->sec->out_req.hook ? &p->sec->out_req.hook->stats : NULL; + + u32 pri_routes = p->pri->in_limit.count; + u32 sec_routes = p->sec->in_limit.count; /* * Pipe stats (as anything related to pipes) are a bit tricky. There @@ -244,24 +266,22 @@ pipe_show_stats(struct pipe_proto *p) */ cli_msg(-1006, " Routes: %u imported, %u exported", - s1->imp_routes, s2->imp_routes); + pri_routes, sec_routes); cli_msg(-1006, " Route change stats: received rejected filtered ignored accepted"); cli_msg(-1006, " Import updates: %10u %10u %10u %10u %10u", - s2->exp_updates_received, s2->exp_updates_rejected + s1->imp_updates_invalid, - s2->exp_updates_filtered, s1->imp_updates_ignored, s1->imp_updates_accepted); + rs2e->updates_received, s2e->updates_rejected + s1i->updates_invalid, + s2e->updates_filtered, rs1i->updates_ignored, rs1i->updates_accepted); cli_msg(-1006, " Import withdraws: %10u %10u --- %10u %10u", - s2->exp_withdraws_received, s1->imp_withdraws_invalid, - s1->imp_withdraws_ignored, s1->imp_withdraws_accepted); + rs2e->withdraws_received, s1i->withdraws_invalid, + rs1i->withdraws_ignored, rs1i->withdraws_accepted); cli_msg(-1006, " Export updates: %10u %10u %10u %10u %10u", - s1->exp_updates_received, s1->exp_updates_rejected + s2->imp_updates_invalid, - s1->exp_updates_filtered, s2->imp_updates_ignored, s2->imp_updates_accepted); + rs1e->updates_received, s1e->updates_rejected + s2i->updates_invalid, + s1e->updates_filtered, rs2i->updates_ignored, rs2i->updates_accepted); cli_msg(-1006, " Export withdraws: %10u %10u --- %10u %10u", - s1->exp_withdraws_received, s2->imp_withdraws_invalid, - s2->imp_withdraws_ignored, s2->imp_withdraws_accepted); + rs1e->withdraws_received, s2i->withdraws_invalid, + rs2i->withdraws_ignored, rs2i->withdraws_accepted); } -static const char *pipe_feed_state[] = { [ES_DOWN] = "down", [ES_FEEDING] = "feed", [ES_READY] = "up" }; - static void pipe_show_proto_info(struct proto *P) { @@ -270,13 +290,17 @@ pipe_show_proto_info(struct proto *P) cli_msg(-1006, " Channel %s", "main"); cli_msg(-1006, " Table: %s", p->pri->table->name); cli_msg(-1006, " Peer table: %s", p->sec->table->name); - cli_msg(-1006, " Import state: %s", pipe_feed_state[p->sec->export_state]); - cli_msg(-1006, " Export state: %s", pipe_feed_state[p->pri->export_state]); + cli_msg(-1006, " Import state: %s", rt_export_state_name(rt_export_get_state(p->sec->out_req.hook))); + cli_msg(-1006, " Export state: %s", rt_export_state_name(rt_export_get_state(p->pri->out_req.hook))); cli_msg(-1006, " Import filter: %s", filter_name(p->sec->out_filter)); cli_msg(-1006, " Export filter: %s", filter_name(p->pri->out_filter)); - channel_show_limit(&p->pri->in_limit, "Import limit:"); - channel_show_limit(&p->sec->in_limit, "Export limit:"); + + + channel_show_limit(&p->pri->in_limit, "Import limit:", + (p->pri->limit_active & (1 << PLD_IN)), p->pri->limit_actions[PLD_IN]); + channel_show_limit(&p->sec->in_limit, "Export limit:", + (p->sec->limit_active & (1 << PLD_IN)), p->sec->limit_actions[PLD_IN]); if (P->proto_state != PS_DOWN) pipe_show_stats(p); diff --git a/proto/pipe/pipe.h b/proto/pipe/pipe.h index 038c6666..60c857eb 100644 --- a/proto/pipe/pipe.h +++ b/proto/pipe/pipe.h @@ -12,12 +12,14 @@ struct pipe_config { struct proto_config c; struct rtable_config *peer; /* Table we're connected to */ + u8 max_generation; }; struct pipe_proto { struct proto p; struct channel *pri; struct channel *sec; + struct tbf rl_gen; }; #endif diff --git a/proto/radv/radv.c b/proto/radv/radv.c index 66e8eb4b..15673555 100644 --- a/proto/radv/radv.c +++ b/proto/radv/radv.c @@ -284,7 +284,7 @@ radv_iface_new(struct radv_proto *p, struct iface *iface, struct radv_iface_conf RADV_TRACE(D_EVENTS, "Adding interface %s", iface->name); - pool *pool = rp_new(p->p.pool, iface->name); + pool *pool = rp_new(p->p.pool, p->p.loop, iface->name); ifa = mb_allocz(pool, sizeof(struct radv_iface)); ifa->pool = pool; ifa->ra = p; @@ -317,7 +317,7 @@ radv_iface_remove(struct radv_iface *ifa) rem_node(NODE ifa); - rfree(ifa->pool); + rp_free(ifa->pool, p->p.pool); } static void @@ -385,18 +385,18 @@ radv_trigger_valid(struct radv_config *cf) } static inline int -radv_net_match_trigger(struct radv_config *cf, net *n) +radv_net_match_trigger(struct radv_config *cf, const net_addr *n) { - return radv_trigger_valid(cf) && net_equal(n->n.addr, &cf->trigger); + return radv_trigger_valid(cf) && net_equal(n, &cf->trigger); } int -radv_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +radv_preexport(struct channel *c, rte *new) { // struct radv_proto *p = (struct radv_proto *) P; - struct radv_config *cf = (struct radv_config *) (P->cf); + struct radv_config *cf = (struct radv_config *) (c->proto->cf); - if (radv_net_match_trigger(cf, (*new)->net)) + if (radv_net_match_trigger(cf, new->net)) return RIC_PROCESS; if (cf->propagate_routes) @@ -406,7 +406,7 @@ radv_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) } static void -radv_rt_notify(struct proto *P, struct channel *ch UNUSED, net *n, rte *new, rte *old UNUSED) +radv_rt_notify(struct proto *P, struct channel *ch UNUSED, const net_addr *n, rte *new, const rte *old UNUSED) { struct radv_proto *p = (struct radv_proto *) P; struct radv_config *cf = (struct radv_config *) (P->cf); @@ -457,14 +457,14 @@ radv_rt_notify(struct proto *P, struct channel *ch UNUSED, net *n, rte *new, rte (preference != RA_PREF_HIGH)) { log(L_WARN "%s: Invalid ra_preference value %u on route %N", - p->p.name, preference, n->n.addr); + p->p.name, preference, n); preference = RA_PREF_MEDIUM; preference_set = 1; lifetime = 0; lifetime_set = 1; } - rt = fib_get(&p->routes, n->n.addr); + rt = fib_get(&p->routes, n); /* Ignore update if nothing changed */ if (rt->valid && @@ -487,7 +487,7 @@ radv_rt_notify(struct proto *P, struct channel *ch UNUSED, net *n, rte *new, rte else { /* Withdraw */ - rt = fib_find(&p->routes, n->n.addr); + rt = fib_find(&p->routes, n); if (!rt || !rt->valid) return; @@ -555,7 +555,10 @@ radv_check_active(struct radv_proto *p) return 1; struct channel *c = p->p.main_channel; - return rt_examine(c->table, &cf->trigger, &p->p, c->out_filter); + RT_LOCK(c->table); + int active = rt_examine(RT_PRIV(c->table), &cf->trigger, c, c->out_filter); + RT_UNLOCK(c->table); + return active; } static void @@ -660,8 +663,9 @@ radv_reconfigure(struct proto *P, struct proto_config *CF) if (!old->propagate_routes && new->propagate_routes) channel_request_feeding(p->p.main_channel); + IFACE_LEGACY_ACCESS; struct iface *iface; - WALK_LIST(iface, iface_list) + WALK_LIST(iface, global_iface_list) { if (!(iface->flags & IF_UP)) continue; diff --git a/proto/rip/rip.c b/proto/rip/rip.c index e1a235a0..0f8b10ad 100644 --- a/proto/rip/rip.c +++ b/proto/rip/rip.c @@ -145,7 +145,7 @@ rip_announce_rte(struct rip_proto *p, struct rip_entry *en) { /* Update */ rta a0 = { - .src = p->p.main_source, + .pref = p->p.main_channel->preference, .source = RTS_RIP, .scope = SCOPE_UNIVERSE, .dest = RTD_UNICAST, @@ -188,21 +188,34 @@ rip_announce_rte(struct rip_proto *p, struct rip_entry *en) a0.nh.iface = rt->from->ifa->iface; } - rta *a = rta_lookup(&a0); - rte *e = rte_get_temp(a); + a0.eattrs = alloca(sizeof(ea_list) + 3*sizeof(eattr)); + memset(a0.eattrs, 0, sizeof(ea_list)); /* Zero-ing only the ea_list header */ + a0.eattrs->count = 3; + a0.eattrs->attrs[0] = (eattr) { + .id = EA_RIP_METRIC, + .type = EAF_TYPE_INT, + .u.data = rt_metric, + }; + a0.eattrs->attrs[1] = (eattr) { + .id = EA_RIP_TAG, + .type = EAF_TYPE_INT, + .u.data = rt_tag, + }; + a0.eattrs->attrs[2] = (eattr) { + .id = EA_RIP_FROM, + .type = EAF_TYPE_PTR, + .u.data = (uintptr_t) a0.nh.iface, + }; - e->u.rip.from = a0.nh.iface; - e->u.rip.metric = rt_metric; - e->u.rip.tag = rt_tag; - e->pflags = EA_ID_FLAG(EA_RIP_METRIC) | EA_ID_FLAG(EA_RIP_TAG); + rte e0 = { + .attrs = &a0, + .src = p->p.main_source, + }; - rte_update(&p->p, en->n.addr, e); + rte_update(p->p.main_channel, en->n.addr, &e0, p->p.main_source); } else - { - /* Withdraw */ - rte_update(&p->p, en->n.addr, NULL); - } + rte_update(p->p.main_channel, en->n.addr, NULL, p->p.main_source); } /** @@ -297,8 +310,8 @@ rip_withdraw_rte(struct rip_proto *p, net_addr *n, struct rip_neighbor *from) * it into our data structures. */ static void -rip_rt_notify(struct proto *P, struct channel *ch UNUSED, struct network *net, struct rte *new, - struct rte *old UNUSED) +rip_rt_notify(struct proto *P, struct channel *ch UNUSED, const net_addr *net, struct rte *new, + const struct rte *old UNUSED) { struct rip_proto *p = (struct rip_proto *) P; struct rip_entry *en; @@ -307,20 +320,21 @@ rip_rt_notify(struct proto *P, struct channel *ch UNUSED, struct network *net, s if (new) { /* Update */ - u32 rt_metric = ea_get_int(new->attrs->eattrs, EA_RIP_METRIC, 1); u32 rt_tag = ea_get_int(new->attrs->eattrs, EA_RIP_TAG, 0); + u32 rt_metric = ea_get_int(new->attrs->eattrs, EA_RIP_METRIC, 1); + struct iface *rt_from = (struct iface *) ea_get_int(new->attrs->eattrs, EA_RIP_FROM, 0); if (rt_metric > p->infinity) { log(L_WARN "%s: Invalid rip_metric value %u for route %N", - p->p.name, rt_metric, net->n.addr); + p->p.name, rt_metric, net); rt_metric = p->infinity; } if (rt_tag > 0xffff) { log(L_WARN "%s: Invalid rip_tag value %u for route %N", - p->p.name, rt_tag, net->n.addr); + p->p.name, rt_tag, net); rt_metric = p->infinity; rt_tag = 0; } @@ -332,21 +346,21 @@ rip_rt_notify(struct proto *P, struct channel *ch UNUSED, struct network *net, s * collection. */ - en = fib_get(&p->rtable, net->n.addr); + en = fib_get(&p->rtable, net); old_metric = en->valid ? en->metric : -1; en->valid = RIP_ENTRY_VALID; en->metric = rt_metric; en->tag = rt_tag; - en->from = (new->attrs->src->proto == P) ? new->u.rip.from : NULL; + en->from = (new->src->owner == &P->sources) ? rt_from : NULL; en->iface = new->attrs->nh.iface; en->next_hop = new->attrs->nh.gw; } else { /* Withdraw */ - en = fib_find(&p->rtable, net->n.addr); + en = fib_find(&p->rtable, net); if (!en || en->valid != RIP_ENTRY_VALID) return; @@ -499,7 +513,7 @@ rip_update_bfd(struct rip_proto *p, struct rip_neighbor *n) ip_addr saddr = rip_is_v2(p) ? n->ifa->sk->saddr : n->nbr->ifa->ip; n->bfd_req = bfd_request_session(p->p.pool, n->nbr->addr, saddr, n->nbr->iface, p->p.vrf, - rip_bfd_notify, n, NULL); + rip_bfd_notify, n, birdloop_event_list(p->p.loop), NULL); } if (!use_bfd && n->bfd_req) @@ -762,7 +776,8 @@ rip_reconfigure_ifaces(struct rip_proto *p, struct rip_config *cf) { struct iface *iface; - WALK_LIST(iface, iface_list) + IFACE_LEGACY_ACCESS; + WALK_LIST(iface, global_iface_list) { if (!(iface->flags & IF_UP)) continue; @@ -1068,37 +1083,33 @@ rip_reload_routes(struct channel *C) rip_kick_timer(p); } -static void -rip_make_tmp_attrs(struct rte *rt, struct linpool *pool) -{ - rte_init_tmp_attrs(rt, pool, 2); - rte_make_tmp_attr(rt, EA_RIP_METRIC, EAF_TYPE_INT, rt->u.rip.metric); - rte_make_tmp_attr(rt, EA_RIP_TAG, EAF_TYPE_INT, rt->u.rip.tag); -} +static struct rte_owner_class rip_rte_owner_class; -static void -rip_store_tmp_attrs(struct rte *rt, struct linpool *pool) +static inline struct rip_proto * +rip_rte_proto(struct rte *rte) { - rte_init_tmp_attrs(rt, pool, 2); - rt->u.rip.metric = rte_store_tmp_attr(rt, EA_RIP_METRIC); - rt->u.rip.tag = rte_store_tmp_attr(rt, EA_RIP_TAG); + return (rte->src->owner->class == &rip_rte_owner_class) ? + SKIP_BACK(struct rip_proto, p.sources, rte->src->owner) : NULL; } static int rip_rte_better(struct rte *new, struct rte *old) { - return new->u.rip.metric < old->u.rip.metric; + ASSERT_DIE(new->src == old->src); + struct rip_proto *p = rip_rte_proto(new); + + u32 new_metric = ea_get_int(new->attrs->eattrs, EA_RIP_METRIC, p->infinity); + u32 old_metric = ea_get_int(old->attrs->eattrs, EA_RIP_METRIC, p->infinity); + + return new_metric < old_metric; } -static int -rip_rte_same(struct rte *new, struct rte *old) +static u32 +rip_rte_igp_metric(struct rte *rt) { - return ((new->u.rip.metric == old->u.rip.metric) && - (new->u.rip.tag == old->u.rip.tag) && - (new->u.rip.from == old->u.rip.from)); + return ea_get_int(rt->attrs->eattrs, EA_RIP_METRIC, IGP_METRIC_UNKNOWN); } - static void rip_postconfig(struct proto_config *CF) { @@ -1120,10 +1131,7 @@ rip_init(struct proto_config *CF) P->rt_notify = rip_rt_notify; P->neigh_notify = rip_neigh_notify; P->reload_routes = rip_reload_routes; - P->make_tmp_attrs = rip_make_tmp_attrs; - P->store_tmp_attrs = rip_store_tmp_attrs; - P->rte_better = rip_rte_better; - P->rte_same = rip_rte_same; + P->sources.class = &rip_rte_owner_class; return P; } @@ -1198,10 +1206,14 @@ rip_reconfigure(struct proto *P, struct proto_config *CF) static void rip_get_route_info(rte *rte, byte *buf) { - buf += bsprintf(buf, " (%d/%d)", rte->pref, rte->u.rip.metric); + struct rip_proto *p = rip_rte_proto(rte); + u32 rt_metric = ea_get_int(rte->attrs->eattrs, EA_RIP_METRIC, p->infinity); + u32 rt_tag = ea_get_int(rte->attrs->eattrs, EA_RIP_TAG, 0); + + buf += bsprintf(buf, " (%d/%d)", rte->attrs->pref, rt_metric); - if (rte->u.rip.tag) - bsprintf(buf, " [%04x]", rte->u.rip.tag); + if (rt_tag) + bsprintf(buf, " [%04x]", rt_tag); } static int @@ -1321,6 +1333,12 @@ rip_dump(struct proto *P) } +static struct rte_owner_class rip_rte_owner_class = { + .get_route_info = rip_get_route_info, + .rte_better = rip_rte_better, + .rte_igp_metric = rip_rte_igp_metric, +}; + struct protocol proto_rip = { .name = "RIP", .template = "rip%d", @@ -1335,6 +1353,5 @@ struct protocol proto_rip = { .start = rip_start, .shutdown = rip_shutdown, .reconfigure = rip_reconfigure, - .get_route_info = rip_get_route_info, .get_attr = rip_get_attr }; diff --git a/proto/rip/rip.h b/proto/rip/rip.h index 8d347000..f8713c4a 100644 --- a/proto/rip/rip.h +++ b/proto/rip/rip.h @@ -197,6 +197,7 @@ struct rip_rte #define EA_RIP_METRIC EA_CODE(PROTOCOL_RIP, 0) #define EA_RIP_TAG EA_CODE(PROTOCOL_RIP, 1) +#define EA_RIP_FROM EA_CODE(PROTOCOL_RIP, 2) static inline int rip_is_v2(struct rip_proto *p) { return p->rip2; } diff --git a/proto/rpki/config.Y b/proto/rpki/config.Y index d6d326b8..743b5b42 100644 --- a/proto/rpki/config.Y +++ b/proto/rpki/config.Y @@ -42,6 +42,7 @@ proto: rpki_proto ; rpki_proto_start: proto_start RPKI { this_proto = proto_config_new(&proto_rpki, $1); + this_proto->loop_order = DOMAIN_ORDER(proto); RPKI_CFG->retry_interval = RPKI_RETRY_INTERVAL; RPKI_CFG->refresh_interval = RPKI_REFRESH_INTERVAL; RPKI_CFG->expire_interval = RPKI_EXPIRE_INTERVAL; diff --git a/proto/rpki/packets.c b/proto/rpki/packets.c index d246dd50..d7895a22 100644 --- a/proto/rpki/packets.c +++ b/proto/rpki/packets.c @@ -233,7 +233,12 @@ static const size_t min_pdu_size[] = { [ERROR] = 16, }; -static int rpki_send_error_pdu(struct rpki_cache *cache, const enum pdu_error_type error_code, const u32 err_pdu_len, const struct pdu_header *erroneous_pdu, const char *fmt, ...); +static int rpki_send_error_pdu_(struct rpki_cache *cache, const enum pdu_error_type error_code, const u32 err_pdu_len, const struct pdu_header *erroneous_pdu, const char *fmt, ...); + +#define rpki_send_error_pdu(cache, error_code, err_pdu_len, erroneous_pdu, fmt...) ({ \ + rpki_send_error_pdu_(cache, error_code, err_pdu_len, erroneous_pdu, #fmt); \ + CACHE_TRACE(D_PACKETS, cache, #fmt); \ + }) static void rpki_pdu_to_network_byte_order(struct pdu_header *pdu) @@ -595,6 +600,7 @@ rpki_handle_error_pdu(struct rpki_cache *cache, const struct pdu_error *pdu) case INTERNAL_ERROR: case INVALID_REQUEST: case UNSUPPORTED_PDU_TYPE: + CACHE_TRACE(D_PACKETS, cache, "Got UNSUPPORTED_PDU_TYPE"); rpki_cache_change_state(cache, RPKI_CS_ERROR_FATAL); break; @@ -652,21 +658,7 @@ rpki_handle_cache_response_pdu(struct rpki_cache *cache, const struct pdu_cache_ { if (cache->request_session_id) { - if (cache->last_update) - { - /* - * This isn't the first sync and we already received records. This point - * is after Reset Query and before importing new records from cache - * server. We need to load new ones and kick out missing ones. So start - * a refresh cycle. - */ - if (cache->p->roa4_channel) - rt_refresh_begin(cache->p->roa4_channel->table, cache->p->roa4_channel); - if (cache->p->roa6_channel) - rt_refresh_begin(cache->p->roa6_channel->table, cache->p->roa6_channel); - - cache->p->refresh_channels = 1; - } + rpki_start_refresh(cache->p); cache->session_id = pdu->session_id; cache->request_session_id = 0; } @@ -842,14 +834,7 @@ rpki_handle_end_of_data_pdu(struct rpki_cache *cache, const struct pdu_end_of_da (cf->keep_expire_interval ? "keeps " : ""), cache->expire_interval); } - if (cache->p->refresh_channels) - { - cache->p->refresh_channels = 0; - if (cache->p->roa4_channel) - rt_refresh_end(cache->p->roa4_channel->table, cache->p->roa4_channel); - if (cache->p->roa6_channel) - rt_refresh_end(cache->p->roa6_channel->table, cache->p->roa6_channel); - } + rpki_stop_refresh(cache->p); cache->last_update = current_time(); cache->serial_num = pdu->serial_num; @@ -924,6 +909,9 @@ rpki_rx_hook(struct birdsock *sk, uint size) struct rpki_cache *cache = sk->data; struct rpki_proto *p = cache->p; + if ((p->p.proto_state == PS_DOWN) || (p->cache != cache)) + return 0; + byte *pkt_start = sk->rbuf; byte *end = pkt_start + size; @@ -980,6 +968,8 @@ rpki_err_hook(struct birdsock *sk, int error_num) CACHE_TRACE(D_EVENTS, cache, "The other side closed a connection"); } + if (cache->p->cache != cache) + return; rpki_cache_change_state(cache, RPKI_CS_ERROR_TRANSPORT); } @@ -999,6 +989,9 @@ rpki_tx_hook(sock *sk) { struct rpki_cache *cache = sk->data; + if (cache->p->cache != cache) + return; + while (rpki_fire_tx(cache) > 0) ; } @@ -1008,6 +1001,9 @@ rpki_connected_hook(sock *sk) { struct rpki_cache *cache = sk->data; + if (cache->p->cache != cache) + return; + CACHE_TRACE(D_EVENTS, cache, "Connected"); proto_notify_state(&cache->p->p, PS_UP); @@ -1029,7 +1025,7 @@ rpki_connected_hook(sock *sk) * This function prepares Error PDU and sends it to a cache server. */ static int -rpki_send_error_pdu(struct rpki_cache *cache, const enum pdu_error_type error_code, const u32 err_pdu_len, const struct pdu_header *erroneous_pdu, const char *fmt, ...) +rpki_send_error_pdu_(struct rpki_cache *cache, const enum pdu_error_type error_code, const u32 err_pdu_len, const struct pdu_header *erroneous_pdu, const char *fmt, ...) { va_list args; char msg[128]; diff --git a/proto/rpki/rpki.c b/proto/rpki/rpki.c index ab0837f3..afba2216 100644 --- a/proto/rpki/rpki.c +++ b/proto/rpki/rpki.c @@ -109,6 +109,7 @@ static void rpki_schedule_next_expire_check(struct rpki_cache *cache); static void rpki_stop_refresh_timer_event(struct rpki_cache *cache); static void rpki_stop_retry_timer_event(struct rpki_cache *cache); static void rpki_stop_expire_timer_event(struct rpki_cache *cache); +static void rpki_stop_all_timers(struct rpki_cache *cache); /* @@ -121,27 +122,48 @@ rpki_table_add_roa(struct rpki_cache *cache, struct channel *channel, const net_ struct rpki_proto *p = cache->p; rta a0 = { - .src = p->p.main_source, + .pref = channel->preference, .source = RTS_RPKI, .scope = SCOPE_UNIVERSE, .dest = RTD_NONE, }; - rta *a = rta_lookup(&a0); - rte *e = rte_get_temp(a); + rte e0 = { .attrs = &a0, .src = p->p.main_source, }; - e->pflags = 0; - - rte_update2(channel, &pfxr->n, e, a0.src); + rte_update(channel, &pfxr->n, &e0, p->p.main_source); } void rpki_table_remove_roa(struct rpki_cache *cache, struct channel *channel, const net_addr_union *pfxr) { struct rpki_proto *p = cache->p; - rte_update2(channel, &pfxr->n, NULL, p->p.main_source); + rte_update(channel, &pfxr->n, NULL, p->p.main_source); +} + +void +rpki_start_refresh(struct rpki_proto *p) +{ + if (p->roa4_channel) + rt_refresh_begin(&p->roa4_channel->in_req); + if (p->roa6_channel) + rt_refresh_begin(&p->roa6_channel->in_req); + + p->refresh_channels = 1; } +void +rpki_stop_refresh(struct rpki_proto *p) +{ + if (!p->refresh_channels) + return; + + p->refresh_channels = 0; + + if (p->roa4_channel) + rt_refresh_end(&p->roa4_channel->in_req); + if (p->roa6_channel) + rt_refresh_end(&p->roa6_channel->in_req); +} /* * RPKI Protocol Logic @@ -198,6 +220,8 @@ rpki_force_restart_proto(struct rpki_proto *p) { if (p->cache) { + rpki_tr_close(p->cache->tr_sock); + rpki_stop_all_timers(p->cache); CACHE_DBG(p->cache, "Connection object destroying"); } @@ -321,7 +345,7 @@ rpki_schedule_next_refresh(struct rpki_cache *cache) btime t = cache->refresh_interval S; CACHE_DBG(cache, "after %t s", t); - tm_start(cache->refresh_timer, t); + tm_start_in(cache->refresh_timer, t, cache->p->p.loop); } static void @@ -330,7 +354,7 @@ rpki_schedule_next_retry(struct rpki_cache *cache) btime t = cache->retry_interval S; CACHE_DBG(cache, "after %t s", t); - tm_start(cache->retry_timer, t); + tm_start_in(cache->retry_timer, t, cache->p->p.loop); } static void @@ -341,7 +365,7 @@ rpki_schedule_next_expire_check(struct rpki_cache *cache) t = MAX(t, 1 S); CACHE_DBG(cache, "after %t s", t); - tm_start(cache->expire_timer, t); + tm_start_in(cache->expire_timer, t, cache->p->p.loop); } static void @@ -358,13 +382,21 @@ rpki_stop_retry_timer_event(struct rpki_cache *cache) tm_stop(cache->retry_timer); } -static void UNUSED +static void rpki_stop_expire_timer_event(struct rpki_cache *cache) { CACHE_DBG(cache, "Stop"); tm_stop(cache->expire_timer); } +static void +rpki_stop_all_timers(struct rpki_cache *cache) +{ + rpki_stop_refresh_timer_event(cache); + rpki_stop_retry_timer_event(cache); + rpki_stop_expire_timer_event(cache); +} + static int rpki_do_we_recv_prefix_pdu_in_last_seconds(struct rpki_cache *cache) { @@ -387,6 +419,9 @@ rpki_refresh_hook(timer *tm) { struct rpki_cache *cache = tm->data; + if (cache->p->cache != cache) + return; + CACHE_DBG(cache, "%s", rpki_cache_state_to_str(cache->state)); switch (cache->state) @@ -433,6 +468,9 @@ rpki_retry_hook(timer *tm) { struct rpki_cache *cache = tm->data; + if (cache->p->cache != cache) + return; + CACHE_DBG(cache, "%s", rpki_cache_state_to_str(cache->state)); switch (cache->state) @@ -478,6 +516,9 @@ rpki_expire_hook(timer *tm) { struct rpki_cache *cache = tm->data; + if (cache->p->cache != cache) + return; + if (!cache->last_update) return; @@ -555,7 +596,7 @@ rpki_check_expire_interval(uint seconds) static struct rpki_cache * rpki_init_cache(struct rpki_proto *p, struct rpki_config *cf) { - pool *pool = rp_new(p->p.pool, cf->hostname); + pool *pool = rp_new(p->p.pool, p->p.loop, cf->hostname); struct rpki_cache *cache = mb_allocz(pool, sizeof(struct rpki_cache)); @@ -620,6 +661,7 @@ rpki_close_connection(struct rpki_cache *cache) { CACHE_TRACE(D_EVENTS, cache, "Closing a connection"); rpki_tr_close(cache->tr_sock); + rpki_stop_refresh(cache->p); proto_notify_state(&cache->p->p, PS_START); } diff --git a/proto/rpki/rpki.h b/proto/rpki/rpki.h index 8a5c38fd..a70a2027 100644 --- a/proto/rpki/rpki.h +++ b/proto/rpki/rpki.h @@ -83,6 +83,8 @@ const char *rpki_cache_state_to_str(enum rpki_cache_state state); void rpki_table_add_roa(struct rpki_cache *cache, struct channel *channel, const net_addr_union *pfxr); void rpki_table_remove_roa(struct rpki_cache *cache, struct channel *channel, const net_addr_union *pfxr); +void rpki_start_refresh(struct rpki_proto *p); +void rpki_stop_refresh(struct rpki_proto *p); /* * RPKI Protocol Logic diff --git a/proto/rpki/ssh_transport.c b/proto/rpki/ssh_transport.c index 6333f367..223afa80 100644 --- a/proto/rpki/ssh_transport.c +++ b/proto/rpki/ssh_transport.c @@ -38,6 +38,8 @@ rpki_tr_ssh_open(struct rpki_tr_sock *tr) if (sk_open(sk) != 0) return RPKI_TR_ERROR; + sk_start(sk); + return RPKI_TR_SUCCESS; } diff --git a/proto/rpki/tcp_transport.c b/proto/rpki/tcp_transport.c index 132f8e2d..4e850c44 100644 --- a/proto/rpki/tcp_transport.c +++ b/proto/rpki/tcp_transport.c @@ -31,6 +31,8 @@ rpki_tr_tcp_open(struct rpki_tr_sock *tr) if (sk_open(sk) != 0) return RPKI_TR_ERROR; + sk_start(sk); + return RPKI_TR_SUCCESS; } diff --git a/proto/rpki/transport.c b/proto/rpki/transport.c index a1ac7587..26609764 100644 --- a/proto/rpki/transport.c +++ b/proto/rpki/transport.c @@ -85,6 +85,8 @@ rpki_tr_open(struct rpki_tr_sock *tr) sk->rbsize = RPKI_RX_BUFFER_SIZE; sk->tbsize = RPKI_TX_BUFFER_SIZE; sk->tos = IP_PREC_INTERNET_CONTROL; + sk->flags |= SKF_THREAD; + sk->loop = cache->p->p.loop; if (ipa_zero(sk->daddr) && sk->host) { @@ -119,6 +121,7 @@ rpki_tr_close(struct rpki_tr_sock *tr) if (tr->sk) { + sk_stop(tr->sk); rfree(tr->sk); tr->sk = NULL; } diff --git a/proto/static/static.c b/proto/static/static.c index 2789c1bb..d89ca8b0 100644 --- a/proto/static/static.c +++ b/proto/static/static.c @@ -52,14 +52,18 @@ static linpool *static_lp; static inline struct rte_src * static_get_source(struct static_proto *p, uint i) { return i ? rt_get_source(&p->p, i) : p->p.main_source; } +static inline void static_free_source(struct rte_src *src, uint i) +{ if (i) rt_unlock_source(src); } + static void static_announce_rte(struct static_proto *p, struct static_route *r) { + struct rte_src *src; rta *a = allocz(RTA_MAX_SIZE); - a->src = static_get_source(p, r->index); a->source = RTS_STATIC; a->scope = SCOPE_UNIVERSE; a->dest = r->dest; + a->pref = p->p.main_channel->preference; if (r->dest == RTD_UNICAST) { @@ -94,7 +98,7 @@ static_announce_rte(struct static_proto *p, struct static_route *r) if (r->dest == RTDX_RECURSIVE) { rtable *tab = ipa_is_ip4(r->via) ? p->igp_table_ip4 : p->igp_table_ip6; - rta_set_recursive_next_hop(p->p.main_channel->table, a, tab, r->via, IPA_NONE, r->mls); + rta_set_recursive_next_hop(p->p.main_channel->table, a, tab, r->via, IPA_NONE, r->mls, static_lp); } /* Already announced */ @@ -102,24 +106,16 @@ static_announce_rte(struct static_proto *p, struct static_route *r) return; /* We skip rta_lookup() here */ - rte *e = rte_get_temp(a); - e->pflags = 0; + src = static_get_source(p, r->index); + rte e0 = { .attrs = a, .src = src, .net = r->net, }, *e = &e0; + /* Evaluate the filter */ if (r->cmds) - { - /* Create a temporary table node */ - e->net = alloca(sizeof(net) + r->net->length); - memset(e->net, 0, sizeof(net) + r->net->length); - net_copy(e->net->n.addr, r->net); + f_eval_rte(r->cmds, e, static_lp); - /* Evaluate the filter */ - f_eval_rte(r->cmds, &e, static_lp); + rte_update(p->p.main_channel, r->net, e, src); + static_free_source(src, r->index); - /* Remove the temporary node */ - e->net = NULL; - } - - rte_update2(p->p.main_channel, r->net, e, a->src); r->state = SRS_CLEAN; if (r->cmds) @@ -131,7 +127,9 @@ withdraw: if (r->state == SRS_DOWN) return; - rte_update2(p->p.main_channel, r->net, NULL, a->src); + src = static_get_source(p, r->index); + rte_update(p->p.main_channel, r->net, NULL, src); + static_free_source(src, r->index); r->state = SRS_DOWN; } @@ -208,7 +206,7 @@ static_update_bfd(struct static_proto *p, struct static_route *r) // ip_addr local = ipa_nonzero(r->local) ? r->local : nb->ifa->ip; r->bfd_req = bfd_request_session(p->p.pool, r->via, nb->ifa->ip, nb->iface, p->p.vrf, - static_bfd_notify, r, NULL); + static_bfd_notify, r, birdloop_event_list(p->p.loop), NULL); } if (!bfd_up && r->bfd_req) @@ -297,7 +295,11 @@ static void static_remove_rte(struct static_proto *p, struct static_route *r) { if (r->state) - rte_update2(p->p.main_channel, r->net, NULL, static_get_source(p, r->index)); + { + struct rte_src *src = static_get_source(p, r->index); + rte_update(p->p.main_channel, r->net, NULL, src); + static_free_source(src, r->index); + } static_reset_rte(p, r); } @@ -454,6 +456,8 @@ static_postconfig(struct proto_config *CF) static_index_routes(cf); } +static struct rte_owner_class static_rte_owner_class; + static struct proto * static_init(struct proto_config *CF) { @@ -465,8 +469,7 @@ static_init(struct proto_config *CF) P->neigh_notify = static_neigh_notify; P->reload_routes = static_reload_routes; - P->rte_better = static_rte_better; - P->rte_mergable = static_rte_mergable; + P->sources.class = &static_rte_owner_class; if (cf->igp_table_ip4) p->igp_table_ip4 = cf->igp_table_ip4->table; @@ -488,10 +491,12 @@ static_start(struct proto *P) static_lp = lp_new(&root_pool, LP_GOOD_SIZE(1024)); if (p->igp_table_ip4) - rt_lock_table(p->igp_table_ip4); + RT_LOCKED(p->igp_table_ip4, t) + rt_lock_table(t); if (p->igp_table_ip6) - rt_lock_table(p->igp_table_ip6); + RT_LOCKED(p->igp_table_ip6, t) + rt_lock_table(t); p->event = ev_new_init(p->p.pool, static_announce_marked, p); @@ -517,19 +522,15 @@ static_shutdown(struct proto *P) WALK_LIST(r, cf->routes) static_reset_rte(p, r); - return PS_DOWN; -} - -static void -static_cleanup(struct proto *P) -{ - struct static_proto *p = (void *) P; - if (p->igp_table_ip4) - rt_unlock_table(p->igp_table_ip4); + RT_LOCKED(p->igp_table_ip4, t) + rt_unlock_table(t); if (p->igp_table_ip6) - rt_unlock_table(p->igp_table_ip6); + RT_LOCKED(p->igp_table_ip6, t) + rt_unlock_table(t); + + return PS_DOWN; } static void @@ -721,9 +722,9 @@ static_get_route_info(rte *rte, byte *buf) { eattr *a = ea_find(rte->attrs->eattrs, EA_GEN_IGP_METRIC); if (a) - buf += bsprintf(buf, " (%d/%u)", rte->pref, a->u.data); + buf += bsprintf(buf, " (%d/%u)", rte->attrs->pref, a->u.data); else - buf += bsprintf(buf, " (%d)", rte->pref); + buf += bsprintf(buf, " (%d)", rte->attrs->pref); } static void @@ -773,6 +774,11 @@ static_show(struct proto *P) static_show_rt(r); } +static struct rte_owner_class static_rte_owner_class = { + .get_route_info = static_get_route_info, + .rte_better = static_rte_better, + .rte_mergable = static_rte_mergable, +}; struct protocol proto_static = { .name = "Static", @@ -787,8 +793,6 @@ struct protocol proto_static = { .dump = static_dump, .start = static_start, .shutdown = static_shutdown, - .cleanup = static_cleanup, .reconfigure = static_reconfigure, .copy_config = static_copy_config, - .get_route_info = static_get_route_info, }; diff --git a/sysdep/bsd/krt-sock.c b/sysdep/bsd/krt-sock.c index 521f43f3..fb407739 100644 --- a/sysdep/bsd/krt-sock.c +++ b/sysdep/bsd/krt-sock.c @@ -197,9 +197,9 @@ sockaddr_fill_dl(struct sockaddr_dl *sa, struct iface *ifa) } static int -krt_send_route(struct krt_proto *p, int cmd, rte *e) +krt_send_route(struct krt_proto *p, int cmd, const rte *e) { - net *net = e->net; + const net_addr *net = e->net; rta *a = e->attrs; static int msg_seq; struct iface *j, *i = a->nh.iface; @@ -208,7 +208,7 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) char *body = (char *)msg.buf; sockaddr gate, mask, dst; - DBG("krt-sock: send %I/%d via %I\n", net->n.prefix, net->n.pxlen, a->gw); +// DBG("krt-sock: send %I/%d via %I\n", net->prefix, net->pxlen, a->gw); bzero(&msg,sizeof (struct rt_msghdr)); msg.rtm.rtm_version = RTM_VERSION; @@ -218,7 +218,7 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) msg.rtm.rtm_flags = RTF_UP | RTF_PROTO1; /* XXXX */ - if (net_pxlen(net->n.addr) == net_max_prefix_length[net->n.addr->type]) + if (net_pxlen(net) == net_max_prefix_length[net->type]) msg.rtm.rtm_flags |= RTF_HOST; else msg.rtm.rtm_addrs |= RTA_NETMASK; @@ -242,7 +242,8 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) */ if (!i) { - WALK_LIST(j, iface_list) + IFACE_LOCK; + WALK_LIST(j, global_iface_list) { if (j->flags & IF_LOOPBACK) { @@ -250,6 +251,7 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) break; } } + IFACE_UNLOCK; if (!i) { @@ -260,7 +262,7 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) int af = AF_UNSPEC; - switch (net->n.addr->type) { + switch (net->type) { case NET_IP4: af = AF_INET; break; @@ -268,12 +270,12 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) af = AF_INET6; break; default: - log(L_ERR "KRT: Not sending route %N to kernel", net->n.addr); + log(L_ERR "KRT: Not sending route %N to kernel", net); return -1; } - sockaddr_fill(&dst, af, net_prefix(net->n.addr), NULL, 0); - sockaddr_fill(&mask, af, net_pxmask(net->n.addr), NULL, 0); + sockaddr_fill(&dst, af, net_prefix(net), NULL, 0); + sockaddr_fill(&mask, af, net_pxmask(net), NULL, 0); switch (a->dest) { @@ -303,7 +305,7 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) #if __OpenBSD__ /* Keeping temporarily old code for OpenBSD */ - struct ifa *addr = (net->n.addr->type == NET_IP4) ? i->addr4 : (i->addr6 ?: i->llv6); + struct ifa *addr = (net->type == NET_IP4) ? i->addr4 : (i->addr6 ?: i->llv6); if (!addr) { @@ -339,7 +341,7 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) msg.rtm.rtm_msglen = l; if ((l = write(p->sys.sk->fd, (char *)&msg, l)) < 0) { - log(L_ERR "KRT: Error sending route %N to kernel: %m", net->n.addr); + log(L_ERR "KRT: Error sending route %N to kernel: %m", net); return -1; } @@ -347,7 +349,7 @@ krt_send_route(struct krt_proto *p, int cmd, rte *e) } void -krt_replace_rte(struct krt_proto *p, net *n, rte *new, rte *old) +krt_replace_rte(struct krt_proto *p, const net_addr *n UNUSED, rte *new, const rte *old) { int err = 0; @@ -398,8 +400,6 @@ krt_read_route(struct ks_msg *msg, struct krt_proto *p, int scan) /* p is NULL iff KRT_SHARED_SOCKET and !scan */ int ipv6; - rte *e; - net *net; sockaddr dst, gate, mask; ip_addr idst, igate, imask; net_addr ndst; @@ -516,10 +516,7 @@ krt_read_route(struct ks_msg *msg, struct krt_proto *p, int scan) else src = KRT_SRC_KERNEL; - net = net_get(p->p.main_channel->table, &ndst); - rta a = { - .src = p->p.main_source, .source = RTS_INHERIT, .scope = SCOPE_UNIVERSE, }; @@ -545,7 +542,7 @@ krt_read_route(struct ks_msg *msg, struct krt_proto *p, int scan) if (!a.nh.iface) { log(L_ERR "KRT: Received route %N with unknown ifindex %u", - net->n.addr, msg->rtm.rtm_index); + &ndst, msg->rtm.rtm_index); return; } @@ -574,24 +571,28 @@ krt_read_route(struct ks_msg *msg, struct krt_proto *p, int scan) return; log(L_ERR "KRT: Received route %N with strange next-hop %I", - net->n.addr, a.nh.gw); + &ndst, a.nh.gw); return; } } - done: - e = rte_get_temp(&a); - e->net = net; - e->u.krt.src = src; - e->u.krt.proto = src2; - e->u.krt.seen = 0; - e->u.krt.best = 0; - e->u.krt.metric = 0; + done:; + rte e0 = { .attrs = &a, .net = &ndst, }; + + ea_list *ea = alloca(sizeof(ea_list) + 1 * sizeof(eattr)); + *ea = (ea_list) { .count = 1, .next = e0.attrs->eattrs }; + e0.attrs->eattrs = ea; + + ea->attrs[0] = (eattr) { + .id = EA_KRT_SOURCE, + .type = EAF_TYPE_INT, + .u.data = src2, + }; if (scan) - krt_got_route(p, e); + krt_got_route(p, &e0, src); else - krt_got_route_async(p, e, new); + krt_got_route_async(p, &e0, new, src); } static void diff --git a/sysdep/linux/netlink.c b/sysdep/linux/netlink.c index fdf3f2db..f52a796f 100644 --- a/sysdep/linux/netlink.c +++ b/sysdep/linux/netlink.c @@ -105,7 +105,7 @@ struct nl_parse_state int scan; int merge; - net *net; + net_addr *net; rta *attrs; struct krt_proto *proto; s8 new; @@ -1147,8 +1147,9 @@ kif_do_scan(struct kif_proto *p UNUSED) log(L_DEBUG "nl_scan_ifaces: Unknown packet received (type=%d)", h->nlmsg_type); /* Re-resolve master interface for slaves */ + IFACE_LEGACY_ACCESS; struct iface *i; - WALK_LIST(i, iface_list) + WALK_LIST(i, global_iface_list) if (i->master_index) { struct iface f = { @@ -1233,10 +1234,9 @@ nh_bufsize(struct nexthop *nh) } static int -nl_send_route(struct krt_proto *p, rte *e, int op, int dest, struct nexthop *nh) +nl_send_route(struct krt_proto *p, const rte *e, int op, int dest, struct nexthop *nh) { eattr *ea; - net *net = e->net; rta *a = e->attrs; ea_list *eattrs = a->eattrs; int bufsize = 128 + KRT_METRICS_MAX*8 + nh_bufsize(&(a->nh)); @@ -1251,7 +1251,7 @@ nl_send_route(struct krt_proto *p, rte *e, int op, int dest, struct nexthop *nh) int rsize = sizeof(*r) + bufsize; r = alloca(rsize); - DBG("nl_send_route(%N,op=%x)\n", net->n.addr, op); + DBG("nl_send_route(%N,op=%x)\n", e->net, op); bzero(&r->h, sizeof(r->h)); bzero(&r->r, sizeof(r->r)); @@ -1260,7 +1260,7 @@ nl_send_route(struct krt_proto *p, rte *e, int op, int dest, struct nexthop *nh) r->h.nlmsg_flags = op | NLM_F_REQUEST | NLM_F_ACK; r->r.rtm_family = p->af; - r->r.rtm_dst_len = net_pxlen(net->n.addr); + r->r.rtm_dst_len = net_pxlen(e->net); r->r.rtm_protocol = RTPROT_BIRD; r->r.rtm_scope = RT_SCOPE_NOWHERE; #ifdef HAVE_MPLS_KERNEL @@ -1272,7 +1272,7 @@ nl_send_route(struct krt_proto *p, rte *e, int op, int dest, struct nexthop *nh) * 2) Never use RTA_PRIORITY */ - u32 label = net_mpls(net->n.addr); + u32 label = net_mpls(e->net); nl_add_attr_mpls(&r->h, rsize, RTA_DST, 1, &label); r->r.rtm_scope = RT_SCOPE_UNIVERSE; r->r.rtm_type = RTN_UNICAST; @@ -1280,12 +1280,12 @@ nl_send_route(struct krt_proto *p, rte *e, int op, int dest, struct nexthop *nh) else #endif { - nl_add_attr_ipa(&r->h, rsize, RTA_DST, net_prefix(net->n.addr)); + nl_add_attr_ipa(&r->h, rsize, RTA_DST, net_prefix(e->net)); /* Add source address for IPv6 SADR routes */ - if (net->n.addr->type == NET_IP6_SADR) + if (e->net->type == NET_IP6_SADR) { - net_addr_ip6_sadr *a = (void *) &net->n.addr; + net_addr_ip6_sadr *a = (void *) &e->net; nl_add_attr_ip6(&r->h, rsize, RTA_SRC, a->src_prefix); r->r.rtm_src_len = a->src_pxlen; } @@ -1305,8 +1305,6 @@ nl_send_route(struct krt_proto *p, rte *e, int op, int dest, struct nexthop *nh) if (p->af == AF_MPLS) priority = 0; - else if (a->source == RTS_DUMMY) - priority = e->u.krt.metric; else if (KRT_CF->sys.metric) priority = KRT_CF->sys.metric; else if ((op != NL_OP_DELETE) && (ea = ea_find(eattrs, EA_KRT_METRIC))) @@ -1408,7 +1406,7 @@ nl_add_rte(struct krt_proto *p, rte *e) } static inline int -nl_delete_rte(struct krt_proto *p, rte *e) +nl_delete_rte(struct krt_proto *p, const rte *e) { int err = 0; @@ -1429,7 +1427,7 @@ nl_replace_rte(struct krt_proto *p, rte *e) void -krt_replace_rte(struct krt_proto *p, net *n UNUSED, rte *new, rte *old) +krt_replace_rte(struct krt_proto *p, const net_addr *n UNUSED, rte *new, const rte *old) { int err = 0; @@ -1468,7 +1466,7 @@ krt_replace_rte(struct krt_proto *p, net *n UNUSED, rte *new, rte *old) } static int -nl_mergable_route(struct nl_parse_state *s, net *net, struct krt_proto *p, uint priority, uint krt_type, uint rtm_family) +nl_mergable_route(struct nl_parse_state *s, const net_addr *net, struct krt_proto *p, uint priority, uint krt_type, uint rtm_family) { /* Route merging is used for IPv6 scans */ if (!s->scan || (rtm_family != AF_INET6)) @@ -1488,18 +1486,30 @@ nl_mergable_route(struct nl_parse_state *s, net *net, struct krt_proto *p, uint static void nl_announce_route(struct nl_parse_state *s) { - rte *e = rte_get_temp(s->attrs); - e->net = s->net; - e->u.krt.src = s->krt_src; - e->u.krt.proto = s->krt_proto; - e->u.krt.seen = 0; - e->u.krt.best = 0; - e->u.krt.metric = s->krt_metric; + rte e0 = { + .attrs = s->attrs, + .net = s->net, + }; + + ea_list *ea = alloca(sizeof(ea_list) + 2 * sizeof(eattr)); + *ea = (ea_list) { .count = 2, .next = e0.attrs->eattrs }; + e0.attrs->eattrs = ea; + + ea->attrs[0] = (eattr) { + .id = EA_KRT_SOURCE, + .type = EAF_TYPE_INT, + .u.data = s->krt_proto, + }; + ea->attrs[1] = (eattr) { + .id = EA_KRT_METRIC, + .type = EAF_TYPE_INT, + .u.data = s->krt_metric, + }; if (s->scan) - krt_got_route(s->proto, e); + krt_got_route(s->proto, &e0, s->krt_src); else - krt_got_route_async(s->proto, e, s->new); + krt_got_route_async(s->proto, &e0, s->new, s->krt_src); s->net = NULL; s->attrs = NULL; @@ -1645,21 +1655,18 @@ nl_parse_route(struct nl_parse_state *s, struct nlmsghdr *h) krt_src = KRT_SRC_ALIEN; } - net_addr *n = &dst; + net_addr *net = &dst; if (p->p.net_type == NET_IP6_SADR) { - n = alloca(sizeof(net_addr_ip6_sadr)); - net_fill_ip6_sadr(n, net6_prefix(&dst), net6_pxlen(&dst), + net = alloca(sizeof(net_addr_ip6_sadr)); + net_fill_ip6_sadr(net, net6_prefix(&dst), net6_pxlen(&dst), net6_prefix(&src), net6_pxlen(&src)); } - net *net = net_get(p->p.main_channel->table, n); - if (s->net && !nl_mergable_route(s, net, p, priority, i->rtm_type, i->rtm_family)) nl_announce_route(s); rta *ra = lp_allocz(s->pool, RTA_MAX_SIZE); - ra->src = p->p.main_source; ra->source = RTS_INHERIT; ra->scope = SCOPE_UNIVERSE; @@ -1678,7 +1685,7 @@ nl_parse_route(struct nl_parse_state *s, struct nlmsghdr *h) struct nexthop *nh = nl_parse_multipath(s, p, a[RTA_MULTIPATH], i->rtm_family); if (!nh) { - log(L_ERR "KRT: Received strange multipath route %N", net->n.addr); + log(L_ERR "KRT: Received strange multipath route %N", net); return; } @@ -1692,7 +1699,7 @@ nl_parse_route(struct nl_parse_state *s, struct nlmsghdr *h) ra->nh.iface = if_find_by_index(oif); if (!ra->nh.iface) { - log(L_ERR "KRT: Received route %N with unknown ifindex %u", net->n.addr, oif); + log(L_ERR "KRT: Received route %N with unknown ifindex %u", net, oif); return; } @@ -1719,8 +1726,7 @@ nl_parse_route(struct nl_parse_state *s, struct nlmsghdr *h) (ra->nh.flags & RNF_ONLINK) ? NEF_ONLINK : 0); if (!nbr || (nbr->scope == SCOPE_HOST)) { - log(L_ERR "KRT: Received route %N with strange next-hop %I", net->n.addr, - ra->nh.gw); + log(L_ERR "KRT: Received route %N with strange next-hop %I", net, ra->nh.gw); return; } } @@ -1819,7 +1825,7 @@ nl_parse_route(struct nl_parse_state *s, struct nlmsghdr *h) if (nl_parse_metrics(a[RTA_METRICS], metrics, ARRAY_SIZE(metrics)) < 0) { - log(L_ERR "KRT: Received route %N with strange RTA_METRICS attribute", net->n.addr); + log(L_ERR "KRT: Received route %N with strange RTA_METRICS attribute", net); return; } @@ -1853,7 +1859,9 @@ nl_parse_route(struct nl_parse_state *s, struct nlmsghdr *h) if (!s->net) { /* Store the new route */ - s->net = net; + s->net = lp_alloc(s->pool, net->length); + net_copy(s->net, net); + s->attrs = ra; s->proto = p; s->new = new; diff --git a/sysdep/unix/Makefile b/sysdep/unix/Makefile index d0d36b5f..07f454ab 100644 --- a/sysdep/unix/Makefile +++ b/sysdep/unix/Makefile @@ -1,4 +1,4 @@ -src := alloc.c io.c krt.c log.c main.c random.c +src := alloc.c io.c io-loop.c krt.c log.c main.c random.c coroutine.c obj := $(src-o-files) $(all-daemon) $(cf-local) diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index 5dd70c99..c09a8356 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -11,77 +11,73 @@ #include "lib/lists.h" #include "lib/event.h" +#include "sysdep/unix/io-loop.h" + #include <stdlib.h> #include <unistd.h> +#include <stdatomic.h> +#include <errno.h> #ifdef HAVE_MMAP #include <sys/mman.h> #endif -#ifdef HAVE_MMAP -#define KEEP_PAGES 512 +long page_size = 0; -static u64 page_size = 0; +#ifdef HAVE_MMAP +#if DEBUGGING +#define FP_NODE_OFFSET 42 +#else +#define FP_NODE_OFFSET 1 +#endif static _Bool use_fake = 0; - -uint pages_kept = 0; -static list pages_list; - -static void cleanup_pages(void *data); -static event page_cleanup_event = { .hook = cleanup_pages }; - #else -static const u64 page_size = 4096; /* Fake page size */ +static _Bool use_fake = 1; #endif -u64 get_page_size(void) +static void * +alloc_sys_page(void) { - if (page_size) - return page_size; + void *ptr = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -#ifdef HAVE_MMAP - if (page_size = sysconf(_SC_PAGESIZE)) - { - if ((u64_popcount(page_size) > 1) || (page_size > 16384)) - { - /* Too big or strange page, use the aligned allocator instead */ - page_size = 4096; - use_fake = 1; - } - return page_size; - } + if (ptr == MAP_FAILED) + bug("mmap(%lu) failed: %m", page_size); - bug("Page size must be non-zero"); -#endif + return ptr; } +extern int shutting_down; /* Shutdown requested. */ + void * alloc_page(void) { #ifdef HAVE_MMAP - if (pages_kept) - { - node *page = TAIL(pages_list); - rem_node(page); - pages_kept--; - memset(page, 0, get_page_size()); - return page; - } - if (!use_fake) { - void *ret = mmap(NULL, get_page_size(), PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ret == MAP_FAILED) - bug("mmap(%lu) failed: %m", page_size); - return ret; + struct free_pages *fp = &birdloop_current->pages; + if (!fp->cnt) + return alloc_sys_page(); + + node *n = HEAD(fp->list); + rem_node(n); + if ((--fp->cnt < fp->min) && !shutting_down) + ev_send(fp->cleanup->list, fp->cleanup); + + void *ptr = n - FP_NODE_OFFSET; + memset(ptr, 0, page_size); + return ptr; } else #endif { +#ifdef HAVE_ALIGNED_ALLOC void *ret = aligned_alloc(page_size, page_size); if (!ret) bug("aligned_alloc(%lu) failed", page_size); return ret; +#else + bug("BIRD should have already died on fatal error."); +#endif } } @@ -91,14 +87,14 @@ free_page(void *ptr) #ifdef HAVE_MMAP if (!use_fake) { - if (!pages_kept) - init_list(&pages_list); - - memset(ptr, 0, sizeof(node)); - add_tail(&pages_list, ptr); - - if (++pages_kept > KEEP_PAGES) - ev_schedule(&page_cleanup_event); + struct free_pages *fp = &birdloop_current->pages; + struct node *n = ptr; + n += FP_NODE_OFFSET; + + memset(n, 0, sizeof(node)); + add_tail(&fp->list, n); + if ((++fp->cnt > fp->max) && !shutting_down) + ev_send(fp->cleanup->list, fp->cleanup); } else #endif @@ -106,24 +102,147 @@ free_page(void *ptr) } #ifdef HAVE_MMAP + +#define GFP (&main_birdloop.pages) + +void +flush_pages(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_inside(loop->parent->loop)); + + struct free_pages *fp = &loop->pages; + struct free_pages *pfp = &loop->parent->loop->pages; + + add_tail_list(&pfp->list, &fp->list); + pfp->cnt += fp->cnt; + + fp->cnt = 0; + fp->list = (list) {}; + fp->min = 0; + fp->max = 0; + + rfree(fp->cleanup); + fp->cleanup = NULL; +} + +static void +cleanup_pages(void *data) +{ + struct birdloop *loop = data; + birdloop_enter(loop); + + ASSERT_DIE(birdloop_inside(loop->parent->loop)); + + struct free_pages *fp = &loop->pages; + struct free_pages *pfp = &loop->parent->loop->pages; + + while ((fp->cnt < fp->min) && (pfp->cnt > pfp->min)) + { + node *n = HEAD(pfp->list); + rem_node(n); + add_tail(&fp->list, n); + fp->cnt++; + pfp->cnt--; + } + + while (fp->cnt < fp->min) + { + node *n = alloc_sys_page(); + add_tail(&fp->list, n + FP_NODE_OFFSET); + fp->cnt++; + } + + while (fp->cnt > fp->max) + { + node *n = HEAD(fp->list); + rem_node(n); + add_tail(&pfp->list, n); + fp->cnt--; + pfp->cnt++; + } + + birdloop_leave(loop); + + if (!shutting_down && (pfp->cnt > pfp->max)) + ev_send(pfp->cleanup->list, pfp->cleanup); +} + static void -cleanup_pages(void *data UNUSED) +cleanup_global_pages(void *data UNUSED) { - for (uint seen = 0; (pages_kept > KEEP_PAGES) && (seen < KEEP_PAGES); seen++) + while (GFP->cnt < GFP->max) { - void *ptr = HEAD(pages_list); - rem_node(ptr); - if (munmap(ptr, get_page_size()) == 0) - pages_kept--; -#ifdef ENOMEM + node *n = alloc_sys_page(); + add_tail(&GFP->list, n + FP_NODE_OFFSET); + GFP->cnt++; + } + + for (uint limit = GFP->cnt; (limit > 0) && (GFP->cnt > GFP->max); limit--) + { + node *n = TAIL(GFP->list); + rem_node(n); + + if (munmap(n - FP_NODE_OFFSET, page_size) == 0) + GFP->cnt--; else if (errno == ENOMEM) - add_tail(&pages_list, ptr); -#endif + add_head(&GFP->list, n); else - bug("munmap(%p) failed: %m", ptr); + bug("munmap(%p) failed: %m", n - FP_NODE_OFFSET); + } +} + +void +init_pages(struct birdloop *loop) +{ + struct free_pages *fp = &loop->pages; + + init_list(&fp->list); + fp->cleanup = ev_new_init(loop->parent->loop->pool, cleanup_pages, loop); + fp->cleanup->list = (loop->parent->loop == &main_birdloop) ? &global_work_list : birdloop_event_list(loop->parent->loop); + fp->min = 4; + fp->max = 16; + + for (fp->cnt = 0; fp->cnt < fp->min; fp->cnt++) + { + node *n = alloc_sys_page(); + add_tail(&fp->list, n + FP_NODE_OFFSET); + } +} + +static event global_free_pages_cleanup_event = { .hook = cleanup_global_pages, .list = &global_work_list }; + +void resource_sys_init(void) +{ + if (!(page_size = sysconf(_SC_PAGESIZE))) + die("System page size must be non-zero"); + + if (u64_popcount(page_size) == 1) + { + init_list(&GFP->list); + GFP->cleanup = &global_free_pages_cleanup_event; + GFP->min = 0; + GFP->max = 256; + return; } - if (pages_kept > KEEP_PAGES) - ev_schedule(&page_cleanup_event); +#ifdef HAVE_ALIGNED_ALLOC + log(L_WARN "Got strange memory page size (%lu), using the aligned allocator instead", page_size); +#else + die("Got strange memory page size (%lu) and aligned_alloc is not available", page_size); +#endif + + /* Too big or strange page, use the aligned allocator instead */ + page_size = 4096; + use_fake = 1; } + +#else + +void +resource_sys_init(void) +{ + page_size = 4096; + use_fake = 1; +} + #endif diff --git a/sysdep/unix/coroutine.c b/sysdep/unix/coroutine.c new file mode 100644 index 00000000..4747d01a --- /dev/null +++ b/sysdep/unix/coroutine.c @@ -0,0 +1,206 @@ +/* + * BIRD Coroutines + * + * (c) 2017 Martin Mares <mj@ucw.cz> + * (c) 2020 Maria Matejka <mq@jmq.cz> + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#undef LOCAL_DEBUG + +#undef DEBUG_LOCKING + +#include "lib/birdlib.h" +#include "lib/locking.h" +#include "lib/coro.h" +#include "lib/rcu.h" +#include "lib/resource.h" +#include "lib/timer.h" + +#include "conf/conf.h" + +#define CORO_STACK_SIZE 65536 + +/* + * Implementation of coroutines based on POSIX threads + */ + +#include <errno.h> +#include <fcntl.h> +#include <poll.h> +#include <pthread.h> +#include <semaphore.h> +#include <stdatomic.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +/* + * Locking subsystem + */ + +_Thread_local struct lock_order locking_stack = {}; +_Thread_local struct domain_generic **last_locked = NULL; + +#define ASSERT_NO_LOCK ASSERT_DIE(last_locked == NULL) + +struct domain_generic { + pthread_mutex_t mutex; + uint order; + struct domain_generic **prev; + struct lock_order *locked_by; + const char *name; +}; + +#define DOMAIN_INIT(_name, _order) { .mutex = PTHREAD_MUTEX_INITIALIZER, .name = _name, .order = _order } + +static struct domain_generic the_bird_domain_gen = DOMAIN_INIT("The BIRD", OFFSETOF(struct lock_order, the_bird)); + +DOMAIN(the_bird) the_bird_domain = { .the_bird = &the_bird_domain_gen }; + +struct domain_generic * +domain_new(const char *name, uint order) +{ + ASSERT_DIE(order < sizeof(struct lock_order)); + struct domain_generic *dg = xmalloc(sizeof(struct domain_generic)); + *dg = (struct domain_generic) DOMAIN_INIT(name, order); + return dg; +} + +void +domain_free(struct domain_generic *dg) +{ + pthread_mutex_destroy(&dg->mutex); + xfree(dg); +} + +uint dg_order(struct domain_generic *dg) +{ + return dg->order; +} + +void do_lock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to lock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + + if (lsp <= last_locked) + bug("Trying to lock in a bad order"); + if (*lsp) + bug("Inconsistent locking stack state on lock"); + + btime lock_begin = current_time(); + pthread_mutex_lock(&dg->mutex); + btime duration = current_time() - lock_begin; + if (config && (duration > config->watchdog_warning)) + log(L_WARN "Locking of %s took %d ms", dg->name, (int) (duration TO_MS)); + + if (dg->prev || dg->locked_by) + bug("Previous unlock not finished correctly"); + dg->prev = last_locked; + *lsp = dg; + last_locked = lsp; + dg->locked_by = &locking_stack; +} + +void do_unlock(struct domain_generic *dg, struct domain_generic **lsp) +{ + if ((char *) lsp - (char *) &locking_stack != dg->order) + bug("Trying to unlock on bad position: order=%u, lsp=%p, base=%p", dg->order, lsp, &locking_stack); + + if (dg->locked_by != &locking_stack) + bug("Inconsistent domain state on unlock"); + if ((last_locked != lsp) || (*lsp != dg)) + bug("Inconsistent locking stack state on unlock"); + dg->locked_by = NULL; + last_locked = dg->prev; + *lsp = NULL; + dg->prev = NULL; + pthread_mutex_unlock(&dg->mutex); +} + +/* Coroutines */ +struct coroutine { + resource r; + pthread_t id; + pthread_attr_t attr; + struct rcu_coro rcu; + void (*entry)(void *); + void *data; +}; + +static _Thread_local _Bool coro_cleaned_up = 0; + +static void coro_free(resource *r) +{ + struct coroutine *c = (void *) r; + rcu_coro_stop(&c->rcu); + ASSERT_DIE(pthread_equal(pthread_self(), c->id)); + pthread_attr_destroy(&c->attr); + coro_cleaned_up = 1; +} + +static void coro_dump(resource *r UNUSED) { } + +static struct resclass coro_class = { + .name = "Coroutine", + .size = sizeof(struct coroutine), + .free = coro_free, + .dump = coro_dump, +}; + +_Thread_local struct coroutine *this_coro = NULL; + +static void *coro_entry(void *p) +{ + struct coroutine *c = p; + + ASSERT_DIE(c->entry); + + this_coro = c; + rcu_coro_start(&c->rcu); + + c->entry(c->data); + ASSERT_DIE(coro_cleaned_up); + + return NULL; +} + +struct coroutine *coro_run(pool *p, void (*entry)(void *), void *data) +{ + ASSERT_DIE(entry); + ASSERT_DIE(p); + + struct coroutine *c = ralloc(p, &coro_class); + + c->entry = entry; + c->data = data; + + int e = 0; + + if (e = pthread_attr_init(&c->attr)) + die("pthread_attr_init() failed: %M", e); + + if (e = pthread_attr_setstacksize(&c->attr, CORO_STACK_SIZE)) + die("pthread_attr_setstacksize(%u) failed: %M", CORO_STACK_SIZE, e); + + if (e = pthread_attr_setdetachstate(&c->attr, PTHREAD_CREATE_DETACHED)) + die("pthread_attr_setdetachstate(PTHREAD_CREATE_DETACHED) failed: %M", e); + + if (e = pthread_create(&c->id, &c->attr, coro_entry, c)) + die("pthread_create() failed: %M", e); + + return c; +} + +void +coro_yield(void) +{ + const struct timespec req = { .tv_nsec = 100 }; + nanosleep(&req, NULL); +} diff --git a/sysdep/unix/io-loop.c b/sysdep/unix/io-loop.c new file mode 100644 index 00000000..9b107a1a --- /dev/null +++ b/sysdep/unix/io-loop.c @@ -0,0 +1,620 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <fcntl.h> +#include <poll.h> +#include <pthread.h> +#include <time.h> +#include <sys/time.h> + +#include "nest/bird.h" + +#include "lib/buffer.h" +#include "lib/coro.h" +#include "lib/lists.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/timer.h" +#include "lib/socket.h" + +#include "lib/io-loop.h" +#include "sysdep/unix/io-loop.h" +#include "conf/conf.h" + +/* + * Current thread context + */ + +_Thread_local struct birdloop *birdloop_current = NULL; +static _Thread_local struct birdloop *birdloop_wakeup_masked; +static _Thread_local uint birdloop_wakeup_masked_count; + +event_list * +birdloop_event_list(struct birdloop *loop) +{ + return &loop->event_list; +} + +struct timeloop * +birdloop_time_loop(struct birdloop *loop) +{ + return &loop->time; +} + +pool * +birdloop_pool(struct birdloop *loop) +{ + return loop->pool; +} + +_Bool +birdloop_inside(struct birdloop *loop) +{ + for (struct birdloop *c = birdloop_current; c; c = c->prev_loop) + if (loop == c) + return 1; + + return 0; +} + +/* + * Wakeup code for birdloop + */ + +static void +pipe_new(int *pfds) +{ + int rv = pipe(pfds); + if (rv < 0) + die("pipe: %m"); + + if (fcntl(pfds[0], F_SETFL, O_NONBLOCK) < 0) + die("fcntl(O_NONBLOCK): %m"); + + if (fcntl(pfds[1], F_SETFL, O_NONBLOCK) < 0) + die("fcntl(O_NONBLOCK): %m"); +} + +void +pipe_drain(int fd) +{ + char buf[64]; + int rv; + + try: + rv = read(fd, buf, 64); + if (rv < 0) + { + if (errno == EINTR) + goto try; + if (errno == EAGAIN) + return; + die("wakeup read: %m"); + } + if (rv == 64) + goto try; +} + +void +pipe_kick(int fd) +{ + u64 v = 1; + int rv; + + try: + rv = write(fd, &v, sizeof(u64)); + if (rv < 0) + { + if (errno == EINTR) + goto try; + if (errno == EAGAIN) + return; + die("wakeup write: %m"); + } +} + +static inline void +wakeup_init(struct birdloop *loop) +{ + pipe_new(loop->wakeup_fds); +} + +static inline void +wakeup_drain(struct birdloop *loop) +{ + pipe_drain(loop->wakeup_fds[0]); +} + +static inline void +wakeup_do_kick(struct birdloop *loop) +{ + pipe_kick(loop->wakeup_fds[1]); +} + +void +birdloop_ping(struct birdloop *loop) +{ + u32 ping_sent = atomic_fetch_add_explicit(&loop->ping_sent, 1, memory_order_acq_rel); + if (ping_sent) + return; + + if (loop == birdloop_wakeup_masked) + birdloop_wakeup_masked_count++; + else + wakeup_do_kick(loop); +} + + +/* + * Sockets + */ + +static void +sockets_init(struct birdloop *loop) +{ + init_list(&loop->sock_list); + loop->sock_num = 0; + + BUFFER_INIT(loop->poll_sk, loop->pool, 4); + BUFFER_INIT(loop->poll_fd, loop->pool, 4); + loop->poll_changed = 1; /* add wakeup fd */ +} + +static void +sockets_add(struct birdloop *loop, sock *s) +{ + ASSERT_DIE(!enlisted(&s->n)); + + add_tail(&loop->sock_list, &s->n); + loop->sock_num++; + + s->loop = loop; + s->index = -1; + loop->poll_changed = 1; + + birdloop_ping(loop); +} + +void +sk_start(sock *s) +{ + ASSERT_DIE(birdloop_current != &main_birdloop); + sockets_add(birdloop_current, s); +} + +static void +sockets_remove(struct birdloop *loop, sock *s) +{ + ASSERT_DIE(s->loop == loop); + + if (!enlisted(&s->n)) + return; + + rem_node(&s->n); + loop->sock_num--; + + if (s->index >= 0) + { + loop->poll_sk.data[s->index] = NULL; + s->index = -1; + loop->poll_changed = 1; + birdloop_ping(loop); + } + + s->loop = NULL; +} + +void +sk_stop(sock *s) +{ + sockets_remove(birdloop_current, s); +} + +static inline uint sk_want_events(sock *s) +{ + uint out = ((s->ttx != s->tpos) ? POLLOUT : 0); + if (s->rx_hook) + if (s->cork) + { + LOCK_DOMAIN(cork, s->cork->lock); + if (!enlisted(&s->cork_node)) + if (s->cork->count) + { +// log(L_TRACE "Socket %p corked", s); + add_tail(&s->cork->sockets, &s->cork_node); + } + else + out |= POLLIN; + UNLOCK_DOMAIN(cork, s->cork->lock); + } + else + out |= POLLIN; + +// log(L_TRACE "sk_want_events(%p) = %x", s, out); + return out; +} + + +void +sk_ping(sock *s) +{ + s->loop->poll_changed = 1; + birdloop_ping(s->loop); +} + +/* +FIXME: this should be called from sock code + +static void +sockets_update(struct birdloop *loop, sock *s) +{ + if (s->index >= 0) + loop->poll_fd.data[s->index].events = sk_want_events(s); +} +*/ + +static void +sockets_prepare(struct birdloop *loop) +{ + BUFFER_SET(loop->poll_sk, loop->sock_num + 1); + BUFFER_SET(loop->poll_fd, loop->sock_num + 1); + + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + uint i = 0; + node *n; + + WALK_LIST(n, loop->sock_list) + { + sock *s = SKIP_BACK(sock, n, n); + + ASSERT(i < loop->sock_num); + + s->index = i; + *psk = s; + pfd->fd = s->fd; + pfd->events = sk_want_events(s); + pfd->revents = 0; + + pfd++; + psk++; + i++; + } + + ASSERT(i == loop->sock_num); + + /* Add internal wakeup fd */ + *psk = NULL; + pfd->fd = loop->wakeup_fds[0]; + pfd->events = POLLIN; + pfd->revents = 0; + + loop->poll_changed = 0; +} + +int sk_read(sock *s, int revents); +int sk_write(sock *s); + +static void +sockets_fire(struct birdloop *loop) +{ + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + int poll_num = loop->poll_fd.used - 1; + + times_update(); + + /* Last fd is internal wakeup fd */ + if (pfd[poll_num].revents & POLLIN) + { + wakeup_drain(loop); + loop->poll_changed = 1; + } + + int i; + for (i = 0; i < poll_num; pfd++, psk++, i++) + { + if (!*psk) + continue; + + if (! pfd->revents) + continue; + + if (pfd->revents & POLLNVAL) + bug("poll: invalid fd %d", pfd->fd); + + int e = 1; + + if (pfd->revents & POLLIN) + while (e && *psk && (*psk)->rx_hook) + e = sk_read(*psk, pfd->revents); + + e = 1; + if (pfd->revents & POLLOUT) + { + loop->poll_changed = 1; + while (e && *psk) + e = sk_write(*psk); + } + } +} + + +/* + * Birdloop + */ + +struct birdloop main_birdloop; + +static void birdloop_enter_locked(struct birdloop *loop); + +void +birdloop_init(void) +{ + wakeup_init(&main_birdloop); + + main_birdloop.time.domain = the_bird_domain.the_bird; + main_birdloop.time.loop = &main_birdloop; + + times_update(); + timers_init(&main_birdloop.time, &root_pool); + + root_pool.loop = &main_birdloop; + main_birdloop.pool = &root_pool; + + birdloop_enter_locked(&main_birdloop); +} + +static void birdloop_main(void *arg); + +void +birdloop_free(resource *r) +{ + struct birdloop *loop = (void *) r; + + ASSERT_DIE(loop->links == 0); + domain_free(loop->time.domain); +} + +void +birdloop_dump(resource *r) +{ + struct birdloop *loop = (void *) r; + + debug("%s\n", loop->pool->name); +} + +struct resmem birdloop_memsize(resource *r) +{ + struct birdloop *loop = (void *) r; + + return (struct resmem) { + .effective = sizeof(struct birdloop) - sizeof(resource) - ALLOC_OVERHEAD, + .overhead = ALLOC_OVERHEAD + sizeof(resource) + page_size * list_length(&loop->pages.list), + }; +} + +struct resclass birdloop_class = { + .name = "IO Loop", + .size = sizeof(struct birdloop), + .free = birdloop_free, + .dump = birdloop_dump, + .memsize = birdloop_memsize, +}; + +struct birdloop * +birdloop_new(pool *pp, uint order, const char *name) +{ + struct domain_generic *dg = domain_new(name, order); + + struct birdloop *loop = ralloc(pp, &birdloop_class); + + loop->time.domain = dg; + loop->time.loop = loop; + + birdloop_enter(loop); + + loop->pool = rp_new(pp, loop, name); + loop->parent = pp; + rmove(&loop->r, loop->pool); + + wakeup_init(loop); + ev_init_list(&loop->event_list, loop, name); + timers_init(&loop->time, loop->pool); + sockets_init(loop); + + init_pages(loop); + + loop->time.coro = coro_run(loop->pool, birdloop_main, loop); + + birdloop_leave(loop); + + return loop; +} + +static void +birdloop_do_stop(struct birdloop *loop, void (*stopped)(void *data), void *data) +{ + loop->stopped = stopped; + loop->stop_data = data; + wakeup_do_kick(loop); +} + +void +birdloop_stop(struct birdloop *loop, void (*stopped)(void *data), void *data) +{ + DG_LOCK(loop->time.domain); + birdloop_do_stop(loop, stopped, data); + DG_UNLOCK(loop->time.domain); +} + +void +birdloop_stop_self(struct birdloop *loop, void (*stopped)(void *data), void *data) +{ + ASSERT_DIE(loop == birdloop_current); + ASSERT_DIE(DG_IS_LOCKED(loop->time.domain)); + + birdloop_do_stop(loop, stopped, data); +} + +static void +birdloop_enter_locked(struct birdloop *loop) +{ + ASSERT_DIE(DG_IS_LOCKED(loop->time.domain)); + ASSERT_DIE(!birdloop_inside(loop)); + + /* Store the old context */ + loop->prev_loop = birdloop_current; + + /* Put the new context */ + birdloop_current = loop; +} + +void +birdloop_enter(struct birdloop *loop) +{ + DG_LOCK(loop->time.domain); + return birdloop_enter_locked(loop); +} + +static void +birdloop_leave_locked(struct birdloop *loop) +{ + /* Check the current context */ + ASSERT_DIE(birdloop_current == loop); + + /* Restore the old context */ + birdloop_current = loop->prev_loop; +} + +void +birdloop_leave(struct birdloop *loop) +{ + birdloop_leave_locked(loop); + DG_UNLOCK(loop->time.domain); +} + +void +birdloop_mask_wakeups(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_wakeup_masked == NULL); + birdloop_wakeup_masked = loop; +} + +void +birdloop_unmask_wakeups(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_wakeup_masked == loop); + birdloop_wakeup_masked = NULL; + if (birdloop_wakeup_masked_count) + wakeup_do_kick(loop); + + birdloop_wakeup_masked_count = 0; +} + +void +birdloop_link(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_inside(loop)); + loop->links++; +} + +void +birdloop_unlink(struct birdloop *loop) +{ + ASSERT_DIE(birdloop_inside(loop)); + ASSERT_DIE(loop->links); + if (!--loop->links) + birdloop_ping(loop); +} + +static void +birdloop_main(void *arg) +{ + struct birdloop *loop = arg; + timer *t; + int rv, timeout; + + btime loop_begin = current_time(); + + birdloop_enter(loop); + while (1) + { + timers_fire(&loop->time, 0); + if (ev_run_list(&loop->event_list)) + timeout = 0; + else if (t = timers_first(&loop->time)) + timeout = (tm_remains(t) TO_MS) + 1; + else + timeout = -1; + + if (loop->poll_changed) + sockets_prepare(loop); + + btime duration = current_time_update() - loop_begin; + if (duration > config->watchdog_warning) + log(L_WARN "I/O loop cycle took %d ms", (int) (duration TO_MS)); + + birdloop_leave(loop); + + try: + rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout); + if (rv < 0) + { + if (errno == EINTR || errno == EAGAIN) + goto try; + die("poll: %m"); + } + + birdloop_enter(loop); + + if (loop->stopped && !loop->links) + break; + + loop_begin = current_time_update(); + + if (rv) + sockets_fire(loop); + + atomic_exchange_explicit(&loop->ping_sent, 0, memory_order_acq_rel); + } + + /* Flush remaining events */ + ASSERT_DIE(!ev_run_list(&loop->event_list)); + + /* No timers allowed */ + ASSERT_DIE(timers_count(&loop->time) == 0); + ASSERT_DIE(EMPTY_LIST(loop->sock_list)); + ASSERT_DIE(loop->sock_num == 0); + + birdloop_leave(loop); + + /* Lock parent loop */ + pool *parent = loop->parent; + birdloop_enter(parent->loop); + + /* Move the loop temporarily to parent pool */ + birdloop_enter(loop); + rmove(&loop->r, parent); + birdloop_leave(loop); + + /* Announce loop stop */ + loop->stopped(loop->stop_data); + + /* Free the pool and loop */ + birdloop_enter(loop); + rp_free(loop->pool, parent); + flush_pages(loop); + birdloop_leave(loop); + rfree(&loop->r); + + /* And finally leave the parent loop before finishing */ + birdloop_leave(parent->loop); +} diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h new file mode 100644 index 00000000..1727637a --- /dev/null +++ b/sysdep/unix/io-loop.h @@ -0,0 +1,58 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_SYSDEP_UNIX_IO_LOOP_H_ +#define _BIRD_SYSDEP_UNIX_IO_LOOP_H_ + +#include "nest/bird.h" + +#include "lib/lists.h" +#include "lib/event.h" +#include "lib/timer.h" + +struct free_pages +{ + list list; /* List of empty pages */ + event *cleanup; /* Event to call when number of pages is outside bounds */ + u16 min, max; /* Minimal and maximal number of free pages kept */ + uint cnt; /* Number of empty pages */ +}; + +struct birdloop +{ + resource r; + + pool *pool; + pool *parent; + + struct timeloop time; + event_list event_list; + list sock_list; + uint sock_num; + + BUFFER(sock *) poll_sk; + BUFFER(struct pollfd) poll_fd; + u8 poll_changed; + + _Atomic u32 ping_sent; + int wakeup_fds[2]; + + uint links; + + struct free_pages pages; + + void (*stopped)(void *data); + void *stop_data; + + struct birdloop *prev_loop; +}; + +extern _Thread_local struct birdloop *birdloop_current; + +void init_pages(struct birdloop *loop); +void flush_pages(struct birdloop *loop); + +#endif diff --git a/sysdep/unix/io.c b/sysdep/unix/io.c index 3d67d0a7..e3e388af 100644 --- a/sysdep/unix/io.c +++ b/sysdep/unix/io.c @@ -36,12 +36,14 @@ #include "lib/resource.h" #include "lib/socket.h" #include "lib/event.h" +#include "lib/locking.h" #include "lib/timer.h" #include "lib/string.h" #include "nest/iface.h" #include "conf/conf.h" #include "sysdep/unix/unix.h" +#include "sysdep/unix/io-loop.h" #include CONFIG_INCLUDE_SYSIO_H /* Maximum number of calls of tx handler for one socket in one @@ -122,55 +124,50 @@ rf_fileno(struct rfile *f) btime boot_time; + void -times_init(struct timeloop *loop) +times_update(void) { struct timespec ts; int rv; + btime old_time = current_time(); + btime old_real_time = current_real_time(); + rv = clock_gettime(CLOCK_MONOTONIC, &ts); if (rv < 0) die("Monotonic clock is missing"); if ((ts.tv_sec < 0) || (((u64) ts.tv_sec) > ((u64) 1 << 40))) log(L_WARN "Monotonic clock is crazy"); - - loop->last_time = ts.tv_sec S + ts.tv_nsec NS; - loop->real_time = 0; -} - -void -times_update(struct timeloop *loop) -{ - struct timespec ts; - int rv; - - rv = clock_gettime(CLOCK_MONOTONIC, &ts); - if (rv < 0) - die("clock_gettime: %m"); - + btime new_time = ts.tv_sec S + ts.tv_nsec NS; - if (new_time < loop->last_time) + if (new_time < old_time) log(L_ERR "Monotonic clock is broken"); - loop->last_time = new_time; - loop->real_time = 0; -} - -void -times_update_real_time(struct timeloop *loop) -{ - struct timespec ts; - int rv; - rv = clock_gettime(CLOCK_REALTIME, &ts); if (rv < 0) die("clock_gettime: %m"); - loop->real_time = ts.tv_sec S + ts.tv_nsec NS; -} + btime new_real_time = ts.tv_sec S + ts.tv_nsec NS; + + if (!atomic_compare_exchange_strong_explicit( + &last_time, + &old_time, + new_time, + memory_order_acq_rel, + memory_order_relaxed)) + DBG("Time update collision: last_time"); + if (!atomic_compare_exchange_strong_explicit( + &real_time, + &old_real_time, + new_real_time, + memory_order_acq_rel, + memory_order_relaxed)) + DBG("Time update collision: real_time"); +} /** * DOC: Sockets @@ -797,6 +794,7 @@ sk_free(resource *r) { sock *s = (sock *) r; + ASSERT_DIE(!s->loop || birdloop_inside(s->loop)); sk_free_bufs(s); #ifdef HAVE_LIBSSH @@ -804,20 +802,30 @@ sk_free(resource *r) sk_ssh_free(s); #endif - if (s->fd < 0) - return; + if (s->cork) + { + LOCK_DOMAIN(cork, s->cork->lock); + if (enlisted(&s->cork_node)) + rem_node(&s->cork_node); + UNLOCK_DOMAIN(cork, s->cork->lock); + } - /* FIXME: we should call sk_stop() for SKF_THREAD sockets */ - if (!(s->flags & SKF_THREAD)) + if (!s->loop) + ; + else if (s->flags & SKF_THREAD) + sk_stop(s); + else { if (s == current_sock) current_sock = sk_next(s); if (s == stored_sock) stored_sock = sk_next(s); - rem_node(&s->n); + + if (enlisted(&s->n)) + rem_node(&s->n); } - if (s->type != SK_SSH && s->type != SK_SSH_ACTIVE) + if (s->type != SK_SSH && s->type != SK_SSH_ACTIVE && s->fd != -1) close(s->fd); s->fd = -1; @@ -1108,7 +1116,15 @@ sk_passive_connected(sock *s, int type) return 1; } - sk_insert(t); + if (s->flags & SKF_PASSIVE_THREAD) + t->flags |= SKF_THREAD; + else + { + ASSERT_DIE(s->loop == &main_birdloop); + t->loop = &main_birdloop; + sk_insert(t); + } + sk_alloc_bufs(t); s->rx_hook(t, 0); return 1; @@ -1329,6 +1345,17 @@ sk_open(sock *s) ip_addr bind_addr = IPA_NONE; sockaddr sa; + if (s->flags & SKF_THREAD) + { + ASSERT_DIE(s->loop && (s->loop != &main_birdloop)); + ASSERT_DIE(birdloop_inside(s->loop)); + } + else + { + ASSERT_DIE(!s->loop); + s->loop = &main_birdloop; + } + if (s->type <= SK_IP) { /* @@ -1508,10 +1535,41 @@ sk_open_unix(sock *s, char *name) return -1; s->fd = fd; + s->loop = &main_birdloop; sk_insert(s); return 0; } +static void +sk_reloop_hook(void *_vs) +{ + sock *s = _vs; + if (birdloop_inside(&main_birdloop)) + { + s->flags &= ~SKF_THREAD; + sk_insert(s); + } + else + { + s->flags |= SKF_THREAD; + sk_start(s); + } +} + +void +sk_reloop(sock *s, struct birdloop *loop) +{ + if (enlisted(&s->n)) + rem_node(&s->n); + + s->reloop = (event) { + .hook = sk_reloop_hook, + .data = s, + }; + + ev_send_loop(loop, &s->reloop); +} + #define CMSG_RX_SPACE MAX(CMSG4_SPACE_PKTINFO+CMSG4_SPACE_TTL, \ CMSG6_SPACE_PKTINFO+CMSG6_SPACE_TTL) @@ -1657,6 +1715,7 @@ sk_maybe_write(sock *s) s->err_hook(s, (errno != EPIPE) ? errno : 0); return -1; } + sk_ping(s); return 0; } s->ttx += e; @@ -1864,6 +1923,25 @@ sk_read(sock *s, int revents) case SK_TCP: case SK_UNIX: { + if (s->cork) + { + int cont = 0; + LOCK_DOMAIN(cork, s->cork->lock); + if (!enlisted(&s->cork_node)) + if (s->cork->count) + { +// log(L_TRACE "Socket %p corked", s); + add_tail(&s->cork->sockets, &s->cork_node); + sk_ping(s); + } + else + cont = 1; + UNLOCK_DOMAIN(cork, s->cork->lock); + + if (!cont) + return 0; + } + int c = read(s->fd, s->rpos, s->rbuf + s->rbsize - s->rpos); if (c < 0) @@ -2016,30 +2094,17 @@ struct event_log_entry static struct event_log_entry event_log[EVENT_LOG_LENGTH]; static struct event_log_entry *event_open; static int event_log_pos, event_log_num, watchdog_active; -static btime last_time; +static btime last_io_time; static btime loop_time; static void io_update_time(void) { - struct timespec ts; - int rv; - - /* - * This is third time-tracking procedure (after update_times() above and - * times_update() in BFD), dedicated to internal event log and latency - * tracking. Hopefully, we consolidate these sometimes. - */ - - rv = clock_gettime(CLOCK_MONOTONIC, &ts); - if (rv < 0) - die("clock_gettime: %m"); - - last_time = ts.tv_sec S + ts.tv_nsec NS; + last_io_time = current_time_update(); if (event_open) { - event_open->duration = last_time - event_open->timestamp; + event_open->duration = last_io_time - event_open->timestamp; if (event_open->duration > config->latency_limit) log(L_WARN "Event 0x%p 0x%p took %d ms", @@ -2068,7 +2133,7 @@ io_log_event(void *hook, void *data) en->hook = hook; en->data = data; - en->timestamp = last_time; + en->timestamp = last_io_time; en->duration = 0; event_log_num++; @@ -2096,14 +2161,14 @@ io_log_dump(void) struct event_log_entry *en = event_log + (event_log_pos + i) % EVENT_LOG_LENGTH; if (en->hook) log(L_DEBUG " Event 0x%p 0x%p at %8d for %d ms", en->hook, en->data, - (int) ((last_time - en->timestamp) TO_MS), (int) (en->duration TO_MS)); + (int) ((last_io_time - en->timestamp) TO_MS), (int) (en->duration TO_MS)); } } void watchdog_sigalrm(int sig UNUSED) { - /* Update last_time and duration, but skip latency check */ + /* Update last_io_time and duration, but skip latency check */ config->latency_limit = 0xffffffff; io_update_time(); @@ -2116,7 +2181,7 @@ watchdog_start1(void) { io_update_time(); - loop_time = last_time; + loop_time = last_io_time; } static inline void @@ -2124,7 +2189,7 @@ watchdog_start(void) { io_update_time(); - loop_time = last_time; + loop_time = last_io_time; event_log_num = 0; if (config->watchdog_timeout) @@ -2145,7 +2210,7 @@ watchdog_stop(void) watchdog_active = 0; } - btime duration = last_time - loop_time; + btime duration = last_io_time - loop_time; if (duration > config->watchdog_warning) log(L_WARN "I/O loop cycle took %d ms for %d events", (int) (duration TO_MS), event_log_num); @@ -2160,8 +2225,9 @@ void io_init(void) { init_list(&sock_list); - init_list(&global_event_list); - init_list(&global_work_list); + ev_init_list(&global_event_list, &main_birdloop, "Global event list"); + ev_init_list(&global_work_list, &main_birdloop, "Global work list"); + ev_init_list(&main_birdloop.event_list, &main_birdloop, "Global fast event list"); krt_io_init(); // XXX init_times(); // XXX update_times(); @@ -2175,11 +2241,15 @@ static int short_loops = 0; #define SHORT_LOOP_MAX 10 #define WORK_EVENTS_MAX 10 +void pipe_drain(int fd); +void check_stored_pages(void); + void io_loop(void) { int poll_tout, timeout; int nfds, events, pout; + int reload_requested = 0; timer *t; sock *s; node *n; @@ -2189,27 +2259,39 @@ io_loop(void) watchdog_start1(); for(;;) { - times_update(&main_timeloop); + times_update(); events = ev_run_list(&global_event_list); events = ev_run_list_limited(&global_work_list, WORK_EVENTS_MAX) || events; - timers_fire(&main_timeloop); + events = ev_run_list(&main_birdloop.event_list) || events; + timers_fire(&main_birdloop.time, 1); io_close_event(); +#if DEBUGGING +#define PERIODIC_WAKEUP 86400000 +#else +#define PERIODIC_WAKEUP 3000 +#endif +restart_poll: // FIXME - poll_tout = (events ? 0 : 3000); /* Time in milliseconds */ - if (t = timers_first(&main_timeloop)) + poll_tout = ((reload_requested || events) ? 0 : PERIODIC_WAKEUP); /* Time in milliseconds */ + if (t = timers_first(&main_birdloop.time)) { - times_update(&main_timeloop); + times_update(); timeout = (tm_remains(t) TO_MS) + 1; poll_tout = MIN(poll_tout, timeout); } - nfds = 0; + /* A hack to reload main io_loop() when something has changed asynchronously. */ + pfd[0].fd = main_birdloop.wakeup_fds[0]; + pfd[0].events = POLLIN; + + nfds = 1; + WALK_LIST(n, sock_list) { pfd[nfds] = (struct pollfd) { .fd = -1 }; /* everything other set to 0 by this */ s = SKIP_BACK(sock, n, n); - if (s->rx_hook) + if (s->rx_hook && !ev_corked(s->cork)) { pfd[nfds].fd = s->fd; pfd[nfds].events |= POLLIN; @@ -2263,7 +2345,9 @@ io_loop(void) /* And finally enter poll() to find active sockets */ watchdog_stop(); + birdloop_leave(&main_birdloop); pout = poll(pfd, nfds, poll_tout); + birdloop_enter(&main_birdloop); watchdog_start(); if (pout < 0) @@ -2272,9 +2356,24 @@ io_loop(void) continue; die("poll: %m"); } + + if (pout && (pfd[0].revents & POLLIN)) + { + /* IO loop reload requested */ + pipe_drain(main_birdloop.wakeup_fds[0]); + reload_requested = 1; + goto restart_poll; + } + + if (reload_requested) + { + reload_requested = 0; + atomic_exchange_explicit(&main_birdloop.ping_sent, 0, memory_order_acq_rel); + } + if (pout) { - times_update(&main_timeloop); + times_update(); /* guaranteed to be non-empty */ current_sock = SKIP_BACK(sock, n, HEAD(sock_list)); diff --git a/sysdep/unix/krt.c b/sysdep/unix/krt.c index 7c2614b1..0cb86213 100644 --- a/sysdep/unix/krt.c +++ b/sysdep/unix/krt.c @@ -74,7 +74,7 @@ static list krt_proto_list; void krt_io_init(void) { - krt_pool = rp_new(&root_pool, "Kernel Syncer"); + krt_pool = rp_new(&root_pool, &main_birdloop, "Kernel Syncer"); krt_filter_lp = lp_new_default(krt_pool); init_list(&krt_proto_list); krt_sys_io_init(); @@ -251,14 +251,14 @@ static inline void krt_trace_in(struct krt_proto *p, rte *e, char *msg) { if (p->p.debug & D_PACKETS) - log(L_TRACE "%s: %N: %s", p->p.name, e->net->n.addr, msg); + log(L_TRACE "%s: %N: %s", p->p.name, e->net, msg); } static inline void krt_trace_in_rl(struct tbf *f, struct krt_proto *p, rte *e, char *msg) { if (p->p.debug & D_PACKETS) - log_rl(f, L_TRACE "%s: %N: %s", p->p.name, e->net->n.addr, msg); + log_rl(f, L_TRACE "%s: %N: %s", p->p.name, e->net, msg); } /* @@ -277,261 +277,33 @@ static struct tbf rl_alien = TBF_DEFAULT_LOG_LIMITS; * the same key. */ -static inline int -krt_same_key(rte *a, rte *b) +static inline u32 +krt_metric(rte *a) { - return a->u.krt.metric == b->u.krt.metric; + eattr *ea = ea_find(a->attrs->eattrs, EA_KRT_METRIC); + return ea ? ea->u.data : 0; } static inline int -krt_uptodate(rte *a, rte *b) -{ - if (a->attrs != b->attrs) - return 0; - - if (a->u.krt.proto != b->u.krt.proto) - return 0; - - return 1; -} - -static void -krt_learn_announce_update(struct krt_proto *p, rte *e) -{ - net *n = e->net; - rta *aa = rta_clone(e->attrs); - rte *ee = rte_get_temp(aa); - ee->pflags = EA_ID_FLAG(EA_KRT_SOURCE) | EA_ID_FLAG(EA_KRT_METRIC); - ee->u.krt = e->u.krt; - rte_update(&p->p, n->n.addr, ee); -} - -static void -krt_learn_announce_delete(struct krt_proto *p, net *n) +krt_rte_better(rte *a, rte *b) { - rte_update(&p->p, n->n.addr, NULL); + return (krt_metric(a) > krt_metric(b)); } /* Called when alien route is discovered during scan */ static void -krt_learn_scan(struct krt_proto *p, rte *e) -{ - net *n0 = e->net; - net *n = net_get(p->krt_table, n0->n.addr); - rte *m, **mm; - - e->attrs = rta_lookup(e->attrs); - - for(mm=&n->routes; m = *mm; mm=&m->next) - if (krt_same_key(m, e)) - break; - if (m) - { - if (krt_uptodate(m, e)) - { - krt_trace_in_rl(&rl_alien, p, e, "[alien] seen"); - rte_free(e); - m->u.krt.seen = 1; - } - else - { - krt_trace_in(p, e, "[alien] updated"); - *mm = m->next; - rte_free(m); - m = NULL; - } - } - else - krt_trace_in(p, e, "[alien] created"); - if (!m) - { - e->next = n->routes; - n->routes = e; - e->u.krt.seen = 1; - } -} - -static void -krt_learn_prune(struct krt_proto *p) -{ - struct fib *fib = &p->krt_table->fib; - struct fib_iterator fit; - - KRT_TRACE(p, D_EVENTS, "Pruning inherited routes"); - - FIB_ITERATE_INIT(&fit, fib); -again: - FIB_ITERATE_START(fib, &fit, net, n) - { - rte *e, **ee, *best, **pbest, *old_best; - - /* - * Note that old_best may be NULL even if there was an old best route in - * the previous step, because it might be replaced in krt_learn_scan(). - * But in that case there is a new valid best route. - */ - - old_best = NULL; - best = NULL; - pbest = NULL; - ee = &n->routes; - while (e = *ee) - { - if (e->u.krt.best) - old_best = e; - - if (!e->u.krt.seen) - { - *ee = e->next; - rte_free(e); - continue; - } - - if (!best || best->u.krt.metric > e->u.krt.metric) - { - best = e; - pbest = ee; - } - - e->u.krt.seen = 0; - e->u.krt.best = 0; - ee = &e->next; - } - if (!n->routes) - { - DBG("%I/%d: deleting\n", n->n.prefix, n->n.pxlen); - if (old_best) - krt_learn_announce_delete(p, n); - - FIB_ITERATE_PUT(&fit); - fib_delete(fib, n); - goto again; - } - - best->u.krt.best = 1; - *pbest = best->next; - best->next = n->routes; - n->routes = best; - - if ((best != old_best) || p->reload) - { - DBG("%I/%d: announcing (metric=%d)\n", n->n.prefix, n->n.pxlen, best->u.krt.metric); - krt_learn_announce_update(p, best); - } - else - DBG("%I/%d: uptodate (metric=%d)\n", n->n.prefix, n->n.pxlen, best->u.krt.metric); - } - FIB_ITERATE_END; - - p->reload = 0; -} - -static void -krt_learn_async(struct krt_proto *p, rte *e, int new) +krt_learn_rte(struct krt_proto *p, rte *e) { - net *n0 = e->net; - net *n = net_get(p->krt_table, n0->n.addr); - rte *g, **gg, *best, **bestp, *old_best; - - e->attrs = rta_lookup(e->attrs); - - old_best = n->routes; - for(gg=&n->routes; g = *gg; gg = &g->next) - if (krt_same_key(g, e)) - break; - if (new) - { - if (g) - { - if (krt_uptodate(g, e)) - { - krt_trace_in(p, e, "[alien async] same"); - rte_free(e); - return; - } - krt_trace_in(p, e, "[alien async] updated"); - *gg = g->next; - rte_free(g); - } - else - krt_trace_in(p, e, "[alien async] created"); - - e->next = n->routes; - n->routes = e; - } - else if (!g) - { - krt_trace_in(p, e, "[alien async] delete failed"); - rte_free(e); - return; - } - else - { - krt_trace_in(p, e, "[alien async] removed"); - *gg = g->next; - rte_free(e); - rte_free(g); - } - best = n->routes; - bestp = &n->routes; - for(gg=&n->routes; g=*gg; gg=&g->next) - { - if (best->u.krt.metric > g->u.krt.metric) - { - best = g; - bestp = gg; - } - - g->u.krt.best = 0; - } - - if (best) - { - best->u.krt.best = 1; - *bestp = best->next; - best->next = n->routes; - n->routes = best; - } - - if (best != old_best) - { - DBG("krt_learn_async: distributing change\n"); - if (best) - krt_learn_announce_update(p, best); - else - krt_learn_announce_delete(p, n); - } + struct rte_src *src = e->src = rt_get_source(&p->p, krt_metric(e)); + rte_update(p->p.main_channel, e->net, e, e->src); + rt_unlock_source(src); } static void krt_learn_init(struct krt_proto *p) { if (KRT_CF->learn) - { - struct rtable_config *cf = mb_allocz(p->p.pool, sizeof(struct rtable_config)); - cf->name = "Inherited"; - cf->addr_type = p->p.net_type; - cf->internal = 1; - - p->krt_table = rt_setup(p->p.pool, cf); - } -} - -static void -krt_dump(struct proto *P) -{ - struct krt_proto *p = (struct krt_proto *) P; - - if (!KRT_CF->learn) - return; - debug("KRT: Table of inheritable routes\n"); - rt_dump(p->krt_table); -} - -static void -krt_dump_attrs(rte *e) -{ - debug(" [m=%d,p=%d]", e->u.krt.metric, e->u.krt.proto); + channel_setup_in_table(p->p.main_channel, 1); } #endif @@ -543,47 +315,58 @@ krt_dump_attrs(rte *e) static inline int krt_is_installed(struct krt_proto *p, net *n) { - return n->routes && bmap_test(&p->p.main_channel->export_map, n->routes->id); + return n->routes && bmap_test(&p->p.main_channel->export_map, n->routes->rte.id); } -static void -krt_flush_routes(struct krt_proto *p) +static uint +rte_feed_count(net *n) { - struct rtable *t = p->p.main_channel->table; + uint count = 0; + for (struct rte_storage *e = n->routes; e; e = e->next) + if (rte_is_valid(RTES_OR_NULL(e))) + count++; + return count; +} - KRT_TRACE(p, D_EVENTS, "Flushing kernel routes"); - FIB_WALK(&t->fib, net, n) +static void +rte_feed_obtain(net *n, rte **feed, uint count) +{ + uint i = 0; + for (struct rte_storage *e = n->routes; e; e = e->next) + if (rte_is_valid(RTES_OR_NULL(e))) { - if (krt_is_installed(p, n)) - { - /* FIXME: this does not work if gw is changed in export filter */ - krt_replace_rte(p, n, NULL, n->routes); - } + ASSERT_DIE(i < count); + feed[i++] = &e->rte; } - FIB_WALK_END; + ASSERT_DIE(i == count); } static struct rte * -krt_export_net(struct krt_proto *p, net *net, rte **rt_free) +krt_export_net(struct krt_proto *p, net *net) { struct channel *c = p->p.main_channel; const struct filter *filter = c->out_filter; - rte *rt; if (c->ra_mode == RA_MERGED) - return rt_export_merged(c, net, rt_free, krt_filter_lp, 1); + { + uint count = rte_feed_count(net); + if (!count) + return NULL; - rt = net->routes; - *rt_free = NULL; + rte **feed = alloca(count * sizeof(rte *)); + rte_feed_obtain(net, feed, count); + return rt_export_merged(c, feed, count, krt_filter_lp, 1); + } - if (!rte_is_valid(rt)) + static _Thread_local rte rt; + rt = net->routes->rte; + + if (!rte_is_valid(&rt)) return NULL; if (filter == FILTER_REJECT) return NULL; - rte_make_tmp_attrs(&rt, krt_filter_lp, NULL); - /* We could run krt_preexport() here, but it is already handled by krt_is_installed() */ if (filter == FILTER_ACCEPT) @@ -594,13 +377,9 @@ krt_export_net(struct krt_proto *p, net *net, rte **rt_free) accept: - if (rt != net->routes) - *rt_free = rt; - return rt; + return &rt; reject: - if (rt != net->routes) - rte_free(rt); return NULL; } @@ -624,13 +403,13 @@ krt_same_dest(rte *k, rte *e) */ void -krt_got_route(struct krt_proto *p, rte *e) +krt_got_route(struct krt_proto *p, rte *e, s8 src) { - rte *new = NULL, *rt_free = NULL; - net *n = e->net; + rte *new = NULL; + e->pflags = 0; #ifdef KRT_ALLOW_LEARN - switch (e->u.krt.src) + switch (src) { case KRT_SRC_KERNEL: goto ignore; @@ -640,26 +419,28 @@ krt_got_route(struct krt_proto *p, rte *e) case KRT_SRC_ALIEN: if (KRT_CF->learn) - krt_learn_scan(p, e); + krt_learn_rte(p, e); else - { - krt_trace_in_rl(&rl_alien, p, e, "[alien] ignored"); - rte_free(e); - } + krt_trace_in_rl(&rl_alien, p, e, "[alien] ignored"); return; } #endif /* The rest is for KRT_SRC_BIRD (or KRT_SRC_UNKNOWN) */ + RT_LOCK(p->p.main_channel->table); + /* Deleting all routes if flush is requested */ + if (p->flush_routes) + goto delete; /* We wait for the initial feed to have correct installed state */ if (!p->ready) goto ignore; - if (!krt_is_installed(p, n)) + net *net = net_find(RT_PRIV(p->p.main_channel->table), e->net); + if (!net || !krt_is_installed(p, net)) goto delete; - new = krt_export_net(p, n, &rt_free); + new = krt_export_net(p, net); /* Rejected by filters */ if (!new) @@ -692,20 +473,16 @@ ignore: update: krt_trace_in(p, new, "updating"); - krt_replace_rte(p, n, new, e); + krt_replace_rte(p, e->net, new, e); goto done; delete: krt_trace_in(p, e, "deleting"); - krt_replace_rte(p, n, NULL, e); + krt_replace_rte(p, e->net, NULL, e); goto done; done: - rte_free(e); - - if (rt_free) - rte_free(rt_free); - + RT_UNLOCK(p->p.main_channel->table); lp_flush(krt_filter_lp); } @@ -713,50 +490,65 @@ static void krt_init_scan(struct krt_proto *p) { bmap_reset(&p->seen_map, 1024); + +#ifdef KRT_ALLOW_LEARN + if (KRT_CF->learn) + channel_refresh_begin(p->p.main_channel); +#endif } static void krt_prune(struct krt_proto *p) { - struct rtable *t = p->p.main_channel->table; + RT_LOCK(p->p.main_channel->table); + rtable_private *t = RT_PRIV(p->p.main_channel->table); KRT_TRACE(p, D_EVENTS, "Pruning table %s", t->name); FIB_WALK(&t->fib, net, n) { - if (p->ready && krt_is_installed(p, n) && !bmap_test(&p->seen_map, n->routes->id)) + if (p->ready && krt_is_installed(p, n) && !bmap_test(&p->seen_map, n->routes->rte.id)) { - rte *rt_free = NULL; - rte *new = krt_export_net(p, n, &rt_free); + rte *new = krt_export_net(p, n); if (new) { krt_trace_in(p, new, "installing"); - krt_replace_rte(p, n, new, NULL); + krt_replace_rte(p, n->n.addr, new, NULL); } - if (rt_free) - rte_free(rt_free); - lp_flush(krt_filter_lp); } } FIB_WALK_END; + RT_UNLOCK(p->p.main_channel->table); + #ifdef KRT_ALLOW_LEARN if (KRT_CF->learn) - krt_learn_prune(p); + channel_refresh_end(p->p.main_channel); #endif if (p->ready) p->initialized = 1; } +static void +krt_flush_routes(struct krt_proto *p) +{ + KRT_TRACE(p, D_EVENTS, "Flushing kernel routes"); + p->flush_routes = 1; + krt_init_scan(p); + krt_do_scan(p); + /* No prune! */ + p->flush_routes = 0; +} + void -krt_got_route_async(struct krt_proto *p, rte *e, int new) +krt_got_route_async(struct krt_proto *p, rte *e, int new, s8 src) { - net *net = e->net; + e->pflags = 0; - switch (e->u.krt.src) + switch (src) { case KRT_SRC_BIRD: /* Should be filtered by the back end */ @@ -766,7 +558,7 @@ krt_got_route_async(struct krt_proto *p, rte *e, int new) if (new) { krt_trace_in(p, e, "[redirect] deleting"); - krt_replace_rte(p, net, NULL, e); + krt_replace_rte(p, e->net, NULL, e); } /* If !new, it is probably echo of our deletion */ break; @@ -775,12 +567,11 @@ krt_got_route_async(struct krt_proto *p, rte *e, int new) case KRT_SRC_ALIEN: if (KRT_CF->learn) { - krt_learn_async(p, e, new); + krt_learn_rte(p, e); return; } #endif } - rte_free(e); } /* @@ -886,30 +677,15 @@ krt_scan_timer_kick(struct krt_proto *p) * Updates */ -static void -krt_make_tmp_attrs(struct rte *rt, struct linpool *pool) -{ - rte_init_tmp_attrs(rt, pool, 2); - rte_make_tmp_attr(rt, EA_KRT_SOURCE, EAF_TYPE_INT, rt->u.krt.proto); - rte_make_tmp_attr(rt, EA_KRT_METRIC, EAF_TYPE_INT, rt->u.krt.metric); -} - -static void -krt_store_tmp_attrs(struct rte *rt, struct linpool *pool) -{ - rte_init_tmp_attrs(rt, pool, 2); - rt->u.krt.proto = rte_store_tmp_attr(rt, EA_KRT_SOURCE); - rt->u.krt.metric = rte_store_tmp_attr(rt, EA_KRT_METRIC); -} - static int -krt_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) +krt_preexport(struct channel *c, rte *e) { - // struct krt_proto *p = (struct krt_proto *) P; - rte *e = *new; - - if (e->attrs->src->proto == P) + if (e->src->owner == &c->proto->sources) +#ifdef CONFIG_SINGLE_ROUTE + return 1; /* Passing the route directly for rt_notify() to ignore */ +#else return -1; +#endif if (!krt_capable(e)) return -1; @@ -918,8 +694,8 @@ krt_preexport(struct proto *P, rte **new, struct linpool *pool UNUSED) } static void -krt_rt_notify(struct proto *P, struct channel *ch UNUSED, net *net, - rte *new, rte *old) +krt_rt_notify(struct proto *P, struct channel *ch UNUSED, const net_addr *net, + rte *new, const rte *old) { struct krt_proto *p = (struct krt_proto *) P; @@ -928,13 +704,10 @@ krt_rt_notify(struct proto *P, struct channel *ch UNUSED, net *net, #ifdef CONFIG_SINGLE_ROUTE /* - * Implicit withdraw - when the imported kernel route becomes the best one, - * we know that the previous one exported to the kernel was already removed, - * but if we processed the update as usual, we would send withdraw to the - * kernel, which would remove the new imported route instead. + * When the imported kernel route becomes the best one, we get it directly and + * we simply know that it is already there. Nothing to do. */ - rte *best = net->routes; - if (!new && best && (best->attrs->src->proto == P)) + if (new->src->owner == &P->sources) return; #endif @@ -983,14 +756,6 @@ krt_feed_end(struct channel *C) } -static int -krt_rte_same(rte *a, rte *b) -{ - /* src is always KRT_SRC_ALIEN and type is irrelevant */ - return (a->u.krt.proto == b->u.krt.proto) && (a->u.krt.metric == b->u.krt.metric); -} - - /* * Protocol glue */ @@ -1049,9 +814,7 @@ krt_init(struct proto_config *CF) p->p.if_notify = krt_if_notify; p->p.reload_routes = krt_reload_routes; p->p.feed_end = krt_feed_end; - p->p.make_tmp_attrs = krt_make_tmp_attrs; - p->p.store_tmp_attrs = krt_store_tmp_attrs; - p->p.rte_same = krt_rte_same; + p->p.rte_better = krt_rte_better; krt_sys_init(p); return &p->p; @@ -1207,8 +970,4 @@ struct protocol proto_unix_kernel = { .reconfigure = krt_reconfigure, .copy_config = krt_copy_config, .get_attr = krt_get_attr, -#ifdef KRT_ALLOW_LEARN - .dump = krt_dump, - .dump_attrs = krt_dump_attrs, -#endif }; diff --git a/sysdep/unix/krt.h b/sysdep/unix/krt.h index 62228f08..968c5b16 100644 --- a/sysdep/unix/krt.h +++ b/sysdep/unix/krt.h @@ -24,6 +24,9 @@ struct kif_proto; #define EA_KRT_SOURCE EA_CODE(PROTOCOL_KERNEL, 0) #define EA_KRT_METRIC EA_CODE(PROTOCOL_KERNEL, 1) +#define KRT_REF_SEEN 0x1 /* Seen in table */ +#define KRT_REF_BEST 0x2 /* Best in table */ + /* Whenever we recognize our own routes, we allow learing of foreign routes */ #ifdef CONFIG_SELF_CONSCIOUS @@ -48,10 +51,6 @@ struct krt_proto { struct proto p; struct krt_state sys; /* Sysdep state */ -#ifdef KRT_ALLOW_LEARN - struct rtable *krt_table; /* Internal table of inherited routes */ -#endif - #ifndef CONFIG_ALL_TABLES_AT_ONCE timer *scan_timer; #endif @@ -63,6 +62,7 @@ struct krt_proto { byte ready; /* Initial feed has been finished */ byte initialized; /* First scan has been finished */ byte reload; /* Next scan is doing reload */ + byte flush_routes; /* Scanning to flush */ }; extern pool *krt_pool; @@ -76,8 +76,8 @@ extern pool *krt_pool; struct proto_config * kif_init_config(int class); void kif_request_scan(void); -void krt_got_route(struct krt_proto *p, struct rte *e); -void krt_got_route_async(struct krt_proto *p, struct rte *e, int new); +void krt_got_route(struct krt_proto *p, struct rte *e, s8 src); +void krt_got_route_async(struct krt_proto *p, struct rte *e, int new, s8 src); static inline int krt_get_sync_error(struct krt_proto *p, struct rte *e) @@ -140,7 +140,7 @@ void krt_sys_copy_config(struct krt_config *, struct krt_config *); int krt_capable(rte *e); void krt_do_scan(struct krt_proto *); -void krt_replace_rte(struct krt_proto *p, net *n, rte *new, rte *old); +void krt_replace_rte(struct krt_proto *p, const net_addr *n, rte *new, const rte *old); int krt_sys_get_attr(const eattr *a, byte *buf, int buflen); diff --git a/sysdep/unix/log.c b/sysdep/unix/log.c index 14d18c01..68a04e78 100644 --- a/sysdep/unix/log.c +++ b/sysdep/unix/log.c @@ -15,6 +15,7 @@ * user's manual. */ +#include <stdatomic.h> #include <stdio.h> #include <stdlib.h> #include <stdarg.h> @@ -35,8 +36,10 @@ static FILE *dbgf; static list *current_log_list; static char *current_syslog_name; /* NULL -> syslog closed */ +static _Atomic uint max_coro_id = ATOMIC_VAR_INIT(1); +static _Thread_local uint this_coro_id; -#ifdef USE_PTHREADS +#define THIS_CORO_ID (this_coro_id ?: (this_coro_id = atomic_fetch_add_explicit(&max_coro_id, 1, memory_order_acq_rel))) #include <pthread.h> @@ -48,15 +51,6 @@ static pthread_t main_thread; void main_thread_init(void) { main_thread = pthread_self(); } static int main_thread_self(void) { return pthread_equal(pthread_self(), main_thread); } -#else - -static inline void log_lock(void) { } -static inline void log_unlock(void) { } -void main_thread_init(void) { } -static int main_thread_self(void) { return 1; } - -#endif - #ifdef HAVE_SYSLOG_H #include <sys/syslog.h> @@ -189,7 +183,7 @@ log_commit(int class, buffer *buf) l->pos += msg_len; } - fprintf(l->fh, "%s <%s> ", tbuf, class_names[class]); + fprintf(l->fh, "%s [%04x] <%s> ", tbuf, THIS_CORO_ID, class_names[class]); } fputs(buf->start, l->fh); fputc('\n', l->fh); @@ -299,6 +293,8 @@ die(const char *msg, ...) exit(1); } +static struct timespec dbg_time_start; + /** * debug - write to debug output * @msg: a printf-like message @@ -309,22 +305,36 @@ die(const char *msg, ...) void debug(const char *msg, ...) { -#define MAX_DEBUG_BUFSIZE 65536 +#define MAX_DEBUG_BUFSIZE 16384 va_list args; - static uint bufsize = 4096; - static char *buf = NULL; - - if (!buf) - buf = mb_alloc(&root_pool, bufsize); + char buf[MAX_DEBUG_BUFSIZE], *pos = buf; + int max = MAX_DEBUG_BUFSIZE; va_start(args, msg); if (dbgf) { - while (bvsnprintf(buf, bufsize, msg, args) < 0) - if (bufsize >= MAX_DEBUG_BUFSIZE) - bug("Extremely long debug output, split it."); - else - buf = mb_realloc(buf, (bufsize *= 2)); + struct timespec dbg_time; + clock_gettime(CLOCK_MONOTONIC, &dbg_time); + uint nsec; + uint sec; + + if (dbg_time.tv_nsec > dbg_time_start.tv_nsec) + { + nsec = dbg_time.tv_nsec - dbg_time_start.tv_nsec; + sec = dbg_time.tv_sec - dbg_time_start.tv_sec; + } + else + { + nsec = 1000000000 + dbg_time.tv_nsec - dbg_time_start.tv_nsec; + sec = dbg_time.tv_sec - dbg_time_start.tv_sec - 1; + } + + int n = bsnprintf(pos, max, "%u.%09u: [%04x] ", sec, nsec, THIS_CORO_ID); + pos += n; + max -= n; + + if (bvsnprintf(pos, max, msg, args) < 0) + bug("Extremely long debug output, split it."); fputs(buf, dbgf); } @@ -378,6 +388,21 @@ default_log_list(int initial, const char **syslog_name) } void +log_cleanup(int syslog) +{ + struct log_config *l; + + if (current_log_list) + WALK_LIST(l, *current_log_list) + if (l->rf) + log_close(l); + + if (syslog && current_syslog_name) + closelog(); +} + + +void log_switch(int initial, list *logs, const char *new_syslog_name) { struct log_config *l; @@ -389,10 +414,7 @@ log_switch(int initial, list *logs, const char *new_syslog_name) logs = default_log_list(initial, &new_syslog_name); /* Close the logs to avoid pinning them on disk when deleted */ - if (current_log_list) - WALK_LIST(l, *current_log_list) - if (l->rf) - log_close(l); + log_cleanup(0); /* Reopen the logs, needed for 'configure undo' */ if (logs) @@ -429,6 +451,8 @@ done: void log_init_debug(char *f) { + clock_gettime(CLOCK_MONOTONIC, &dbg_time_start); + if (dbgf && dbgf != stderr) fclose(dbgf); if (!f) diff --git a/sysdep/unix/main.c b/sysdep/unix/main.c index 392aff9d..c326ba2b 100644 --- a/sysdep/unix/main.c +++ b/sysdep/unix/main.c @@ -28,6 +28,7 @@ #include "lib/resource.h" #include "lib/socket.h" #include "lib/event.h" +#include "lib/locking.h" #include "lib/timer.h" #include "lib/string.h" #include "nest/route.h" @@ -51,7 +52,7 @@ async_dump(void) { debug("INTERNAL STATE DUMP\n\n"); - rdump(&root_pool); + rp_dump(&root_pool); sk_dump_all(); // XXXX tm_dump_all(); if_dump_all(); @@ -201,7 +202,9 @@ sysdep_preconfig(struct config *c) int sysdep_commit(struct config *new, struct config *old UNUSED) { - log_switch(0, &new->logfiles, new->syslog_name); + if (!new->shutdown) + log_switch(0, &new->logfiles, new->syslog_name); + return 0; } @@ -479,6 +482,14 @@ cli_err(sock *s, int err) cli_free(s->data); } +static void +cli_connect_err(sock *s UNUSED, int err) +{ + ASSERT_DIE(err); + if (config->cli_debug) + log(L_INFO "Failed to accept CLI connection: %s", strerror(err)); +} + static int cli_connect(sock *s, uint size UNUSED) { @@ -507,6 +518,7 @@ cli_init_unix(uid_t use_uid, gid_t use_gid) s = cli_sk = sk_new(cli_pool); s->type = SK_UNIX_PASSIVE; s->rx_hook = cli_connect; + s->err_hook = cli_connect_err; s->rbsize = 1024; s->fast_rx = 1; @@ -603,6 +615,7 @@ sysdep_shutdown_done(void) unlink_pid_file(); unlink(path_control_socket); log_msg(L_FATAL "Shutdown completed"); + log_cleanup(1); exit(0); } @@ -673,7 +686,7 @@ signal_init(void) * Parsing of command-line arguments */ -static char *opt_list = "bc:dD:ps:P:u:g:flRh"; +static char *opt_list = "c:dD:ps:P:u:g:flRh"; int parse_and_exit; char *bird_name; static char *use_user; @@ -852,6 +865,8 @@ parse_args(int argc, char **argv) } } +void resource_sys_init(void); + /* * Hic Est main() */ @@ -864,13 +879,17 @@ main(int argc, char **argv) dmalloc_debug(0x2f03d00); #endif + times_update(); + resource_sys_init(); parse_args(argc, argv); log_switch(1, NULL, NULL); + the_bird_lock(); + random_init(); net_init(); resource_init(); - timer_init(); + birdloop_init(); olock_init(); io_init(); rt_init(); @@ -920,6 +939,7 @@ main(int argc, char **argv) dup2(0, 2); } + main_thread_init(); write_pid_file(); diff --git a/sysdep/unix/unix.h b/sysdep/unix/unix.h index ad85d1ea..51ec1b1e 100644 --- a/sysdep/unix/unix.h +++ b/sysdep/unix/unix.h @@ -122,6 +122,7 @@ void krt_io_init(void); void main_thread_init(void); void log_init_debug(char *); /* Initialize debug dump to given file (NULL=stderr, ""=off) */ void log_switch(int initial, list *l, const char *); +void log_cleanup(int syslog); struct log_config { node n; diff --git a/test/birdtest.c b/test/birdtest.c index a1da078f..c6a09684 100644 --- a/test/birdtest.c +++ b/test/birdtest.c @@ -58,11 +58,14 @@ u64 bt_random_state[] = { 0x53d9772877c1b647, 0xab8ce3eb466de6c5, 0xad02844c8a8e865f, 0xe8cc78080295065d }; +void resource_sys_init(void); + void bt_init(int argc, char *argv[]) { int c; + resource_sys_init(); initstate(BT_RANDOM_SEED, (char *) bt_random_state, sizeof(bt_random_state)); bt_verbose = 0; @@ -240,7 +243,7 @@ bt_log_result(int result, u64 time, const char *fmt, va_list argptr) printf("%s\n", result_str); if (do_die && !result) - abort(); + exit(1); } static u64 diff --git a/test/bt-utils.c b/test/bt-utils.c index cbca3a6b..98aaed3d 100644 --- a/test/bt-utils.c +++ b/test/bt-utils.c @@ -53,16 +53,20 @@ cf_file_read(byte *dest, uint max_len, int fd) return l; } +void resource_sys_init(void); + void bt_bird_init(void) { + resource_sys_init(); if(bt_verbose) log_init_debug(""); log_switch(bt_verbose != 0, NULL, NULL); + the_bird_lock(); resource_init(); + birdloop_init(); olock_init(); - timer_init(); io_init(); rt_init(); if_init(); @@ -79,6 +83,7 @@ void bt_bird_cleanup(void) class_to_protocol[i] = NULL; config = new_config = NULL; + the_bird_unlock(); } static char * |
