From 644e9ca94e2d10ba0c2de45f94523da2414328e3 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Wed, 24 Nov 2021 17:30:13 +0100 Subject: Directly mapped pages are kept for future use if temporarily not needed --- sysdep/unix/alloc.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) (limited to 'sysdep/unix/alloc.c') diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index c525f713..5dd70c99 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -8,6 +8,8 @@ #include "nest/bird.h" #include "lib/resource.h" +#include "lib/lists.h" +#include "lib/event.h" #include #include @@ -17,8 +19,17 @@ #endif #ifdef HAVE_MMAP +#define KEEP_PAGES 512 + static u64 page_size = 0; static _Bool use_fake = 0; + +uint pages_kept = 0; +static list pages_list; + +static void cleanup_pages(void *data); +static event page_cleanup_event = { .hook = cleanup_pages }; + #else static const u64 page_size = 4096; /* Fake page size */ #endif @@ -48,6 +59,15 @@ void * alloc_page(void) { #ifdef HAVE_MMAP + if (pages_kept) + { + node *page = TAIL(pages_list); + rem_node(page); + pages_kept--; + memset(page, 0, get_page_size()); + return page; + } + if (!use_fake) { void *ret = mmap(NULL, get_page_size(), PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -71,10 +91,39 @@ free_page(void *ptr) #ifdef HAVE_MMAP if (!use_fake) { - if (munmap(ptr, get_page_size()) < 0) - bug("munmap(%p) failed: %m", ptr); + if (!pages_kept) + init_list(&pages_list); + + memset(ptr, 0, sizeof(node)); + add_tail(&pages_list, ptr); + + if (++pages_kept > KEEP_PAGES) + ev_schedule(&page_cleanup_event); } else #endif free(ptr); } + +#ifdef HAVE_MMAP +static void +cleanup_pages(void *data UNUSED) +{ + for (uint seen = 0; (pages_kept > KEEP_PAGES) && (seen < KEEP_PAGES); seen++) + { + void *ptr = HEAD(pages_list); + rem_node(ptr); + if (munmap(ptr, get_page_size()) == 0) + pages_kept--; +#ifdef ENOMEM + else if (errno == ENOMEM) + add_tail(&pages_list, ptr); +#endif + else + bug("munmap(%p) failed: %m", ptr); + } + + if (pages_kept > KEEP_PAGES) + ev_schedule(&page_cleanup_event); +} +#endif -- cgit v1.2.3 From 2fc8b4c4bac576427b0054a13cc78e395b93d6c4 Mon Sep 17 00:00:00 2001 From: "Ondrej Zajicek (work)" Date: Tue, 8 Feb 2022 22:42:00 +0100 Subject: Alloc: Use posix_memalign() instead of aligned_alloc() For compatibility with older systems use posix_memalign(). We can switch to aligned_alloc() when we commit to C11 for multithreading. --- sysdep/unix/alloc.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'sysdep/unix/alloc.c') diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index 5dd70c99..0e944d57 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -72,16 +72,17 @@ alloc_page(void) { void *ret = mmap(NULL, get_page_size(), PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (ret == MAP_FAILED) - bug("mmap(%lu) failed: %m", page_size); + bug("mmap(%lu) failed: %m", (long unsigned int) page_size); return ret; } else #endif { - void *ret = aligned_alloc(page_size, page_size); - if (!ret) - bug("aligned_alloc(%lu) failed", page_size); - return ret; + void *ptr = NULL; + int err = posix_memalign(&ptr, page_size, page_size); + if (err || !ptr) + bug("posix_memalign(%lu) failed", (long unsigned int) page_size); + return ptr; } } -- cgit v1.2.3 From c78247f9b97802d6fe76d9b7cddf2c1940134d33 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Wed, 9 Mar 2022 09:10:44 +0100 Subject: Single-threaded version of sark-branch memory page management --- lib/resource.c | 2 + lib/resource.h | 5 +- lib/slab.c | 13 ++-- nest/cmds.c | 5 +- sysdep/unix/alloc.c | 191 +++++++++++++++++++++++++++++++++------------------- 5 files changed, 137 insertions(+), 79 deletions(-) (limited to 'sysdep/unix/alloc.c') diff --git a/lib/resource.c b/lib/resource.c index 5636872c..a179afe3 100644 --- a/lib/resource.c +++ b/lib/resource.c @@ -270,6 +270,8 @@ rlookup(unsigned long a) void resource_init(void) { + resource_sys_init(); + root_pool.r.class = &pool_class; root_pool.name = "Root"; init_list(&root_pool.inside); diff --git a/lib/resource.h b/lib/resource.h index 0e4c44d8..992fa5b2 100644 --- a/lib/resource.h +++ b/lib/resource.h @@ -110,10 +110,11 @@ void sl_free(slab *, void *); void buffer_realloc(void **buf, unsigned *size, unsigned need, unsigned item_size); /* Allocator of whole pages; for use in slabs and other high-level allocators. */ -u64 get_page_size(void); +extern long page_size; void *alloc_page(void); void free_page(void *); -extern uint pages_kept; + +void resource_sys_init(void); #ifdef HAVE_LIBDMALLOC /* diff --git a/lib/slab.c b/lib/slab.c index 9a4c3ee2..9be9844d 100644 --- a/lib/slab.c +++ b/lib/slab.c @@ -180,7 +180,7 @@ struct sl_alignment { /* Magic structure for testing of alignment */ int x[0]; }; -#define SL_GET_HEAD(x) ((struct sl_head *) (((uintptr_t) (x)) & ~(get_page_size()-1))) +#define SL_GET_HEAD(x) ((struct sl_head *) (((uintptr_t) (x)) & ~(page_size-1))) /** * sl_new - create a new Slab @@ -202,7 +202,6 @@ sl_new(pool *p, uint size) s->obj_size = size; s->head_size = sizeof(struct sl_head); - u64 page_size = get_page_size(); do { s->objs_per_slab = (page_size - s->head_size) / size; @@ -273,7 +272,7 @@ no_partial: } h = alloc_page(); #ifdef POISON - memset(h, 0xba, get_page_size()); + memset(h, 0xba, page_size); #endif ASSERT_DIE(SL_GET_HEAD(h) == h); memset(h, 0, s->head_size); @@ -332,7 +331,7 @@ sl_free(slab *s, void *oo) if (s->num_empty_heads >= MAX_EMPTY_HEADS) { #ifdef POISON - memset(h, 0xde, get_page_size()); + memset(h, 0xde, page_size); #endif free_page(h); } @@ -399,7 +398,7 @@ slab_memsize(resource *r) return (struct resmem) { .effective = eff, - .overhead = ALLOC_OVERHEAD + sizeof(struct slab) + heads * get_page_size() - eff, + .overhead = ALLOC_OVERHEAD + sizeof(struct slab) + heads * page_size - eff, }; } @@ -410,10 +409,10 @@ slab_lookup(resource *r, unsigned long a) struct sl_head *h; WALK_LIST(h, s->partial_heads) - if ((unsigned long) h < a && (unsigned long) h + get_page_size() < a) + if ((unsigned long) h < a && (unsigned long) h + page_size < a) return r; WALK_LIST(h, s->full_heads) - if ((unsigned long) h < a && (unsigned long) h + get_page_size() < a) + if ((unsigned long) h < a && (unsigned long) h + page_size < a) return r; return NULL; } diff --git a/nest/cmds.c b/nest/cmds.c index 1a16f9c7..8481bf96 100644 --- a/nest/cmds.c +++ b/nest/cmds.c @@ -108,6 +108,7 @@ print_size(char *dsc, struct resmem vals) extern pool *rt_table_pool; extern pool *rta_pool; +extern uint *pages_kept; void cmd_show_memory(void) @@ -119,8 +120,8 @@ cmd_show_memory(void) print_size("Protocols:", rmemsize(proto_pool)); struct resmem total = rmemsize(&root_pool); #ifdef HAVE_MMAP - print_size("Standby memory:", (struct resmem) { .overhead = get_page_size() * pages_kept }); - total.overhead += get_page_size() * pages_kept; + print_size("Standby memory:", (struct resmem) { .overhead = page_size * *pages_kept }); + total.overhead += page_size * *pages_kept; #endif print_size("Total:", total); cli_msg(0, ""); diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index 0e944d57..f96c0fcf 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -11,6 +11,7 @@ #include "lib/lists.h" #include "lib/event.h" +#include #include #include @@ -18,113 +19,167 @@ #include #endif -#ifdef HAVE_MMAP -#define KEEP_PAGES 512 +long page_size = 0; -static u64 page_size = 0; -static _Bool use_fake = 0; +#ifdef HAVE_MMAP +#define KEEP_PAGES_MAIN_MAX 256 +#define KEEP_PAGES_MAIN_MIN 8 +#define CLEANUP_PAGES_BULK 256 -uint pages_kept = 0; -static list pages_list; +_Static_assert(KEEP_PAGES_MAIN_MIN * 4 < KEEP_PAGES_MAIN_MAX); -static void cleanup_pages(void *data); -static event page_cleanup_event = { .hook = cleanup_pages }; +static _Bool use_fake = 0; +#if DEBUGGING +struct free_page { + node unused[42]; + node n; +}; #else -static const u64 page_size = 4096; /* Fake page size */ +struct free_page { + node n; +}; #endif -u64 get_page_size(void) +struct free_pages { + list pages; + u16 min, max; /* Minimal and maximal number of free pages kept */ + uint cnt; /* Number of empty pages */ + event cleanup; +}; + +static void global_free_pages_cleanup_event(void *); + +static struct free_pages global_free_pages = { + .min = KEEP_PAGES_MAIN_MIN, + .max = KEEP_PAGES_MAIN_MAX, + .cleanup = { .hook = global_free_pages_cleanup_event }, +}; + +uint *pages_kept = &global_free_pages.cnt; + +static void * +alloc_sys_page(void) { - if (page_size) - return page_size; + void *ptr = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -#ifdef HAVE_MMAP - if (page_size = sysconf(_SC_PAGESIZE)) - { - if ((u64_popcount(page_size) > 1) || (page_size > 16384)) - { - /* Too big or strange page, use the aligned allocator instead */ - page_size = 4096; - use_fake = 1; - } - return page_size; - } + if (ptr == MAP_FAILED) + bug("mmap(%lu) failed: %m", page_size); - bug("Page size must be non-zero"); -#endif + return ptr; } +extern int shutting_down; /* Shutdown requested. */ + +#else // ! HAVE_MMAP +#define use_fake 1 +#endif + void * alloc_page(void) { -#ifdef HAVE_MMAP - if (pages_kept) - { - node *page = TAIL(pages_list); - rem_node(page); - pages_kept--; - memset(page, 0, get_page_size()); - return page; - } - - if (!use_fake) - { - void *ret = mmap(NULL, get_page_size(), PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ret == MAP_FAILED) - bug("mmap(%lu) failed: %m", (long unsigned int) page_size); - return ret; - } - else -#endif + if (use_fake) { void *ptr = NULL; int err = posix_memalign(&ptr, page_size, page_size); + if (err || !ptr) bug("posix_memalign(%lu) failed", (long unsigned int) page_size); + return ptr; } + +#ifdef HAVE_MMAP + struct free_pages *fps = &global_free_pages; + + if (fps->cnt) + { + struct free_page *fp = SKIP_BACK(struct free_page, n, HEAD(fps->pages)); + rem_node(&fp->n); + if ((--fps->cnt < fps->min) && !shutting_down) + ev_schedule(&fps->cleanup); + + bzero(fp, page_size); + return fp; + } + + return alloc_sys_page(); +#endif } void free_page(void *ptr) { -#ifdef HAVE_MMAP - if (!use_fake) + if (use_fake) { - if (!pages_kept) - init_list(&pages_list); + free(ptr); + return; + } + +#ifdef HAVE_MMAP + struct free_pages *fps = &global_free_pages; + struct free_page *fp = ptr; - memset(ptr, 0, sizeof(node)); - add_tail(&pages_list, ptr); + fp->n = (node) {}; + add_tail(&fps->pages, &fp->n); - if (++pages_kept > KEEP_PAGES) - ev_schedule(&page_cleanup_event); - } - else + if ((++fps->cnt > fps->max) && !shutting_down) + ev_schedule(&fps->cleanup); #endif - free(ptr); } #ifdef HAVE_MMAP static void -cleanup_pages(void *data UNUSED) +global_free_pages_cleanup_event(void *data UNUSED) { - for (uint seen = 0; (pages_kept > KEEP_PAGES) && (seen < KEEP_PAGES); seen++) + if (shutting_down) + return; + + struct free_pages *fps = &global_free_pages; + + while (fps->cnt / 2 < fps->min) { - void *ptr = HEAD(pages_list); - rem_node(ptr); - if (munmap(ptr, get_page_size()) == 0) - pages_kept--; -#ifdef ENOMEM + struct free_page *fp = alloc_sys_page(); + fp->n = (node) {}; + add_tail(&fps->pages, &fp->n); + fps->cnt++; + } + + for (uint seen = 0; (seen < CLEANUP_PAGES_BULK) && (fps->cnt > fps->max / 2); seen++) + { + struct free_page *fp = SKIP_BACK(struct free_page, n, TAIL(fps->pages)); + rem_node(&fp->n); + + if (munmap(fp, page_size) == 0) + fps->cnt--; else if (errno == ENOMEM) - add_tail(&pages_list, ptr); -#endif + add_head(&fps->pages, &fp->n); else - bug("munmap(%p) failed: %m", ptr); + bug("munmap(%p) failed: %m", fp); } - - if (pages_kept > KEEP_PAGES) - ev_schedule(&page_cleanup_event); } #endif + +void +resource_sys_init(void) +{ +#ifdef HAVE_MMAP + if (!(page_size = sysconf(_SC_PAGESIZE))) + die("System page size must be non-zero"); + + if (u64_popcount(page_size) == 1) + { + struct free_pages *fps = &global_free_pages; + + init_list(&fps->pages); + global_free_pages_cleanup_event(NULL); + return; + } + + /* Too big or strange page, use the aligned allocator instead */ + log(L_WARN "Got strange memory page size (%lu), using the aligned allocator instead", page_size); + use_fake = 1; +#endif + + page_size = 4096; +} -- cgit v1.2.3 From 9e60a1fbc3ef9ab93b414dcf451bbe741e2e8827 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Wed, 9 Mar 2022 10:30:03 +0100 Subject: Fixed resource initialization in unit tests --- lib/bitmap_test.c | 3 --- lib/buffer_test.c | 1 - lib/event_test.c | 1 - lib/flowspec_test.c | 3 --- lib/hash_test.c | 1 - nest/a-path_test.c | 8 -------- nest/a-set_test.c | 9 --------- sysdep/unix/alloc.c | 2 ++ test/birdtest.c | 2 ++ 9 files changed, 4 insertions(+), 26 deletions(-) (limited to 'sysdep/unix/alloc.c') diff --git a/lib/bitmap_test.c b/lib/bitmap_test.c index 0595a4d0..07860c94 100644 --- a/lib/bitmap_test.c +++ b/lib/bitmap_test.c @@ -24,7 +24,6 @@ t_bmap_set_clear_random(void) { struct bmap b; - resource_init(); bmap_init(&b, &root_pool, 1024); char expected[MAX_NUM] = {}; @@ -60,7 +59,6 @@ t_hmap_set_clear_random(void) { struct hmap b; - resource_init(); hmap_init(&b, &root_pool, 1024); char expected[MAX_NUM] = {}; @@ -119,7 +117,6 @@ t_hmap_set_clear_fill(void) { struct hmap b; - resource_init(); hmap_init(&b, &root_pool, 1024); char expected[MAX_NUM] = {}; diff --git a/lib/buffer_test.c b/lib/buffer_test.c index 5b7de330..0629e901 100644 --- a/lib/buffer_test.c +++ b/lib/buffer_test.c @@ -41,7 +41,6 @@ fill_expected_array(void) static void init_buffer(void) { - resource_init(); buffer_pool = &root_pool; BUFFER_INIT(buf, buffer_pool, MAX_NUM); } diff --git a/lib/event_test.c b/lib/event_test.c index e1215bba..e1fbea8f 100644 --- a/lib/event_test.c +++ b/lib/event_test.c @@ -53,7 +53,6 @@ t_ev_run_list(void) { int i; - resource_init(); olock_init(); timer_init(); io_init(); diff --git a/lib/flowspec_test.c b/lib/flowspec_test.c index 115084a3..03649b99 100644 --- a/lib/flowspec_test.c +++ b/lib/flowspec_test.c @@ -446,8 +446,6 @@ t_validation6(void) static int t_builder4(void) { - resource_init(); - struct flow_builder *fb = flow_builder_init(&root_pool); /* Expectation */ @@ -528,7 +526,6 @@ t_builder6(void) { net_addr_ip6 ip; - resource_init(); struct flow_builder *fb = flow_builder_init(&root_pool); fb->ipv6 = 1; diff --git a/lib/hash_test.c b/lib/hash_test.c index 59beb7c0..4bce7017 100644 --- a/lib/hash_test.c +++ b/lib/hash_test.c @@ -61,7 +61,6 @@ dump_nodes(void) static void init_hash_(uint order) { - resource_init(); my_pool = rp_new(&root_pool, "Test pool"); HASH_INIT(hash, my_pool, order); diff --git a/nest/a-path_test.c b/nest/a-path_test.c index abd2abbf..e007a450 100644 --- a/nest/a-path_test.c +++ b/nest/a-path_test.c @@ -23,8 +23,6 @@ static int t_as_path_match(void) { - resource_init(); - int round; for (round = 0; round < TESTS_NUM; round++) { @@ -69,8 +67,6 @@ t_as_path_match(void) static int t_path_format(void) { - resource_init(); - struct adata empty_as_path = {}; struct adata *as_path = &empty_as_path; @@ -114,8 +110,6 @@ count_asn_in_array(const u32 *array, u32 asn) static int t_path_include(void) { - resource_init(); - struct adata empty_as_path = {}; struct adata *as_path = &empty_as_path; @@ -158,8 +152,6 @@ t_path_include(void) static int t_as_path_converting(void) { - resource_init(); - struct adata empty_as_path = {}; struct adata *as_path = &empty_as_path; #define AS_PATH_LENGTH_FOR_CONVERTING_TEST 10 diff --git a/nest/a-set_test.c b/nest/a-set_test.c index 669872e3..904e6764 100644 --- a/nest/a-set_test.c +++ b/nest/a-set_test.c @@ -68,7 +68,6 @@ t_set_int_contains(void) { int i; - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE); bt_assert(int_set_get_size(set_sequence) == SET_SIZE); @@ -88,7 +87,6 @@ t_set_int_contains(void) static int t_set_int_union(void) { - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE); const struct adata *set_union; @@ -106,7 +104,6 @@ t_set_int_union(void) static int t_set_int_format(void) { - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE_FOR_FORMAT_OUTPUT); bt_assert(int_set_format(set_sequence, 0, 0, buf, BUFFER_SIZE) == 0); @@ -126,7 +123,6 @@ t_set_int_format(void) static int t_set_int_delete(void) { - resource_init(); generate_set_sequence(SET_TYPE_INT, SET_SIZE); const struct adata *deleting_sequence = set_sequence; @@ -154,7 +150,6 @@ t_set_ec_contains(void) { u32 i; - resource_init(); generate_set_sequence(SET_TYPE_EC, SET_SIZE); bt_assert(ec_set_get_size(set_sequence) == SET_SIZE); @@ -174,7 +169,6 @@ t_set_ec_contains(void) static int t_set_ec_union(void) { - resource_init(); generate_set_sequence(SET_TYPE_EC, SET_SIZE); const struct adata *set_union; @@ -192,8 +186,6 @@ t_set_ec_union(void) static int t_set_ec_format(void) { - resource_init(); - const struct adata empty_as_path = {}; set_sequence = set_sequence_same = set_sequence_higher = set_random = &empty_as_path; @@ -212,7 +204,6 @@ t_set_ec_format(void) static int t_set_ec_delete(void) { - resource_init(); generate_set_sequence(SET_TYPE_EC, SET_SIZE); const struct adata *deleting_sequence = set_sequence; diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index f96c0fcf..755b5fa5 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -164,6 +164,8 @@ void resource_sys_init(void) { #ifdef HAVE_MMAP + ASSERT_DIE(global_free_pages.cnt == 0); + if (!(page_size = sysconf(_SC_PAGESIZE))) die("System page size must be non-zero"); diff --git a/test/birdtest.c b/test/birdtest.c index 10d6d6de..ae05d1a5 100644 --- a/test/birdtest.c +++ b/test/birdtest.c @@ -20,6 +20,7 @@ #include "test/birdtest.h" #include "lib/string.h" +#include "lib/event.h" #ifdef HAVE_EXECINFO_H #include @@ -120,6 +121,7 @@ bt_init(int argc, char *argv[]) bt_suite_case_begin = bt_suite_begin = bt_begin; resource_init(); + ev_init_list(&global_event_list); return; -- cgit v1.2.3 From 36f5fea31a61d9067d81ffb1620fdf174f47d5e4 Mon Sep 17 00:00:00 2001 From: Maria Matejka Date: Wed, 9 Mar 2022 13:27:49 +0100 Subject: Fixed a static assert in page allocator --- sysdep/unix/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sysdep/unix/alloc.c') diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c index 755b5fa5..edad6209 100644 --- a/sysdep/unix/alloc.c +++ b/sysdep/unix/alloc.c @@ -26,7 +26,7 @@ long page_size = 0; #define KEEP_PAGES_MAIN_MIN 8 #define CLEANUP_PAGES_BULK 256 -_Static_assert(KEEP_PAGES_MAIN_MIN * 4 < KEEP_PAGES_MAIN_MAX); +STATIC_ASSERT(KEEP_PAGES_MAIN_MIN * 4 < KEEP_PAGES_MAIN_MAX); static _Bool use_fake = 0; -- cgit v1.2.3