summaryrefslogtreecommitdiff
path: root/sysdep/unix/alloc.c
diff options
context:
space:
mode:
Diffstat (limited to 'sysdep/unix/alloc.c')
-rw-r--r--sysdep/unix/alloc.c239
1 files changed, 179 insertions, 60 deletions
diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c
index 5dd70c99..c09a8356 100644
--- a/sysdep/unix/alloc.c
+++ b/sysdep/unix/alloc.c
@@ -11,77 +11,73 @@
#include "lib/lists.h"
#include "lib/event.h"
+#include "sysdep/unix/io-loop.h"
+
#include <stdlib.h>
#include <unistd.h>
+#include <stdatomic.h>
+#include <errno.h>
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
-#ifdef HAVE_MMAP
-#define KEEP_PAGES 512
+long page_size = 0;
-static u64 page_size = 0;
+#ifdef HAVE_MMAP
+#if DEBUGGING
+#define FP_NODE_OFFSET 42
+#else
+#define FP_NODE_OFFSET 1
+#endif
static _Bool use_fake = 0;
-
-uint pages_kept = 0;
-static list pages_list;
-
-static void cleanup_pages(void *data);
-static event page_cleanup_event = { .hook = cleanup_pages };
-
#else
-static const u64 page_size = 4096; /* Fake page size */
+static _Bool use_fake = 1;
#endif
-u64 get_page_size(void)
+static void *
+alloc_sys_page(void)
{
- if (page_size)
- return page_size;
+ void *ptr = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-#ifdef HAVE_MMAP
- if (page_size = sysconf(_SC_PAGESIZE))
- {
- if ((u64_popcount(page_size) > 1) || (page_size > 16384))
- {
- /* Too big or strange page, use the aligned allocator instead */
- page_size = 4096;
- use_fake = 1;
- }
- return page_size;
- }
+ if (ptr == MAP_FAILED)
+ bug("mmap(%lu) failed: %m", page_size);
- bug("Page size must be non-zero");
-#endif
+ return ptr;
}
+extern int shutting_down; /* Shutdown requested. */
+
void *
alloc_page(void)
{
#ifdef HAVE_MMAP
- if (pages_kept)
- {
- node *page = TAIL(pages_list);
- rem_node(page);
- pages_kept--;
- memset(page, 0, get_page_size());
- return page;
- }
-
if (!use_fake)
{
- void *ret = mmap(NULL, get_page_size(), PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (ret == MAP_FAILED)
- bug("mmap(%lu) failed: %m", page_size);
- return ret;
+ struct free_pages *fp = &birdloop_current->pages;
+ if (!fp->cnt)
+ return alloc_sys_page();
+
+ node *n = HEAD(fp->list);
+ rem_node(n);
+ if ((--fp->cnt < fp->min) && !shutting_down)
+ ev_send(fp->cleanup->list, fp->cleanup);
+
+ void *ptr = n - FP_NODE_OFFSET;
+ memset(ptr, 0, page_size);
+ return ptr;
}
else
#endif
{
+#ifdef HAVE_ALIGNED_ALLOC
void *ret = aligned_alloc(page_size, page_size);
if (!ret)
bug("aligned_alloc(%lu) failed", page_size);
return ret;
+#else
+ bug("BIRD should have already died on fatal error.");
+#endif
}
}
@@ -91,14 +87,14 @@ free_page(void *ptr)
#ifdef HAVE_MMAP
if (!use_fake)
{
- if (!pages_kept)
- init_list(&pages_list);
-
- memset(ptr, 0, sizeof(node));
- add_tail(&pages_list, ptr);
-
- if (++pages_kept > KEEP_PAGES)
- ev_schedule(&page_cleanup_event);
+ struct free_pages *fp = &birdloop_current->pages;
+ struct node *n = ptr;
+ n += FP_NODE_OFFSET;
+
+ memset(n, 0, sizeof(node));
+ add_tail(&fp->list, n);
+ if ((++fp->cnt > fp->max) && !shutting_down)
+ ev_send(fp->cleanup->list, fp->cleanup);
}
else
#endif
@@ -106,24 +102,147 @@ free_page(void *ptr)
}
#ifdef HAVE_MMAP
+
+#define GFP (&main_birdloop.pages)
+
+void
+flush_pages(struct birdloop *loop)
+{
+ ASSERT_DIE(birdloop_inside(loop->parent->loop));
+
+ struct free_pages *fp = &loop->pages;
+ struct free_pages *pfp = &loop->parent->loop->pages;
+
+ add_tail_list(&pfp->list, &fp->list);
+ pfp->cnt += fp->cnt;
+
+ fp->cnt = 0;
+ fp->list = (list) {};
+ fp->min = 0;
+ fp->max = 0;
+
+ rfree(fp->cleanup);
+ fp->cleanup = NULL;
+}
+
+static void
+cleanup_pages(void *data)
+{
+ struct birdloop *loop = data;
+ birdloop_enter(loop);
+
+ ASSERT_DIE(birdloop_inside(loop->parent->loop));
+
+ struct free_pages *fp = &loop->pages;
+ struct free_pages *pfp = &loop->parent->loop->pages;
+
+ while ((fp->cnt < fp->min) && (pfp->cnt > pfp->min))
+ {
+ node *n = HEAD(pfp->list);
+ rem_node(n);
+ add_tail(&fp->list, n);
+ fp->cnt++;
+ pfp->cnt--;
+ }
+
+ while (fp->cnt < fp->min)
+ {
+ node *n = alloc_sys_page();
+ add_tail(&fp->list, n + FP_NODE_OFFSET);
+ fp->cnt++;
+ }
+
+ while (fp->cnt > fp->max)
+ {
+ node *n = HEAD(fp->list);
+ rem_node(n);
+ add_tail(&pfp->list, n);
+ fp->cnt--;
+ pfp->cnt++;
+ }
+
+ birdloop_leave(loop);
+
+ if (!shutting_down && (pfp->cnt > pfp->max))
+ ev_send(pfp->cleanup->list, pfp->cleanup);
+}
+
static void
-cleanup_pages(void *data UNUSED)
+cleanup_global_pages(void *data UNUSED)
{
- for (uint seen = 0; (pages_kept > KEEP_PAGES) && (seen < KEEP_PAGES); seen++)
+ while (GFP->cnt < GFP->max)
{
- void *ptr = HEAD(pages_list);
- rem_node(ptr);
- if (munmap(ptr, get_page_size()) == 0)
- pages_kept--;
-#ifdef ENOMEM
+ node *n = alloc_sys_page();
+ add_tail(&GFP->list, n + FP_NODE_OFFSET);
+ GFP->cnt++;
+ }
+
+ for (uint limit = GFP->cnt; (limit > 0) && (GFP->cnt > GFP->max); limit--)
+ {
+ node *n = TAIL(GFP->list);
+ rem_node(n);
+
+ if (munmap(n - FP_NODE_OFFSET, page_size) == 0)
+ GFP->cnt--;
else if (errno == ENOMEM)
- add_tail(&pages_list, ptr);
-#endif
+ add_head(&GFP->list, n);
else
- bug("munmap(%p) failed: %m", ptr);
+ bug("munmap(%p) failed: %m", n - FP_NODE_OFFSET);
+ }
+}
+
+void
+init_pages(struct birdloop *loop)
+{
+ struct free_pages *fp = &loop->pages;
+
+ init_list(&fp->list);
+ fp->cleanup = ev_new_init(loop->parent->loop->pool, cleanup_pages, loop);
+ fp->cleanup->list = (loop->parent->loop == &main_birdloop) ? &global_work_list : birdloop_event_list(loop->parent->loop);
+ fp->min = 4;
+ fp->max = 16;
+
+ for (fp->cnt = 0; fp->cnt < fp->min; fp->cnt++)
+ {
+ node *n = alloc_sys_page();
+ add_tail(&fp->list, n + FP_NODE_OFFSET);
+ }
+}
+
+static event global_free_pages_cleanup_event = { .hook = cleanup_global_pages, .list = &global_work_list };
+
+void resource_sys_init(void)
+{
+ if (!(page_size = sysconf(_SC_PAGESIZE)))
+ die("System page size must be non-zero");
+
+ if (u64_popcount(page_size) == 1)
+ {
+ init_list(&GFP->list);
+ GFP->cleanup = &global_free_pages_cleanup_event;
+ GFP->min = 0;
+ GFP->max = 256;
+ return;
}
- if (pages_kept > KEEP_PAGES)
- ev_schedule(&page_cleanup_event);
+#ifdef HAVE_ALIGNED_ALLOC
+ log(L_WARN "Got strange memory page size (%lu), using the aligned allocator instead", page_size);
+#else
+ die("Got strange memory page size (%lu) and aligned_alloc is not available", page_size);
+#endif
+
+ /* Too big or strange page, use the aligned allocator instead */
+ page_size = 4096;
+ use_fake = 1;
}
+
+#else
+
+void
+resource_sys_init(void)
+{
+ page_size = 4096;
+ use_fake = 1;
+}
+
#endif
generated by cgit v1.2.3 (git 2.39.1) at 2025-02-03 23:43:53 +0000