diff options
author | Ondrej Zajicek <santiago@crfreenet.org> | 2013-11-23 11:50:34 +0100 |
---|---|---|
committer | Ondrej Zajicek <santiago@crfreenet.org> | 2013-11-23 11:50:34 +0100 |
commit | 736e143fa50607fcd88132291e96089b899af979 (patch) | |
tree | c0fcd5fb3174bae8a39b3a32dfe582b2ccb6df17 /proto/bfd | |
parent | 094d2bdb79e1ffa0a02761fd651aa0f0b6b0c585 (diff) | |
parent | 2b3d52aa421ae1c31e30107beefd82fddbb42854 (diff) |
Merge branch 'master' into add-path
Conflicts:
filter/filter.c
nest/proto.c
nest/rt-table.c
proto/bgp/bgp.h
proto/bgp/config.Y
Diffstat (limited to 'proto/bfd')
-rw-r--r-- | proto/bfd/Doc | 1 | ||||
-rw-r--r-- | proto/bfd/Makefile | 5 | ||||
-rw-r--r-- | proto/bfd/bfd.c | 1114 | ||||
-rw-r--r-- | proto/bfd/bfd.h | 191 | ||||
-rw-r--r-- | proto/bfd/config.Y | 138 | ||||
-rw-r--r-- | proto/bfd/io.c | 768 | ||||
-rw-r--r-- | proto/bfd/io.h | 99 | ||||
-rw-r--r-- | proto/bfd/packets.c | 248 |
8 files changed, 2564 insertions, 0 deletions
diff --git a/proto/bfd/Doc b/proto/bfd/Doc new file mode 100644 index 00000000..7ee5d3ef --- /dev/null +++ b/proto/bfd/Doc @@ -0,0 +1 @@ +S bfd.c diff --git a/proto/bfd/Makefile b/proto/bfd/Makefile new file mode 100644 index 00000000..c28cedec --- /dev/null +++ b/proto/bfd/Makefile @@ -0,0 +1,5 @@ +source=bfd.c packets.c io.c +root-rel=../../ +dir-name=proto/bfd + +include ../../Rules diff --git a/proto/bfd/bfd.c b/proto/bfd/bfd.c new file mode 100644 index 00000000..5ebfadc1 --- /dev/null +++ b/proto/bfd/bfd.c @@ -0,0 +1,1114 @@ +/* + * BIRD -- Bidirectional Forwarding Detection (BFD) + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +/** + * DOC: Bidirectional Forwarding Detection + * + * The BFD protocol is implemented in three files: |bfd.c| containing the + * protocol logic and the protocol glue with BIRD core, |packets.c| handling BFD + * packet processing, RX, TX and protocol sockets. |io.c| then contains generic + * code for the event loop, threads and event sources (sockets, microsecond + * timers). This generic code will be merged to the main BIRD I/O code in the + * future. + * + * The BFD implementation uses a separate thread with an internal event loop for + * handling the protocol logic, which requires high-res and low-latency timing, + * so it is not affected by the rest of BIRD, which has several low-granularity + * hooks in the main loop, uses second-based timers and cannot offer good + * latency. The core of BFD protocol (the code related to BFD sessions, + * interfaces and packets) runs in the BFD thread, while the rest (the code + * related to BFD requests, BFD neighbors and the protocol glue) runs in the + * main thread. + * + * BFD sessions are represented by structure &bfd_session that contains a state + * related to the session and two timers (TX timer for periodic packets and hold + * timer for session timeout). These sessions are allocated from @session_slab + * and are accessible by two hash tables, @session_hash_id (by session ID) and + * @session_hash_ip (by IP addresses of neighbors). Slab and both hashes are in + * the main protocol structure &bfd_proto. The protocol logic related to BFD + * sessions is implemented in internal functions bfd_session_*(), which are + * expected to be called from the context of BFD thread, and external functions + * bfd_add_session(), bfd_remove_session() and bfd_reconfigure_session(), which + * form an interface to the BFD core for the rest and are expected to be called + * from the context of main thread. + * + * Each BFD session has an associated BFD interface, represented by structure + * &bfd_iface. A BFD interface contains a socket used for TX (the one for RX is + * shared in &bfd_proto), an interface configuration and reference counter. + * Compared to interface structures of other protocols, these structures are not + * created and removed based on interface notification events, but according to + * the needs of BFD sessions. When a new session is created, it requests a + * proper BFD interface by function bfd_get_iface(), which either finds an + * existing one in &iface_list (from &bfd_proto) or allocates a new one. When a + * session is removed, an associated iface is dicharged by bfd_free_iface(). + * + * BFD requests are the external API for the other protocols. When a protocol + * wants a BFD session, it calls bfd_request_session(), which creates a + * structure &bfd_request containing approprite information and an notify hook. + * This structure is a resource associated with the caller's resource pool. When + * a BFD protocol is available, a BFD request is submitted to the protocol, an + * appropriate BFD session is found or created and the request is attached to + * the session. When a session changes state, all attached requests (and related + * protocols) are notified. Note that BFD requests do not depend on BFD protocol + * running. When the BFD protocol is stopped or removed (or not available from + * beginning), related BFD requests are stored in @bfd_wait_list, where waits + * for a new protocol. + * + * BFD neighbors are just a way to statically configure BFD sessions without + * requests from other protocol. Structures &bfd_neighbor are part of BFD + * configuration (like static routes in the static protocol). BFD neighbors are + * handled by BFD protocol like it is a BFD client -- when a BFD neighbor is + * ready, the protocol just creates a BFD request like any other protocol. + * + * The protocol uses a new generic event loop (structure &birdloop) from |io.c|, + * which supports sockets, timers and events like the main loop. Timers + * (structure &timer2) are new microsecond based timers, while sockets and + * events are the same. A birdloop is associated with a thread (field @thread) + * in which event hooks are executed. Most functions for setting event sources + * (like sk_start() or tm2_start()) must be called from the context of that + * thread. Birdloop allows to temporarily acquire the context of that thread for + * the main thread by calling birdloop_enter() and then birdloop_leave(), which + * also ensures mutual exclusion with all event hooks. Note that resources + * associated with a birdloop (like timers) should be attached to the + * independent resource pool, detached from the main resource tree. + * + * There are two kinds of interaction between the BFD core (running in the BFD + * thread) and the rest of BFD (running in the main thread). The first kind are + * configuration calls from main thread to the BFD thread (like bfd_add_session()). + * These calls are synchronous and use birdloop_enter() mechanism for mutual + * exclusion. The second kind is a notification about session changes from the + * BFD thread to the main thread. This is done in an asynchronous way, sesions + * with pending notifications are linked (in the BFD thread) to @notify_list in + * &bfd_proto, and then bfd_notify_hook() in the main thread is activated using + * bfd_notify_kick() and a pipe. The hook then processes scheduled sessions and + * calls hooks from associated BFD requests. This @notify_list (and state fields + * in structure &bfd_session) is protected by a spinlock in &bfd_proto and + * functions bfd_lock_sessions() / bfd_unlock_sessions(). + * + * There are few data races (accessing @p->p.debug from TRACE() from the BFD + * thread and accessing some some private fields of %bfd_session from + * bfd_show_sessions() from the main thread, but these are harmless (i hope). + * + * TODO: document functions and access restrictions for fields in BFD structures. + * + * Supported standards: + * - RFC 5880 - main BFD standard + * - RFC 5881 - BFD for IP links + * - RFC 5882 - generic application of BFD + * - RFC 5883 - BFD for multihop paths + */ + +#include "bfd.h" + + +#define HASH_ID_KEY(n) n->loc_id +#define HASH_ID_NEXT(n) n->next_id +#define HASH_ID_EQ(a,b) (a == b) +#define HASH_ID_FN(k) (k) + +#define HASH_IP_KEY(n) n->addr +#define HASH_IP_NEXT(n) n->next_ip +#define HASH_IP_EQ(a,b) ipa_equal(a,b) +#define HASH_IP_FN(k) ipa_hash(k) + +static list bfd_proto_list; +static list bfd_wait_list; + +const char *bfd_state_names[] = { "AdminDown", "Down", "Init", "Up" }; + +static void bfd_session_set_min_tx(struct bfd_session *s, u32 val); +static struct bfd_iface *bfd_get_iface(struct bfd_proto *p, ip_addr local, struct iface *iface); +static void bfd_free_iface(struct bfd_iface *ifa); +static inline void bfd_notify_kick(struct bfd_proto *p); + + +/* + * BFD sessions + */ + +static void +bfd_session_update_state(struct bfd_session *s, uint state, uint diag) +{ + struct bfd_proto *p = s->ifa->bfd; + uint old_state = s->loc_state; + int notify; + + if (state == old_state) + return; + + TRACE(D_EVENTS, "Session to %I changed state from %s to %s", + s->addr, bfd_state_names[old_state], bfd_state_names[state]); + + bfd_lock_sessions(p); + s->loc_state = state; + s->loc_diag = diag; + + notify = !NODE_VALID(&s->n); + if (notify) + add_tail(&p->notify_list, &s->n); + bfd_unlock_sessions(p); + + if (state == BFD_STATE_UP) + bfd_session_set_min_tx(s, s->ifa->cf->min_tx_int); + + if (old_state == BFD_STATE_UP) + bfd_session_set_min_tx(s, s->ifa->cf->idle_tx_int); + + if (notify) + bfd_notify_kick(p); +} + +static void +bfd_session_update_tx_interval(struct bfd_session *s) +{ + u32 tx_int = MAX(s->des_min_tx_int, s->rem_min_rx_int); + u32 tx_int_l = tx_int - (tx_int / 4); // 75 % + u32 tx_int_h = tx_int - (tx_int / 10); // 90 % + + s->tx_timer->recurrent = tx_int_l; + s->tx_timer->randomize = tx_int_h - tx_int_l; + + /* Do not set timer if no previous event */ + if (!s->last_tx) + return; + + /* Set timer relative to last tx_timer event */ + tm2_set(s->tx_timer, s->last_tx + tx_int_l); +} + +static void +bfd_session_update_detection_time(struct bfd_session *s, int kick) +{ + btime timeout = (btime) MAX(s->req_min_rx_int, s->rem_min_tx_int) * s->rem_detect_mult; + + if (kick) + s->last_rx = current_time(); + + if (!s->last_rx) + return; + + tm2_set(s->hold_timer, s->last_rx + timeout); +} + +static void +bfd_session_control_tx_timer(struct bfd_session *s, int reset) +{ + // if (!s->opened) goto stop; + + if (s->passive && (s->rem_id == 0)) + goto stop; + + if (s->rem_demand_mode && + !s->poll_active && + (s->loc_state == BFD_STATE_UP) && + (s->rem_state == BFD_STATE_UP)) + goto stop; + + if (s->rem_min_rx_int == 0) + goto stop; + + /* So TX timer should run */ + if (reset || !tm2_active(s->tx_timer)) + { + s->last_tx = 0; + tm2_start(s->tx_timer, 0); + } + + return; + + stop: + tm2_stop(s->tx_timer); + s->last_tx = 0; +} + +static void +bfd_session_request_poll(struct bfd_session *s, u8 request) +{ + /* Not sure about this, but doing poll in this case does not make sense */ + if (s->rem_id == 0) + return; + + s->poll_scheduled |= request; + + if (s->poll_active) + return; + + s->poll_active = s->poll_scheduled; + s->poll_scheduled = 0; + + bfd_session_control_tx_timer(s, 1); +} + +static void +bfd_session_terminate_poll(struct bfd_session *s) +{ + u8 poll_done = s->poll_active & ~s->poll_scheduled; + + if (poll_done & BFD_POLL_TX) + s->des_min_tx_int = s->des_min_tx_new; + + if (poll_done & BFD_POLL_RX) + s->req_min_rx_int = s->req_min_rx_new; + + s->poll_active = s->poll_scheduled; + s->poll_scheduled = 0; + + /* Timers are updated by caller - bfd_session_process_ctl() */ +} + +void +bfd_session_process_ctl(struct bfd_session *s, u8 flags, u32 old_tx_int, u32 old_rx_int) +{ + if (s->poll_active && (flags & BFD_FLAG_FINAL)) + bfd_session_terminate_poll(s); + + if ((s->des_min_tx_int != old_tx_int) || (s->rem_min_rx_int != old_rx_int)) + bfd_session_update_tx_interval(s); + + bfd_session_update_detection_time(s, 1); + + /* Update session state */ + int next_state = 0; + int diag = BFD_DIAG_NOTHING; + + switch (s->loc_state) + { + case BFD_STATE_ADMIN_DOWN: + return; + + case BFD_STATE_DOWN: + if (s->rem_state == BFD_STATE_DOWN) next_state = BFD_STATE_INIT; + else if (s->rem_state == BFD_STATE_INIT) next_state = BFD_STATE_UP; + break; + + case BFD_STATE_INIT: + if (s->rem_state == BFD_STATE_ADMIN_DOWN) next_state = BFD_STATE_DOWN, diag = BFD_DIAG_NEIGHBOR_DOWN; + else if (s->rem_state >= BFD_STATE_INIT) next_state = BFD_STATE_UP; + break; + + case BFD_STATE_UP: + if (s->rem_state <= BFD_STATE_DOWN) next_state = BFD_STATE_DOWN, diag = BFD_DIAG_NEIGHBOR_DOWN; + break; + } + + if (next_state) + bfd_session_update_state(s, next_state, diag); + + bfd_session_control_tx_timer(s, 0); + + if (flags & BFD_FLAG_POLL) + bfd_send_ctl(s->ifa->bfd, s, 1); +} + +static void +bfd_session_timeout(struct bfd_session *s) +{ + struct bfd_proto *p = s->ifa->bfd; + + TRACE(D_EVENTS, "Session to %I expired", s->addr); + + s->rem_state = BFD_STATE_DOWN; + s->rem_id = 0; + s->rem_min_tx_int = 0; + s->rem_min_rx_int = 1; + s->rem_demand_mode = 0; + s->rem_detect_mult = 0; + + s->poll_active = 0; + s->poll_scheduled = 0; + + bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_TIMEOUT); + + bfd_session_control_tx_timer(s, 1); +} + +static void +bfd_session_set_min_tx(struct bfd_session *s, u32 val) +{ + /* Note that des_min_tx_int <= des_min_tx_new */ + + if (val == s->des_min_tx_new) + return; + + s->des_min_tx_new = val; + + /* Postpone timer update if des_min_tx_int increases and the session is up */ + if ((s->loc_state != BFD_STATE_UP) || (val < s->des_min_tx_int)) + { + s->des_min_tx_int = val; + bfd_session_update_tx_interval(s); + } + + bfd_session_request_poll(s, BFD_POLL_TX); +} + +static void +bfd_session_set_min_rx(struct bfd_session *s, u32 val) +{ + /* Note that req_min_rx_int >= req_min_rx_new */ + + if (val == s->req_min_rx_new) + return; + + s->req_min_rx_new = val; + + /* Postpone timer update if req_min_rx_int decreases and the session is up */ + if ((s->loc_state != BFD_STATE_UP) || (val > s->req_min_rx_int)) + { + s->req_min_rx_int = val; + bfd_session_update_detection_time(s, 0); + } + + bfd_session_request_poll(s, BFD_POLL_RX); +} + +struct bfd_session * +bfd_find_session_by_id(struct bfd_proto *p, u32 id) +{ + return HASH_FIND(p->session_hash_id, HASH_ID, id); +} + +struct bfd_session * +bfd_find_session_by_addr(struct bfd_proto *p, ip_addr addr) +{ + return HASH_FIND(p->session_hash_ip, HASH_IP, addr); +} + +static void +bfd_tx_timer_hook(timer2 *t) +{ + struct bfd_session *s = t->data; + + s->last_tx = current_time(); + bfd_send_ctl(s->ifa->bfd, s, 0); +} + +static void +bfd_hold_timer_hook(timer2 *t) +{ + bfd_session_timeout(t->data); +} + +static u32 +bfd_get_free_id(struct bfd_proto *p) +{ + u32 id; + for (id = random_u32(); 1; id++) + if (id && !bfd_find_session_by_id(p, id)) + break; + + return id; +} + +static struct bfd_session * +bfd_add_session(struct bfd_proto *p, ip_addr addr, ip_addr local, struct iface *iface) +{ + birdloop_enter(p->loop); + + struct bfd_iface *ifa = bfd_get_iface(p, local, iface); + + struct bfd_session *s = sl_alloc(p->session_slab); + bzero(s, sizeof(struct bfd_session)); + + s->addr = addr; + s->ifa = ifa; + s->loc_id = bfd_get_free_id(p); + + HASH_INSERT(p->session_hash_id, HASH_ID, s); + HASH_INSERT(p->session_hash_ip, HASH_IP, s); + + + /* Initialization of state variables - see RFC 5880 6.8.1 */ + s->loc_state = BFD_STATE_DOWN; + s->rem_state = BFD_STATE_DOWN; + s->des_min_tx_int = s->des_min_tx_new = ifa->cf->idle_tx_int; + s->req_min_rx_int = s->req_min_rx_new = ifa->cf->min_rx_int; + s->rem_min_rx_int = 1; + s->detect_mult = ifa->cf->multiplier; + s->passive = ifa->cf->passive; + + s->tx_timer = tm2_new_init(p->tpool, bfd_tx_timer_hook, s, 0, 0); + s->hold_timer = tm2_new_init(p->tpool, bfd_hold_timer_hook, s, 0, 0); + bfd_session_update_tx_interval(s); + bfd_session_control_tx_timer(s, 1); + + init_list(&s->request_list); + s->last_state_change = now; + + TRACE(D_EVENTS, "Session to %I added", s->addr); + + birdloop_leave(p->loop); + + return s; +} + +/* +static void +bfd_open_session(struct bfd_proto *p, struct bfd_session *s, ip_addr local, struct iface *ifa) +{ + birdloop_enter(p->loop); + + s->opened = 1; + + bfd_session_control_tx_timer(s); + + birdloop_leave(p->loop); +} + +static void +bfd_close_session(struct bfd_proto *p, struct bfd_session *s) +{ + birdloop_enter(p->loop); + + s->opened = 0; + + bfd_session_update_state(s, BFD_STATE_DOWN, BFD_DIAG_PATH_DOWN); + bfd_session_control_tx_timer(s); + + birdloop_leave(p->loop); +} +*/ + +static void +bfd_remove_session(struct bfd_proto *p, struct bfd_session *s) +{ + ip_addr ip = s->addr; + + birdloop_enter(p->loop); + + bfd_free_iface(s->ifa); + + rfree(s->tx_timer); + rfree(s->hold_timer); + + HASH_REMOVE(p->session_hash_id, HASH_ID, s); + HASH_REMOVE(p->session_hash_ip, HASH_IP, s); + + sl_free(p->session_slab, s); + + TRACE(D_EVENTS, "Session to %I removed", ip); + + birdloop_leave(p->loop); +} + +static void +bfd_reconfigure_session(struct bfd_proto *p, struct bfd_session *s) +{ + birdloop_enter(p->loop); + + struct bfd_iface_config *cf = s->ifa->cf; + + u32 tx = (s->loc_state == BFD_STATE_UP) ? cf->min_tx_int : cf->idle_tx_int; + bfd_session_set_min_tx(s, tx); + bfd_session_set_min_rx(s, cf->min_rx_int); + s->detect_mult = cf->multiplier; + s->passive = cf->passive; + + bfd_session_control_tx_timer(s, 0); + + birdloop_leave(p->loop); + + TRACE(D_EVENTS, "Session to %I reconfigured", s->addr); +} + + +/* + * BFD interfaces + */ + +static struct bfd_iface_config bfd_default_iface = { + .min_rx_int = BFD_DEFAULT_MIN_RX_INT, + .min_tx_int = BFD_DEFAULT_MIN_TX_INT, + .idle_tx_int = BFD_DEFAULT_IDLE_TX_INT, + .multiplier = BFD_DEFAULT_MULTIPLIER +}; + +static inline struct bfd_iface_config * +bfd_find_iface_config(struct bfd_config *cf, struct iface *iface) +{ + struct bfd_iface_config *ic; + + ic = iface ? (void *) iface_patt_find(&cf->patt_list, iface, NULL) : cf->multihop; + + return ic ? ic : &bfd_default_iface; +} + +static struct bfd_iface * +bfd_get_iface(struct bfd_proto *p, ip_addr local, struct iface *iface) +{ + struct bfd_iface *ifa; + + WALK_LIST(ifa, p->iface_list) + if (ipa_equal(ifa->local, local) && (ifa->iface == iface)) + return ifa->uc++, ifa; + + struct bfd_config *cf = (struct bfd_config *) (p->p.cf); + struct bfd_iface_config *ic = bfd_find_iface_config(cf, iface); + + ifa = mb_allocz(p->tpool, sizeof(struct bfd_iface)); + ifa->local = local; + ifa->iface = iface; + ifa->cf = ic; + ifa->bfd = p; + + ifa->sk = bfd_open_tx_sk(p, local, iface); + ifa->uc = 1; + + add_tail(&p->iface_list, &ifa->n); + + return ifa; +} + +static void +bfd_free_iface(struct bfd_iface *ifa) +{ + if (!ifa || --ifa->uc) + return; + + rem_node(&ifa->n); + sk_stop(ifa->sk); + rfree(ifa->sk); + mb_free(ifa); +} + +static void +bfd_reconfigure_iface(struct bfd_proto *p, struct bfd_iface *ifa, struct bfd_config *nc) +{ + struct bfd_iface_config *nic = bfd_find_iface_config(nc, ifa->iface); + ifa->changed = !!memcmp(nic, ifa->cf, sizeof(struct bfd_iface_config)); + + /* This should be probably changed to not access ifa->cf from the BFD thread */ + birdloop_enter(p->loop); + ifa->cf = nic; + birdloop_leave(p->loop); +} + + +/* + * BFD requests + */ + +static void +bfd_request_notify(struct bfd_request *req, u8 state, u8 diag) +{ + u8 old_state = req->state; + + if (state == old_state) + return; + + req->state = state; + req->diag = diag; + req->old_state = old_state; + req->down = (old_state == BFD_STATE_UP) && (state == BFD_STATE_DOWN); + + if (req->hook) + req->hook(req); +} + +static int +bfd_add_request(struct bfd_proto *p, struct bfd_request *req) +{ + struct bfd_session *s = bfd_find_session_by_addr(p, req->addr); + u8 state, diag; + + if (!s) + s = bfd_add_session(p, req->addr, req->local, req->iface); + + rem_node(&req->n); + add_tail(&s->request_list, &req->n); + req->session = s; + + bfd_lock_sessions(p); + state = s->loc_state; + diag = s->loc_diag; + bfd_unlock_sessions(p); + + bfd_request_notify(req, state, diag); + + return 1; +} + +static void +bfd_submit_request(struct bfd_request *req) +{ + node *n; + + WALK_LIST(n, bfd_proto_list) + if (bfd_add_request(SKIP_BACK(struct bfd_proto, bfd_node, n), req)) + return; + + rem_node(&req->n); + add_tail(&bfd_wait_list, &req->n); + req->session = NULL; + bfd_request_notify(req, BFD_STATE_ADMIN_DOWN, 0); +} + +static void +bfd_take_requests(struct bfd_proto *p) +{ + node *n, *nn; + + WALK_LIST_DELSAFE(n, nn, bfd_wait_list) + bfd_add_request(p, SKIP_BACK(struct bfd_request, n, n)); +} + +static void +bfd_drop_requests(struct bfd_proto *p) +{ + node *n; + + HASH_WALK(p->session_hash_id, next_id, s) + { + /* We assume that p is not in bfd_proto_list */ + WALK_LIST_FIRST(n, s->request_list) + bfd_submit_request(SKIP_BACK(struct bfd_request, n, n)); + } + HASH_WALK_END; +} + +static struct resclass bfd_request_class; + +struct bfd_request * +bfd_request_session(pool *p, ip_addr addr, ip_addr local, struct iface *iface, + void (*hook)(struct bfd_request *), void *data) +{ + struct bfd_request *req = ralloc(p, &bfd_request_class); + + /* Hack: self-link req->n, we will call rem_node() on it */ + req->n.prev = req->n.next = &req->n; + + req->addr = addr; + req->local = local; + req->iface = iface; + + bfd_submit_request(req); + + req->hook = hook; + req->data = data; + + return req; +} + +static void +bfd_request_free(resource *r) +{ + struct bfd_request *req = (struct bfd_request *) r; + struct bfd_session *s = req->session; + + rem_node(&req->n); + + /* Remove the session if there is no request for it. Skip that if + inside notify hooks, will be handled by bfd_notify_hook() itself */ + + if (s && EMPTY_LIST(s->request_list) && !s->notify_running) + bfd_remove_session(s->ifa->bfd, s); +} + +static void +bfd_request_dump(resource *r) +{ + struct bfd_request *req = (struct bfd_request *) r; + + debug("(code %p, data %p)\n", req->hook, req->data); +} + +static struct resclass bfd_request_class = { + "BFD request", + sizeof(struct bfd_request), + bfd_request_free, + bfd_request_dump, + NULL, + NULL +}; + + +/* + * BFD neighbors + */ + +static void +bfd_neigh_notify(struct neighbor *nb) +{ + struct bfd_proto *p = (struct bfd_proto *) nb->proto; + struct bfd_neighbor *n = nb->data; + + if (!n) + return; + + if ((nb->scope > 0) && !n->req) + { + ip_addr local = ipa_nonzero(n->local) ? n->local : nb->iface->addr->ip; + n->req = bfd_request_session(p->p.pool, n->addr, local, nb->iface, NULL, NULL); + } + + if ((nb->scope <= 0) && n->req) + { + rfree(n->req); + n->req = NULL; + } +} + +static void +bfd_start_neighbor(struct bfd_proto *p, struct bfd_neighbor *n) +{ + n->active = 1; + + if (n->multihop) + { + n->req = bfd_request_session(p->p.pool, n->addr, n->local, NULL, NULL, NULL); + return; + } + + struct neighbor *nb = neigh_find2(&p->p, &n->addr, n->iface, NEF_STICKY); + if (!nb) + { + log(L_ERR "%s: Invalid remote address %I%J", p->p.name, n->addr, n->iface); + return; + } + + if (nb->data) + { + log(L_ERR "%s: Duplicate neighbor %I", p->p.name, n->addr); + return; + } + + n->neigh = nb; + nb->data = n; + + if (nb->scope > 0) + bfd_neigh_notify(nb); + else + TRACE(D_EVENTS, "Waiting for %I%J to become my neighbor", n->addr, n->iface); +} + +static void +bfd_stop_neighbor(struct bfd_proto *p, struct bfd_neighbor *n) +{ + if (n->neigh) + n->neigh->data = NULL; + n->neigh = NULL; + + rfree(n->req); + n->req = NULL; +} + +static inline int +bfd_same_neighbor(struct bfd_neighbor *x, struct bfd_neighbor *y) +{ + return ipa_equal(x->addr, y->addr) && ipa_equal(x->local, y->local) && + (x->iface == y->iface) && (x->multihop == y->multihop); +} + +static void +bfd_reconfigure_neighbors(struct bfd_proto *p, struct bfd_config *new) +{ + struct bfd_config *old = (struct bfd_config *) (p->p.cf); + struct bfd_neighbor *on, *nn; + + WALK_LIST(on, old->neigh_list) + { + WALK_LIST(nn, new->neigh_list) + if (bfd_same_neighbor(nn, on)) + { + nn->neigh = on->neigh; + if (nn->neigh) + nn->neigh->data = nn; + + nn->req = on->req; + nn->active = 1; + return; + } + + bfd_stop_neighbor(p, on); + } + + WALK_LIST(nn, new->neigh_list) + if (!nn->active) + bfd_start_neighbor(p, nn); +} + + +/* + * BFD notify socket + */ + +/* This core notify code should be replaced after main loop transition to birdloop */ + +int pipe(int pipefd[2]); +void pipe_drain(int fd); +void pipe_kick(int fd); + +static int +bfd_notify_hook(sock *sk, int len) +{ + struct bfd_proto *p = sk->data; + struct bfd_session *s; + list tmp_list; + u8 state, diag; + node *n, *nn; + + pipe_drain(sk->fd); + + bfd_lock_sessions(p); + init_list(&tmp_list); + add_tail_list(&tmp_list, &p->notify_list); + init_list(&p->notify_list); + bfd_unlock_sessions(p); + + WALK_LIST_FIRST(s, tmp_list) + { + bfd_lock_sessions(p); + rem2_node(&s->n); + state = s->loc_state; + diag = s->loc_diag; + bfd_unlock_sessions(p); + + /* FIXME: convert to btime and move to bfd_session_update_state() */ + s->last_state_change = now; + + s->notify_running = 1; + WALK_LIST_DELSAFE(n, nn, s->request_list) + bfd_request_notify(SKIP_BACK(struct bfd_request, n, n), state, diag); + s->notify_running = 0; + + /* Remove the session if all requests were removed in notify hooks */ + if (EMPTY_LIST(s->request_list)) + bfd_remove_session(p, s); + } + + return 0; +} + +static inline void +bfd_notify_kick(struct bfd_proto *p) +{ + pipe_kick(p->notify_ws->fd); +} + +static void +bfd_noterr_hook(sock *sk, int err) +{ + struct bfd_proto *p = sk->data; + log(L_ERR "%s: Notify socket error: %m", p->p.name, err); +} + +static void +bfd_notify_init(struct bfd_proto *p) +{ + int pfds[2]; + sock *sk; + + int rv = pipe(pfds); + if (rv < 0) + die("pipe: %m"); + + sk = sk_new(p->p.pool); + sk->type = SK_MAGIC; + sk->rx_hook = bfd_notify_hook; + sk->err_hook = bfd_noterr_hook; + sk->fd = pfds[0]; + sk->data = p; + if (sk_open(sk) < 0) + die("bfd: sk_open failed"); + p->notify_rs = sk; + + /* The write sock is not added to any event loop */ + sk = sk_new(p->p.pool); + sk->type = SK_MAGIC; + sk->fd = pfds[1]; + sk->data = p; + sk->flags = SKF_THREAD; + if (sk_open(sk) < 0) + die("bfd: sk_open failed"); + p->notify_ws = sk; +} + + +/* + * BFD protocol glue + */ + +void +bfd_init_all(void) +{ + init_list(&bfd_proto_list); + init_list(&bfd_wait_list); +} + +static struct proto * +bfd_init(struct proto_config *c) +{ + struct proto *p = proto_new(c, sizeof(struct bfd_proto)); + + p->neigh_notify = bfd_neigh_notify; + + return p; +} + +static int +bfd_start(struct proto *P) +{ + struct bfd_proto *p = (struct bfd_proto *) P; + struct bfd_config *cf = (struct bfd_config *) (P->cf); + + p->loop = birdloop_new(); + p->tpool = rp_new(NULL, "BFD thread root"); + pthread_spin_init(&p->lock, PTHREAD_PROCESS_PRIVATE); + + p->session_slab = sl_new(P->pool, sizeof(struct bfd_session)); + HASH_INIT(p->session_hash_id, P->pool, 8); + HASH_INIT(p->session_hash_ip, P->pool, 8); + + init_list(&p->iface_list); + + init_list(&p->notify_list); + bfd_notify_init(p); + + add_tail(&bfd_proto_list, &p->bfd_node); + + birdloop_enter(p->loop); + p->rx_1 = bfd_open_rx_sk(p, 0); + p->rx_m = bfd_open_rx_sk(p, 1); + birdloop_leave(p->loop); + + bfd_take_requests(p); + + struct bfd_neighbor *n; + WALK_LIST(n, cf->neigh_list) + bfd_start_neighbor(p, n); + + birdloop_start(p->loop); + + return PS_UP; +} + + +static int +bfd_shutdown(struct proto *P) +{ + struct bfd_proto *p = (struct bfd_proto *) P; + struct bfd_config *cf = (struct bfd_config *) (P->cf); + + rem_node(&p->bfd_node); + + birdloop_stop(p->loop); + + struct bfd_neighbor *n; + WALK_LIST(n, cf->neigh_list) + bfd_stop_neighbor(p, n); + + bfd_drop_requests(p); + + /* FIXME: This is hack */ + birdloop_enter(p->loop); + rfree(p->tpool); + birdloop_leave(p->loop); + + birdloop_free(p->loop); + + return PS_DOWN; +} + +static int +bfd_reconfigure(struct proto *P, struct proto_config *c) +{ + struct bfd_proto *p = (struct bfd_proto *) P; + // struct bfd_config *old = (struct bfd_config *) (P->cf); + struct bfd_config *new = (struct bfd_config *) c; + struct bfd_iface *ifa; + + birdloop_mask_wakeups(p->loop); + + WALK_LIST(ifa, p->iface_list) + bfd_reconfigure_iface(p, ifa, new); + + HASH_WALK(p->session_hash_id, next_id, s) + { + if (s->ifa->changed) + bfd_reconfigure_session(p, s); + } + HASH_WALK_END; + + bfd_reconfigure_neighbors(p, new); + + birdloop_unmask_wakeups(p->loop); + + return 1; +} + +/* Ensure one instance */ +struct bfd_config *bfd_cf; + +static void +bfd_preconfig(struct protocol *P UNUSED, struct config *c UNUSED) +{ + bfd_cf = NULL; +} + +static void +bfd_copy_config(struct proto_config *dest, struct proto_config *src) +{ + struct bfd_config *d = (struct bfd_config *) dest; + // struct bfd_config *s = (struct bfd_config *) src; + + /* We clean up patt_list and neigh_list, neighbors and ifaces are non-sharable */ + init_list(&d->patt_list); + init_list(&d->neigh_list); +} + +void +bfd_show_sessions(struct proto *P) +{ + byte tbuf[TM_DATETIME_BUFFER_SIZE]; + struct bfd_proto *p = (struct bfd_proto *) P; + uint state, diag; + u32 tx_int, timeout; + const char *ifname; + + if (p->p.proto_state != PS_UP) + { + cli_msg(-1013, "%s: is not up", p->p.name); + cli_msg(0, ""); + return; + } + + cli_msg(-1013, "%s:", p->p.name); + cli_msg(-1013, "%-25s %-10s %-10s %-10s %8s %8s", + "IP address", "Interface", "State", "Since", "Interval", "Timeout"); + + + HASH_WALK(p->session_hash_id, next_id, s) + { + /* FIXME: this is thread-unsafe, but perhaps harmless */ + state = s->loc_state; + diag = s->loc_diag; + ifname = (s->ifa && s->ifa->sk->iface) ? s->ifa->sk->iface->name : "---"; + tx_int = s->last_tx ? (MAX(s->des_min_tx_int, s->rem_min_rx_int) TO_MS) : 0; + timeout = (MAX(s->req_min_rx_int, s->rem_min_tx_int) TO_MS) * s->rem_detect_mult; + + state = (state < 4) ? state : 0; + tm_format_datetime(tbuf, &config->tf_proto, s->last_state_change); + + cli_msg(-1013, "%-25I %-10s %-10s %-10s %3u.%03u %3u.%03u", + s->addr, ifname, bfd_state_names[state], tbuf, + tx_int / 1000, tx_int % 1000, timeout / 1000, timeout % 1000); + } + HASH_WALK_END; + + cli_msg(0, ""); +} + + +struct protocol proto_bfd = { + .name = "BFD", + .template = "bfd%d", + .init = bfd_init, + .start = bfd_start, + .shutdown = bfd_shutdown, + .reconfigure = bfd_reconfigure, + .preconfig = bfd_preconfig, + .copy_config = bfd_copy_config, +}; diff --git a/proto/bfd/bfd.h b/proto/bfd/bfd.h new file mode 100644 index 00000000..9b61be64 --- /dev/null +++ b/proto/bfd/bfd.h @@ -0,0 +1,191 @@ +/* + * BIRD -- Bidirectional Forwarding Detection (BFD) + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_BFD_H_ +#define _BIRD_BFD_H_ + +#include <pthread.h> + +#include "nest/bird.h" +#include "nest/cli.h" +#include "nest/iface.h" +#include "nest/protocol.h" +#include "nest/route.h" +#include "conf/conf.h" +#include "lib/hash.h" +#include "lib/resource.h" +#include "lib/socket.h" +#include "lib/string.h" + +#include "nest/bfd.h" +#include "io.h" + + +#define BFD_CONTROL_PORT 3784 +#define BFD_ECHO_PORT 3785 +#define BFD_MULTI_CTL_PORT 4784 + +#define BFD_DEFAULT_MIN_RX_INT (10 MS_) +#define BFD_DEFAULT_MIN_TX_INT (100 MS_) +#define BFD_DEFAULT_IDLE_TX_INT (1 S_) +#define BFD_DEFAULT_MULTIPLIER 5 + + +struct bfd_iface_config; + +struct bfd_config +{ + struct proto_config c; + list patt_list; /* List of iface configs (struct bfd_iface_config) */ + list neigh_list; /* List of configured neighbors (struct bfd_neighbor) */ + struct bfd_iface_config *multihop; /* Multihop pseudoiface config */ +}; + +struct bfd_iface_config +{ + struct iface_patt i; + u32 min_rx_int; + u32 min_tx_int; + u32 idle_tx_int; + u8 multiplier; + u8 passive; +}; + +struct bfd_neighbor +{ + node n; + ip_addr addr; + ip_addr local; + struct iface *iface; + + struct neighbor *neigh; + struct bfd_request *req; + + u8 multihop; + u8 active; +}; + +struct bfd_proto +{ + struct proto p; + struct birdloop *loop; + pool *tpool; + pthread_spinlock_t lock; + node bfd_node; + + slab *session_slab; + HASH(struct bfd_session) session_hash_id; + HASH(struct bfd_session) session_hash_ip; + + sock *notify_rs; + sock *notify_ws; + list notify_list; + + sock *rx_1; + sock *rx_m; + list iface_list; +}; + +struct bfd_iface +{ + node n; + ip_addr local; + struct iface *iface; + struct bfd_iface_config *cf; + struct bfd_proto *bfd; + + sock *sk; + u32 uc; + u8 changed; +}; + +struct bfd_session +{ + node n; + ip_addr addr; /* Address of session */ + struct bfd_iface *ifa; /* Iface associated with session */ + struct bfd_session *next_id; /* Next in bfd.session_hash_id */ + struct bfd_session *next_ip; /* Next in bfd.session_hash_ip */ + + u8 opened_unused; + u8 passive; + u8 poll_active; + u8 poll_scheduled; + + u8 loc_state; + u8 rem_state; + u8 loc_diag; + u8 rem_diag; + u32 loc_id; /* Local session ID (local discriminator) */ + u32 rem_id; /* Remote session ID (remote discriminator) */ + u32 des_min_tx_int; /* Desired min rx interval, local option */ + u32 des_min_tx_new; /* Used for des_min_tx_int change */ + u32 req_min_rx_int; /* Required min tx interval, local option */ + u32 req_min_rx_new; /* Used for req_min_rx_int change */ + u32 rem_min_tx_int; /* Last received des_min_tx_int */ + u32 rem_min_rx_int; /* Last received req_min_rx_int */ + u8 demand_mode; /* Currently unused */ + u8 rem_demand_mode; + u8 detect_mult; /* Announced detect_mult, local option */ + u8 rem_detect_mult; /* Last received detect_mult */ + + btime last_tx; /* Time of last sent periodic control packet */ + btime last_rx; /* Time of last received valid control packet */ + + timer2 *tx_timer; /* Periodic control packet timer */ + timer2 *hold_timer; /* Timer for session down detection time */ + + list request_list; /* List of client requests (struct bfd_request) */ + bird_clock_t last_state_change; /* Time of last state change */ + u8 notify_running; /* 1 if notify hooks are running */ +}; + + +extern const char *bfd_state_names[]; + +#define BFD_STATE_ADMIN_DOWN 0 +#define BFD_STATE_DOWN 1 +#define BFD_STATE_INIT 2 +#define BFD_STATE_UP 3 + +#define BFD_DIAG_NOTHING 0 +#define BFD_DIAG_TIMEOUT 1 +#define BFD_DIAG_ECHO_FAILED 2 +#define BFD_DIAG_NEIGHBOR_DOWN 3 +#define BFD_DIAG_FWD_RESET 4 +#define BFD_DIAG_PATH_DOWN 5 +#define BFD_DIAG_C_PATH_DOWN 6 +#define BFD_DIAG_ADMIN_DOWN 7 +#define BFD_DIAG_RC_PATH_DOWN 8 + +#define BFD_POLL_TX 1 +#define BFD_POLL_RX 2 + +#define BFD_FLAGS 0x3f +#define BFD_FLAG_POLL (1 << 5) +#define BFD_FLAG_FINAL (1 << 4) +#define BFD_FLAG_CPI (1 << 3) +#define BFD_FLAG_AP (1 << 2) +#define BFD_FLAG_DEMAND (1 << 1) +#define BFD_FLAG_MULTIPOINT (1 << 0) + + +static inline void bfd_lock_sessions(struct bfd_proto *p) { pthread_spin_lock(&p->lock); } +static inline void bfd_unlock_sessions(struct bfd_proto *p) { pthread_spin_unlock(&p->lock); } + +/* bfd.c */ +struct bfd_session * bfd_find_session_by_id(struct bfd_proto *p, u32 id); +struct bfd_session * bfd_find_session_by_addr(struct bfd_proto *p, ip_addr addr); +void bfd_session_process_ctl(struct bfd_session *s, u8 flags, u32 old_tx_int, u32 old_rx_int); +void bfd_show_sessions(struct proto *P); + +/* packets.c */ +void bfd_send_ctl(struct bfd_proto *p, struct bfd_session *s, int final); +sock * bfd_open_rx_sk(struct bfd_proto *p, int multihop); +sock * bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa); + + +#endif /* _BIRD_BFD_H_ */ diff --git a/proto/bfd/config.Y b/proto/bfd/config.Y new file mode 100644 index 00000000..1bf8764f --- /dev/null +++ b/proto/bfd/config.Y @@ -0,0 +1,138 @@ +/* + * BIRD -- Router Advertisement Configuration + * + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +CF_HDR + +#include "proto/bfd/bfd.h" + +CF_DEFINES + +#define BFD_CFG ((struct bfd_config *) this_proto) +#define BFD_IFACE ((struct bfd_iface_config *) this_ipatt) +#define BFD_NEIGHBOR this_bfd_neighbor + +static struct bfd_neighbor *this_bfd_neighbor; + +extern struct bfd_config *bfd_cf; + +CF_DECLS + +CF_KEYWORDS(BFD, MIN, IDLE, RX, TX, INTERVAL, MULTIPLIER, PASSIVE, + INTERFACE, MULTIHOP, NEIGHBOR, DEV, LOCAL) + +%type <iface> bfd_neigh_iface +%type <a> bfd_neigh_local +%type <i> bfd_neigh_multihop + +CF_GRAMMAR + +CF_ADDTO(proto, bfd_proto) + +bfd_proto_start: proto_start BFD +{ + this_proto = proto_config_new(&proto_bfd, sizeof(struct bfd_config), $1); + init_list(&BFD_CFG->patt_list); + init_list(&BFD_CFG->neigh_list); + + if (bfd_cf) + cf_error("Only one BFD instance allowed"); + bfd_cf = BFD_CFG; +}; + +bfd_proto_item: + proto_item + | INTERFACE bfd_iface + | MULTIHOP bfd_multihop + | NEIGHBOR bfd_neighbor + ; + +bfd_proto_opts: + /* empty */ + | bfd_proto_opts bfd_proto_item ';' + ; + +bfd_proto: + bfd_proto_start proto_name '{' bfd_proto_opts '}'; + + +bfd_iface_start: +{ + this_ipatt = cfg_allocz(sizeof(struct bfd_iface_config)); + init_list(&this_ipatt->ipn_list); + + BFD_IFACE->min_rx_int = BFD_DEFAULT_MIN_RX_INT; + BFD_IFACE->min_tx_int = BFD_DEFAULT_MIN_TX_INT; + BFD_IFACE->idle_tx_int = BFD_DEFAULT_IDLE_TX_INT; + BFD_IFACE->multiplier = BFD_DEFAULT_MULTIPLIER; +}; + +bfd_iface_item: + INTERVAL expr_us { BFD_IFACE->min_rx_int = BFD_IFACE->min_tx_int = $2; } + | MIN RX INTERVAL expr_us { BFD_IFACE->min_rx_int = $4; } + | MIN TX INTERVAL expr_us { BFD_IFACE->min_tx_int = $4; } + | IDLE TX INTERVAL expr_us { BFD_IFACE->idle_tx_int = $4; } + | MULTIPLIER expr { BFD_IFACE->multiplier = $2; } + | PASSIVE bool { BFD_IFACE->passive = $2; } + ; + +bfd_iface_opts: + /* empty */ + | bfd_iface_opts bfd_iface_item ';' + ; + +bfd_iface_opt_list: + /* empty */ + | '{' bfd_iface_opts '}' + ; + +bfd_iface: bfd_iface_start iface_patt_list bfd_iface_opt_list +{ add_tail(&BFD_CFG->patt_list, NODE this_ipatt); }; + +bfd_multihop: bfd_iface_start bfd_iface_opt_list +{ BFD_CFG->multihop = BFD_IFACE; }; + + +bfd_neigh_iface: + /* empty */ { $$ = NULL; } + | '%' SYM { $$ = if_get_by_name($2->name); } + | DEV TEXT { $$ = if_get_by_name($2); } + ; + +bfd_neigh_local: + /* empty */ { $$ = IPA_NONE; } + | LOCAL ipa { $$ = $2; } + ; + +bfd_neigh_multihop: + /* empty */ { $$ = 0; } + | MULTIHOP bool { $$ = $2; } + ; + +bfd_neighbor: ipa bfd_neigh_iface bfd_neigh_local bfd_neigh_multihop +{ + this_bfd_neighbor = cfg_allocz(sizeof(struct bfd_neighbor)); + add_tail(&BFD_CFG->neigh_list, NODE this_bfd_neighbor); + + BFD_NEIGHBOR->addr = $1; + BFD_NEIGHBOR->local = $3; + BFD_NEIGHBOR->iface = $2; + BFD_NEIGHBOR->multihop = $4; + + if ($4 && $2) + cf_error("Neighbor cannot set both interface and multihop"); + + if ($4 && ipa_zero($3)) + cf_error("Multihop neighbor requires specified local address"); +}; + + +CF_CLI(SHOW BFD SESSIONS, optsym, [<name>], [[Show information about BFD sessions]]) +{ bfd_show_sessions(proto_get_named($4, &proto_bfd)); }; + +CF_CODE + +CF_END diff --git a/proto/bfd/io.c b/proto/bfd/io.c new file mode 100644 index 00000000..fb150040 --- /dev/null +++ b/proto/bfd/io.c @@ -0,0 +1,768 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <fcntl.h> +#include <poll.h> +#include <pthread.h> +#include <time.h> +#include <sys/time.h> + +#include "nest/bird.h" +#include "proto/bfd/io.h" + +#include "lib/buffer.h" +#include "lib/heap.h" +#include "lib/lists.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/socket.h" + + +struct birdloop +{ + pool *pool; + pthread_t thread; + pthread_mutex_t mutex; + + btime last_time; + btime real_time; + u8 use_monotonic_clock; + + u8 stop_called; + u8 poll_active; + u8 wakeup_masked; + int wakeup_fds[2]; + + BUFFER(timer2 *) timers; + list event_list; + list sock_list; + uint sock_num; + + BUFFER(sock *) poll_sk; + BUFFER(struct pollfd) poll_fd; + u8 poll_changed; + u8 close_scheduled; +}; + + +/* + * Current thread context + */ + +static pthread_key_t current_loop_key; + +static inline struct birdloop * +birdloop_current(void) +{ + return pthread_getspecific(current_loop_key); +} + +static inline void +birdloop_set_current(struct birdloop *loop) +{ + pthread_setspecific(current_loop_key, loop); +} + +static inline void +birdloop_init_current(void) +{ + pthread_key_create(¤t_loop_key, NULL); +} + + +/* + * Time clock + */ + +static void times_update_alt(struct birdloop *loop); + +static void +times_init(struct birdloop *loop) +{ + struct timespec ts; + int rv; + + rv = clock_gettime(CLOCK_MONOTONIC, &ts); + if (rv < 0) + { + log(L_WARN "Monotonic clock is missing"); + + loop->use_monotonic_clock = 0; + loop->last_time = 0; + loop->real_time = 0; + times_update_alt(loop); + return; + } + + if ((ts.tv_sec < 0) || (((s64) ts.tv_sec) > ((s64) 1 << 40))) + log(L_WARN "Monotonic clock is crazy"); + + loop->use_monotonic_clock = 1; + loop->last_time = ((s64) ts.tv_sec S) + (ts.tv_nsec / 1000); + loop->real_time = 0; +} + +static void +times_update_pri(struct birdloop *loop) +{ + struct timespec ts; + int rv; + + rv = clock_gettime(CLOCK_MONOTONIC, &ts); + if (rv < 0) + die("clock_gettime: %m"); + + btime new_time = ((s64) ts.tv_sec S) + (ts.tv_nsec / 1000); + + if (new_time < loop->last_time) + log(L_ERR "Monotonic clock is broken"); + + loop->last_time = new_time; + loop->real_time = 0; +} + +static void +times_update_alt(struct birdloop *loop) +{ + struct timeval tv; + int rv; + + rv = gettimeofday(&tv, NULL); + if (rv < 0) + die("gettimeofday: %m"); + + btime new_time = ((s64) tv.tv_sec S) + tv.tv_usec; + btime delta = new_time - loop->real_time; + + if ((delta < 0) || (delta > (60 S))) + { + if (loop->real_time) + log(L_WARN "Time jump, delta %d us", (int) delta); + + delta = 100 MS; + } + + loop->last_time += delta; + loop->real_time = new_time; +} + +static void +times_update(struct birdloop *loop) +{ + if (loop->use_monotonic_clock) + times_update_pri(loop); + else + times_update_alt(loop); +} + +btime +current_time(void) +{ + return birdloop_current()->last_time; +} + + +/* + * Wakeup code for birdloop + */ + +static void +pipe_new(int *pfds) +{ + int rv = pipe(pfds); + if (rv < 0) + die("pipe: %m"); + + if (fcntl(pfds[0], F_SETFL, O_NONBLOCK) < 0) + die("fcntl(O_NONBLOCK): %m"); + + if (fcntl(pfds[1], F_SETFL, O_NONBLOCK) < 0) + die("fcntl(O_NONBLOCK): %m"); +} + +void +pipe_drain(int fd) +{ + char buf[64]; + int rv; + + try: + rv = read(fd, buf, 64); + if (rv < 0) + { + if (errno == EINTR) + goto try; + if (errno == EAGAIN) + return; + die("wakeup read: %m"); + } + if (rv == 64) + goto try; +} + +void +pipe_kick(int fd) +{ + u64 v = 1; + int rv; + + try: + rv = write(fd, &v, sizeof(u64)); + if (rv < 0) + { + if (errno == EINTR) + goto try; + if (errno == EAGAIN) + return; + die("wakeup write: %m"); + } +} + +static inline void +wakeup_init(struct birdloop *loop) +{ + pipe_new(loop->wakeup_fds); +} + +static inline void +wakeup_drain(struct birdloop *loop) +{ + pipe_drain(loop->wakeup_fds[0]); +} + +static inline void +wakeup_do_kick(struct birdloop *loop) +{ + pipe_kick(loop->wakeup_fds[1]); +} + +static inline void +wakeup_kick(struct birdloop *loop) +{ + if (!loop->wakeup_masked) + wakeup_do_kick(loop); + else + loop->wakeup_masked = 2; +} + + +/* + * Events + */ + +static inline uint +events_waiting(struct birdloop *loop) +{ + return !EMPTY_LIST(loop->event_list); +} + +static inline void +events_init(struct birdloop *loop) +{ + init_list(&loop->event_list); +} + +static void +events_fire(struct birdloop *loop) +{ + times_update(loop); + ev_run_list(&loop->event_list); +} + +void +ev2_schedule(event *e) +{ + struct birdloop *loop = birdloop_current(); + + if (loop->poll_active && EMPTY_LIST(loop->event_list)) + wakeup_kick(loop); + + if (e->n.next) + rem_node(&e->n); + + add_tail(&loop->event_list, &e->n); +} + + +/* + * Timers + */ + +#define TIMER_LESS(a,b) ((a)->expires < (b)->expires) +#define TIMER_SWAP(heap,a,b,t) (t = heap[a], heap[a] = heap[b], heap[b] = t, \ + heap[a]->index = (a), heap[b]->index = (b)) + +static inline uint timers_count(struct birdloop *loop) +{ return loop->timers.used - 1; } + +static inline timer2 *timers_first(struct birdloop *loop) +{ return (loop->timers.used > 1) ? loop->timers.data[1] : NULL; } + + +static void +tm2_free(resource *r) +{ + timer2 *t = (timer2 *) r; + + tm2_stop(t); +} + +static void +tm2_dump(resource *r) +{ + timer2 *t = (timer2 *) r; + + debug("(code %p, data %p, ", t->hook, t->data); + if (t->randomize) + debug("rand %d, ", t->randomize); + if (t->recurrent) + debug("recur %d, ", t->recurrent); + if (t->expires) + debug("expires in %d ms)\n", (t->expires - current_time()) TO_MS); + else + debug("inactive)\n"); +} + + +static struct resclass tm2_class = { + "Timer", + sizeof(timer2), + tm2_free, + tm2_dump, + NULL, + NULL +}; + +timer2 * +tm2_new(pool *p) +{ + timer2 *t = ralloc(p, &tm2_class); + t->index = -1; + return t; +} + +void +tm2_set(timer2 *t, btime when) +{ + struct birdloop *loop = birdloop_current(); + uint tc = timers_count(loop); + + if (!t->expires) + { + t->index = ++tc; + t->expires = when; + BUFFER_PUSH(loop->timers) = t; + HEAP_INSERT(loop->timers.data, tc, timer2 *, TIMER_LESS, TIMER_SWAP); + } + else if (t->expires < when) + { + t->expires = when; + HEAP_INCREASE(loop->timers.data, tc, timer2 *, TIMER_LESS, TIMER_SWAP, t->index); + } + else if (t->expires > when) + { + t->expires = when; + HEAP_DECREASE(loop->timers.data, tc, timer2 *, TIMER_LESS, TIMER_SWAP, t->index); + } + + if (loop->poll_active && (t->index == 1)) + wakeup_kick(loop); +} + +void +tm2_start(timer2 *t, btime after) +{ + tm2_set(t, current_time() + MAX(after, 0)); +} + +void +tm2_stop(timer2 *t) +{ + if (!t->expires) + return; + + struct birdloop *loop = birdloop_current(); + uint tc = timers_count(loop); + + HEAP_DELETE(loop->timers.data, tc, timer2 *, TIMER_LESS, TIMER_SWAP, t->index); + BUFFER_POP(loop->timers); + + t->index = -1; + t->expires = 0; +} + +static void +timers_init(struct birdloop *loop) +{ + BUFFER_INIT(loop->timers, loop->pool, 4); + BUFFER_PUSH(loop->timers) = NULL; +} + +static void +timers_fire(struct birdloop *loop) +{ + btime base_time; + timer2 *t; + + times_update(loop); + base_time = loop->last_time; + + while (t = timers_first(loop)) + { + if (t->expires > base_time) + return; + + if (t->recurrent) + { + btime when = t->expires + t->recurrent; + + if (when <= loop->last_time) + when = loop->last_time + t->recurrent; + + if (t->randomize) + when += random() % (t->randomize + 1); + + tm2_set(t, when); + } + else + tm2_stop(t); + + t->hook(t); + } +} + + +/* + * Sockets + */ + +static void +sockets_init(struct birdloop *loop) +{ + init_list(&loop->sock_list); + loop->sock_num = 0; + + BUFFER_INIT(loop->poll_sk, loop->pool, 4); + BUFFER_INIT(loop->poll_fd, loop->pool, 4); + loop->poll_changed = 1; /* add wakeup fd */ +} + +static void +sockets_add(struct birdloop *loop, sock *s) +{ + add_tail(&loop->sock_list, &s->n); + loop->sock_num++; + + s->index = -1; + loop->poll_changed = 1; + + if (loop->poll_active) + wakeup_kick(loop); +} + +void +sk_start(sock *s) +{ + struct birdloop *loop = birdloop_current(); + + sockets_add(loop, s); +} + +static void +sockets_remove(struct birdloop *loop, sock *s) +{ + rem_node(&s->n); + loop->sock_num--; + + if (s->index >= 0) + loop->poll_sk.data[s->index] = NULL; + + s->index = -1; + loop->poll_changed = 1; + + /* Wakeup moved to sk_stop() */ +} + +void +sk_stop(sock *s) +{ + struct birdloop *loop = birdloop_current(); + + sockets_remove(loop, s); + + if (loop->poll_active) + { + loop->close_scheduled = 1; + wakeup_kick(loop); + } + else + close(s->fd); + + s->fd = -1; +} + +static inline uint sk_want_events(sock *s) +{ return (s->rx_hook ? POLLIN : 0) | ((s->ttx != s->tpos) ? POLLOUT : 0); } + +/* +FIXME: this should be called from sock code + +static void +sockets_update(struct birdloop *loop, sock *s) +{ + if (s->index >= 0) + loop->poll_fd.data[s->index].events = sk_want_events(s); +} +*/ + +static void +sockets_prepare(struct birdloop *loop) +{ + BUFFER_SET(loop->poll_sk, loop->sock_num + 1); + BUFFER_SET(loop->poll_fd, loop->sock_num + 1); + + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + int i = 0; + node *n; + + WALK_LIST(n, loop->sock_list) + { + sock *s = SKIP_BACK(sock, n, n); + + ASSERT(i < loop->sock_num); + + s->index = i; + *psk = s; + pfd->fd = s->fd; + pfd->events = sk_want_events(s); + pfd->revents = 0; + + pfd++; + psk++; + i++; + } + + ASSERT(i == loop->sock_num); + + /* Add internal wakeup fd */ + *psk = NULL; + pfd->fd = loop->wakeup_fds[0]; + pfd->events = POLLIN; + pfd->revents = 0; + + loop->poll_changed = 0; +} + +static void +sockets_close_fds(struct birdloop *loop) +{ + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + int poll_num = loop->poll_fd.used - 1; + + int i; + for (i = 0; i < poll_num; i++) + if (psk[i] == NULL) + close(pfd[i].fd); + + loop->close_scheduled = 0; +} + +int sk_read(sock *s); +int sk_write(sock *s); + +static void +sockets_fire(struct birdloop *loop) +{ + struct pollfd *pfd = loop->poll_fd.data; + sock **psk = loop->poll_sk.data; + int poll_num = loop->poll_fd.used - 1; + + times_update(loop); + + /* Last fd is internal wakeup fd */ + if (pfd[loop->sock_num].revents & POLLIN) + wakeup_drain(loop); + + int i; + for (i = 0; i < poll_num; pfd++, psk++, i++) + { + int e = 1; + + if (! pfd->revents) + continue; + + if (pfd->revents & POLLNVAL) + die("poll: invalid fd %d", pfd->fd); + + if (pfd->revents & POLLIN) + while (e && *psk && (*psk)->rx_hook) + e = sk_read(*psk); + + e = 1; + if (pfd->revents & POLLOUT) + while (e && *psk) + e = sk_write(*psk); + } +} + + +/* + * Birdloop + */ + +static void * birdloop_main(void *arg); + +struct birdloop * +birdloop_new(void) +{ + /* FIXME: this init should be elsewhere and thread-safe */ + static int init = 0; + if (!init) + { birdloop_init_current(); init = 1; } + + pool *p = rp_new(NULL, "Birdloop root"); + struct birdloop *loop = mb_allocz(p, sizeof(struct birdloop)); + loop->pool = p; + pthread_mutex_init(&loop->mutex, NULL); + + times_init(loop); + wakeup_init(loop); + + events_init(loop); + timers_init(loop); + sockets_init(loop); + + return loop; +} + +void +birdloop_start(struct birdloop *loop) +{ + int rv = pthread_create(&loop->thread, NULL, birdloop_main, loop); + if (rv) + die("pthread_create(): %M", rv); +} + +void +birdloop_stop(struct birdloop *loop) +{ + pthread_mutex_lock(&loop->mutex); + loop->stop_called = 1; + wakeup_do_kick(loop); + pthread_mutex_unlock(&loop->mutex); + + int rv = pthread_join(loop->thread, NULL); + if (rv) + die("pthread_join(): %M", rv); +} + +void +birdloop_free(struct birdloop *loop) +{ + rfree(loop->pool); +} + + +void +birdloop_enter(struct birdloop *loop) +{ + /* TODO: these functions could save and restore old context */ + pthread_mutex_lock(&loop->mutex); + birdloop_set_current(loop); +} + +void +birdloop_leave(struct birdloop *loop) +{ + /* TODO: these functions could save and restore old context */ + birdloop_set_current(NULL); + pthread_mutex_unlock(&loop->mutex); +} + +void +birdloop_mask_wakeups(struct birdloop *loop) +{ + pthread_mutex_lock(&loop->mutex); + loop->wakeup_masked = 1; + pthread_mutex_unlock(&loop->mutex); +} + +void +birdloop_unmask_wakeups(struct birdloop *loop) +{ + pthread_mutex_lock(&loop->mutex); + if (loop->wakeup_masked == 2) + wakeup_do_kick(loop); + loop->wakeup_masked = 0; + pthread_mutex_unlock(&loop->mutex); +} + +static void * +birdloop_main(void *arg) +{ + struct birdloop *loop = arg; + timer2 *t; + int rv, timeout; + + birdloop_set_current(loop); + + pthread_mutex_lock(&loop->mutex); + while (1) + { + events_fire(loop); + timers_fire(loop); + + times_update(loop); + if (events_waiting(loop)) + timeout = 0; + else if (t = timers_first(loop)) + timeout = (tm2_remains(t) TO_MS) + 1; + else + timeout = -1; + + if (loop->poll_changed) + sockets_prepare(loop); + + loop->poll_active = 1; + pthread_mutex_unlock(&loop->mutex); + + try: + rv = poll(loop->poll_fd.data, loop->poll_fd.used, timeout); + if (rv < 0) + { + if (errno == EINTR || errno == EAGAIN) + goto try; + die("poll: %m"); + } + + pthread_mutex_lock(&loop->mutex); + loop->poll_active = 0; + + if (loop->close_scheduled) + sockets_close_fds(loop); + + if (loop->stop_called) + break; + + if (rv) + sockets_fire(loop); + + timers_fire(loop); + } + + loop->stop_called = 0; + pthread_mutex_unlock(&loop->mutex); + + return NULL; +} + + diff --git a/proto/bfd/io.h b/proto/bfd/io.h new file mode 100644 index 00000000..641ee054 --- /dev/null +++ b/proto/bfd/io.h @@ -0,0 +1,99 @@ +/* + * BIRD -- I/O and event loop + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#ifndef _BIRD_BFD_IO_H_ +#define _BIRD_BFD_IO_H_ + +#include "nest/bird.h" +#include "lib/lists.h" +#include "lib/resource.h" +#include "lib/event.h" +#include "lib/socket.h" +// #include "lib/timer.h" + + +typedef struct timer2 +{ + resource r; + void (*hook)(struct timer2 *); + void *data; + + btime expires; /* 0=inactive */ + uint randomize; /* Amount of randomization */ + uint recurrent; /* Timer recurrence */ + + int index; +} timer2; + + +btime current_time(void); + +void ev2_schedule(event *e); + + +timer2 *tm2_new(pool *p); +void tm2_set(timer2 *t, btime when); +void tm2_start(timer2 *t, btime after); +void tm2_stop(timer2 *t); + +static inline int +tm2_active(timer2 *t) +{ + return t->expires != 0; +} + +static inline btime +tm2_remains(timer2 *t) +{ + btime now = current_time(); + return (t->expires > now) ? (t->expires - now) : 0; +} + +static inline timer2 * +tm2_new_init(pool *p, void (*hook)(struct timer2 *), void *data, uint rec, uint rand) +{ + timer2 *t = tm2_new(p); + t->hook = hook; + t->data = data; + t->recurrent = rec; + t->randomize = rand; + return t; +} + +static inline void +tm2_set_max(timer2 *t, btime when) +{ + if (when > t->expires) + tm2_set(t, when); +} + +/* +static inline void +tm2_start_max(timer2 *t, btime after) +{ + btime rem = tm2_remains(t); + tm2_start(t, MAX_(rem, after)); +} +*/ + + +void sk_start(sock *s); +void sk_stop(sock *s); + + + +struct birdloop *birdloop_new(void); +void birdloop_start(struct birdloop *loop); +void birdloop_stop(struct birdloop *loop); +void birdloop_free(struct birdloop *loop); + +void birdloop_enter(struct birdloop *loop); +void birdloop_leave(struct birdloop *loop); +void birdloop_mask_wakeups(struct birdloop *loop); +void birdloop_unmask_wakeups(struct birdloop *loop); + + +#endif /* _BIRD_BFD_IO_H_ */ diff --git a/proto/bfd/packets.c b/proto/bfd/packets.c new file mode 100644 index 00000000..fc2616ca --- /dev/null +++ b/proto/bfd/packets.c @@ -0,0 +1,248 @@ +/* + * BIRD -- Bidirectional Forwarding Detection (BFD) + * + * Can be freely distributed and used under the terms of the GNU GPL. + */ + +#include "bfd.h" + + +struct bfd_ctl_packet +{ + u8 vdiag; /* version and diagnostic */ + u8 flags; /* state and flags */ + u8 detect_mult; + u8 length; + u32 snd_id; /* sender ID, aka 'my discriminator' */ + u32 rcv_id; /* receiver ID, aka 'your discriminator' */ + u32 des_min_tx_int; + u32 req_min_rx_int; + u32 req_min_echo_rx_int; +}; + +#define BFD_BASE_LEN sizeof(struct bfd_ctl_packet) +#define BFD_MAX_LEN 64 + +static inline u8 bfd_pack_vdiag(u8 version, u8 diag) +{ return (version << 5) | diag; } + +static inline u8 bfd_pack_flags(u8 state, u8 flags) +{ return (state << 6) | flags; } + +static inline u8 bfd_pkt_get_version(struct bfd_ctl_packet *pkt) +{ return pkt->vdiag >> 5; } + +static inline u8 bfd_pkt_get_diag(struct bfd_ctl_packet *pkt) +{ return pkt->vdiag && 0x1f; } + + +static inline u8 bfd_pkt_get_state(struct bfd_ctl_packet *pkt) +{ return pkt->flags >> 6; } + +static inline void bfd_pkt_set_state(struct bfd_ctl_packet *pkt, u8 val) +{ pkt->flags = val << 6; } + + +char * +bfd_format_flags(u8 flags, char *buf) +{ + char *bp = buf; + if (flags & BFD_FLAGS) *bp++ = ' '; + if (flags & BFD_FLAG_POLL) *bp++ = 'P'; + if (flags & BFD_FLAG_FINAL) *bp++ = 'F'; + if (flags & BFD_FLAG_CPI) *bp++ = 'C'; + if (flags & BFD_FLAG_AP) *bp++ = 'A'; + if (flags & BFD_FLAG_DEMAND) *bp++ = 'D'; + if (flags & BFD_FLAG_MULTIPOINT) *bp++ = 'M'; + *bp = 0; + + return buf; +} + +void +bfd_send_ctl(struct bfd_proto *p, struct bfd_session *s, int final) +{ + sock *sk = s->ifa->sk; + struct bfd_ctl_packet *pkt = (struct bfd_ctl_packet *) sk->tbuf; + char fb[8]; + + pkt->vdiag = bfd_pack_vdiag(1, s->loc_diag); + pkt->flags = bfd_pack_flags(s->loc_state, 0); + pkt->detect_mult = s->detect_mult; + pkt->length = BFD_BASE_LEN; + pkt->snd_id = htonl(s->loc_id); + pkt->rcv_id = htonl(s->rem_id); + pkt->des_min_tx_int = htonl(s->des_min_tx_new); + pkt->req_min_rx_int = htonl(s->req_min_rx_new); + pkt->req_min_echo_rx_int = 0; + + if (final) + pkt->flags |= BFD_FLAG_FINAL; + else if (s->poll_active) + pkt->flags |= BFD_FLAG_POLL; + + if (sk->tbuf != sk->tpos) + log(L_WARN "%s: Old packet overwritten in TX buffer", p->p.name); + + TRACE(D_PACKETS, "Sending CTL to %I [%s%s]", s->addr, + bfd_state_names[s->loc_state], bfd_format_flags(pkt->flags, fb)); + + sk_send_to(sk, pkt->length, s->addr, sk->dport); +} + +#define DROP(DSC,VAL) do { err_dsc = DSC; err_val = VAL; goto drop; } while(0) + +static int +bfd_rx_hook(sock *sk, int len) +{ + struct bfd_proto *p = sk->data; + struct bfd_ctl_packet *pkt = (struct bfd_ctl_packet *) sk->rbuf; + const char *err_dsc = NULL; + uint err_val = 0; + char fb[8]; + + if ((sk->sport == BFD_CONTROL_PORT) && (sk->ttl < 255)) + DROP("wrong TTL", sk->ttl); + + if (len < BFD_BASE_LEN) + DROP("too short", len); + + u8 version = bfd_pkt_get_version(pkt); + if (version != 1) + DROP("version mismatch", version); + + if ((pkt->length < BFD_BASE_LEN) || (pkt->length > len)) + DROP("length mismatch", pkt->length); + + if (pkt->detect_mult == 0) + DROP("invalid detect mult", 0); + + if ((pkt->flags & BFD_FLAG_MULTIPOINT) || + ((pkt->flags & BFD_FLAG_POLL) && (pkt->flags & BFD_FLAG_FINAL))) + DROP("invalid flags", pkt->flags); + + if (pkt->snd_id == 0) + DROP("invalid my discriminator", 0); + + struct bfd_session *s; + u32 id = ntohl(pkt->rcv_id); + + if (id) + { + s = bfd_find_session_by_id(p, id); + + if (!s) + DROP("unknown session id", id); + } + else + { + u8 ps = bfd_pkt_get_state(pkt); + if (ps > BFD_STATE_DOWN) + DROP("invalid init state", ps); + + s = bfd_find_session_by_addr(p, sk->faddr); + + /* FIXME: better session matching and message */ + if (!s) + return 1; + } + + /* FIXME: better authentication handling and message */ + if (pkt->flags & BFD_FLAG_AP) + DROP("authentication not supported", 0); + + + u32 old_tx_int = s->des_min_tx_int; + u32 old_rx_int = s->rem_min_rx_int; + + s->rem_id= ntohl(pkt->snd_id); + s->rem_state = bfd_pkt_get_state(pkt); + s->rem_diag = bfd_pkt_get_diag(pkt); + s->rem_demand_mode = pkt->flags & BFD_FLAG_DEMAND; + s->rem_min_tx_int = ntohl(pkt->des_min_tx_int); + s->rem_min_rx_int = ntohl(pkt->req_min_rx_int); + s->rem_detect_mult = pkt->detect_mult; + + TRACE(D_PACKETS, "CTL received from %I [%s%s]", sk->faddr, + bfd_state_names[s->rem_state], bfd_format_flags(pkt->flags, fb)); + + bfd_session_process_ctl(s, pkt->flags, old_tx_int, old_rx_int); + return 1; + + drop: + log(L_REMOTE "%s: Bad packet from %I - %s (%u)", p->p.name, sk->faddr, err_dsc, err_val); + return 1; +} + +static void +bfd_err_hook(sock *sk, int err) +{ + struct bfd_proto *p = sk->data; + log(L_ERR "%s: Socket error: %m", p->p.name, err); +} + +sock * +bfd_open_rx_sk(struct bfd_proto *p, int multihop) +{ + sock *sk = sk_new(p->tpool); + sk->type = SK_UDP; + sk->sport = !multihop ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; + sk->data = p; + + sk->rbsize = BFD_MAX_LEN; + sk->rx_hook = bfd_rx_hook; + sk->err_hook = bfd_err_hook; + + /* TODO: configurable ToS and priority */ + sk->tos = IP_PREC_INTERNET_CONTROL; + sk->priority = sk_priority_control; + sk->flags = SKF_THREAD | SKF_LADDR_RX | (!multihop ? SKF_TTL_RX : 0); + +#ifdef IPV6 + sk->flags |= SKF_V6ONLY; +#endif + + if (sk_open(sk) < 0) + goto err; + + sk_start(sk); + return sk; + + err: + rfree(sk); + return NULL; +} + +sock * +bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa) +{ + sock *sk = sk_new(p->tpool); + sk->type = SK_UDP; + sk->saddr = local; + sk->dport = ifa ? BFD_CONTROL_PORT : BFD_MULTI_CTL_PORT; + sk->iface = ifa; + sk->data = p; + + sk->tbsize = BFD_MAX_LEN; + sk->err_hook = bfd_err_hook; + + /* TODO: configurable ToS, priority and TTL security */ + sk->tos = IP_PREC_INTERNET_CONTROL; + sk->priority = sk_priority_control; + sk->ttl = ifa ? 255 : -1; + sk->flags = SKF_THREAD; + +#ifdef IPV6 + sk->flags |= SKF_V6ONLY; +#endif + + if (sk_open(sk) < 0) + goto err; + + sk_start(sk); + return sk; + + err: + rfree(sk); + return NULL; +} |