diff options
| -rw-r--r-- | src/allowedips.c | 6 | ||||
| -rw-r--r-- | src/compat/compat.h | 33 | ||||
| -rw-r--r-- | src/device.c | 6 | ||||
| -rw-r--r-- | src/noise.c | 2 | ||||
| -rw-r--r-- | src/peer.c | 8 | ||||
| -rw-r--r-- | src/ratelimiter.c | 12 | ||||
| -rw-r--r-- | src/socket.c | 2 |
7 files changed, 57 insertions, 12 deletions
diff --git a/src/allowedips.c b/src/allowedips.c index bfb6020..592501e 100644 --- a/src/allowedips.c +++ b/src/allowedips.c @@ -112,7 +112,7 @@ static void walk_remove_by_peer(struct allowedips_node __rcu **top, if (!node->bit[0] || !node->bit[1]) { rcu_assign_pointer(*nptr, DEREF( &node->bit[!REF(node->bit[0])])); - call_rcu_bh(&node->rcu, node_free_rcu); + call_rcu(&node->rcu, node_free_rcu); node = DEREF(nptr); } } @@ -300,12 +300,12 @@ void wg_allowedips_free(struct allowedips *table, struct mutex *lock) RCU_INIT_POINTER(table->root6, NULL); if (rcu_access_pointer(old4)) { root_remove_peer_lists(old4); - call_rcu_bh(&rcu_dereference_protected(old4, + call_rcu(&rcu_dereference_protected(old4, lockdep_is_held(lock))->rcu, root_free_rcu); } if (rcu_access_pointer(old6)) { root_remove_peer_lists(old6); - call_rcu_bh(&rcu_dereference_protected(old6, + call_rcu(&rcu_dereference_protected(old6, lockdep_is_held(lock))->rcu, root_free_rcu); } } diff --git a/src/compat/compat.h b/src/compat/compat.h index 2dcdbae..396a293 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -778,6 +778,39 @@ struct __kernel_timespec { #define skb_probe_transport_header(a) skb_probe_transport_header(a, 0) #endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +/* Note that all intentional uses of the non-_bh variety need to explicitly + * undef these, conditionalized on COMPAT_CANNOT_DEPRECIATE_BH_RCU. + */ +#include <linux/rcupdate.h> +static __always_inline void old_synchronize_rcu(void) +{ + synchronize_rcu(); +} +static __always_inline void old_call_rcu(void *a, void *b) +{ + call_rcu(a, b); +} +static __always_inline void old_rcu_barrier(void) +{ + rcu_barrier(); +} +#ifdef synchronize_rcu +#undef synchronize_rcu +#endif +#ifdef call_rcu +#undef call_rcu +#endif +#ifdef rcu_barrier +#undef rcu_barrier +#endif +#define synchronize_rcu synchronize_rcu_bh +#define call_rcu call_rcu_bh +#define rcu_barrier rcu_barrier_bh +#define COMPAT_CANNOT_DEPRECIATE_BH_RCU +#endif + /* https://github.com/ClangBuiltLinux/linux/issues/7 */ #if defined( __clang__) && (!defined(CONFIG_CLANG_VERSION) || CONFIG_CLANG_VERSION < 80000) #include <linux/bug.h> diff --git a/src/device.c b/src/device.c index 2866dd9..779c415 100644 --- a/src/device.c +++ b/src/device.c @@ -94,7 +94,7 @@ static int wg_pm_notification(struct notifier_block *nb, unsigned long action, mutex_unlock(&wg->device_update_lock); } rtnl_unlock(); - rcu_barrier_bh(); + rcu_barrier(); return 0; } @@ -244,7 +244,7 @@ static void wg_destruct(struct net_device *dev) destroy_workqueue(wg->packet_crypt_wq); wg_packet_queue_free(&wg->decrypt_queue, true); wg_packet_queue_free(&wg->encrypt_queue, true); - rcu_barrier_bh(); /* Wait for all the peers to be actually freed. */ + rcu_barrier(); /* Wait for all the peers to be actually freed. */ wg_ratelimiter_uninit(); memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); skb_queue_purge(&wg->incoming_handshakes); @@ -468,5 +468,5 @@ void wg_device_uninit(void) #ifdef CONFIG_PM_SLEEP unregister_pm_notifier(&pm_notifier); #endif - rcu_barrier_bh(); + rcu_barrier(); } diff --git a/src/noise.c b/src/noise.c index 4405125..2e05e27 100644 --- a/src/noise.c +++ b/src/noise.c @@ -132,7 +132,7 @@ static void keypair_free_kref(struct kref *kref) keypair->entry.peer->internal_id); wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, &keypair->entry); - call_rcu_bh(&keypair->rcu, keypair_free_rcu); + call_rcu(&keypair->rcu, keypair_free_rcu); } void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now) @@ -99,7 +99,7 @@ static void peer_make_dead(struct wg_peer *peer) /* Mark as dead, so that we don't allow jumping contexts after. */ WRITE_ONCE(peer->is_dead, true); - /* The caller must now synchronize_rcu_bh() for this to take effect. */ + /* The caller must now synchronize_rcu() for this to take effect. */ } static void peer_remove_after_dead(struct wg_peer *peer) @@ -171,7 +171,7 @@ void wg_peer_remove(struct wg_peer *peer) lockdep_assert_held(&peer->device->device_update_lock); peer_make_dead(peer); - synchronize_rcu_bh(); + synchronize_rcu(); peer_remove_after_dead(peer); } @@ -189,7 +189,7 @@ void wg_peer_remove_all(struct wg_device *wg) peer_make_dead(peer); list_add_tail(&peer->peer_list, &dead_peers); } - synchronize_rcu_bh(); + synchronize_rcu(); list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) peer_remove_after_dead(peer); } @@ -228,7 +228,7 @@ static void kref_release(struct kref *refcount) wg_packet_purge_staged_packets(peer); /* Free the memory used. */ - call_rcu_bh(&peer->rcu, rcu_release); + call_rcu(&peer->rcu, rcu_release); } void wg_peer_put(struct wg_peer *peer) diff --git a/src/ratelimiter.c b/src/ratelimiter.c index b3b1d25..fd09190 100644 --- a/src/ratelimiter.c +++ b/src/ratelimiter.c @@ -3,6 +3,18 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. */ +#ifdef COMPAT_CANNOT_DEPRECIATE_BH_RCU +/* We normally alias all non-_bh functions to the _bh ones in the compat layer, + * but that's not appropriate here, where we actually do want non-_bh ones. + */ +#undef synchronize_rcu +#define synchronize_rcu old_synchronize_rcu +#undef call_rcu +#define call_rcu old_call_rcu +#undef rcu_barrier +#define rcu_barrier old_rcu_barrier +#endif + #include "ratelimiter.h" #include <linux/siphash.h> #include <linux/mm.h> diff --git a/src/socket.c b/src/socket.c index 652d798..5a77b0c 100644 --- a/src/socket.c +++ b/src/socket.c @@ -426,7 +426,7 @@ void wg_socket_reinit(struct wg_device *wg, struct sock *new4, if (new4) wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); mutex_unlock(&wg->socket_update_lock); - synchronize_rcu_bh(); + synchronize_rcu(); synchronize_net(); sock_free(old4); sock_free(old6); |