data: we care about per-peer, not per-device, inflight encryptions

Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>

3 files changed, 14 insertions, 21 deletions

diff --git a/src/compat.h b/src/compat.h
index 3c73207..28c1aca 100644
--- a/src/compat.h
+++ b/src/compat.h
@@ -141,26 +141,6 @@ static inline int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp) { retur
 static inline void dst_cache_destroy(struct dst_cache *dst_cache) { }
 #endif
 
-/* https://lkml.org/lkml/2015/6/12/415 */
-#include <linux/netdevice.h>
-static inline struct net_device *netdev_pub(void *dev)
-{
-	return (struct net_device *)((char *)dev - ALIGN(sizeof(struct net_device), NETDEV_ALIGN));
-}
-
-/* https://lkml.org/lkml/2016/10/1/187 */
-#ifdef CONFIG_WIREGUARD_PARALLEL
-#include <linux/padata.h>
-static inline int padata_queue_len(struct padata_instance *pinst)
-{
-	int len;
-	rcu_read_lock_bh();
-	len = atomic_read(&rcu_dereference_bh(pinst->pd)->refcnt);
-	rcu_read_unlock_bh();
-	return len;
-}
-#endif
-
 /* PaX compatibility */
 #ifdef CONSTIFY_PLUGIN
 #include <linux/cache.h>
@@ -168,4 +148,11 @@ static inline int padata_queue_len(struct padata_instance *pinst)
 #define __read_mostly
 #endif
 
+/* https://lkml.org/lkml/2015/6/12/415 */
+#include <linux/netdevice.h>
+static inline struct net_device *netdev_pub(void *dev)
+{
+	return (struct net_device *)((char *)dev - ALIGN(sizeof(struct net_device), NETDEV_ALIGN));
+}
+
 #endif
diff --git a/src/data.c b/src/data.c
index e027510..a3ed739 100644
--- a/src/data.c
+++ b/src/data.c
@@ -215,6 +215,7 @@ static void finish_encryption(struct padata_priv *padata)
 	struct encryption_ctx *ctx = container_of(padata, struct encryption_ctx, padata);
 
 	ctx->callback(&ctx->queue, ctx->peer);
+	atomic_dec(&ctx->peer->parallel_encryption_inflight);
 	peer_put(ctx->peer);
 	kmem_cache_free(encryption_ctx_cache, ctx);
 }
@@ -295,7 +296,7 @@ int packet_create_data(struct sk_buff_head *queue, struct wireguard_peer *peer,
 	}
 
 #ifdef CONFIG_WIREGUARD_PARALLEL
-	if ((skb_queue_len(queue) > 1 || queue->next->len > 256 || padata_queue_len(peer->device->parallel_send) > 0) && cpumask_weight(cpu_online_mask) > 1) {
+	if ((skb_queue_len(queue) > 1 || queue->next->len > 256 || atomic_read(&peer->parallel_encryption_inflight) > 0) && cpumask_weight(cpu_online_mask) > 1) {
 		unsigned int cpu = choose_cpu(keypair->remote_index);
 		struct encryption_ctx *ctx = kmem_cache_alloc(encryption_ctx_cache, GFP_ATOMIC);
 		if (!ctx)
@@ -308,8 +309,10 @@ int packet_create_data(struct sk_buff_head *queue, struct wireguard_peer *peer,
 		ret = -EBUSY;
 		if (unlikely(!ctx->peer))
 			goto err_parallel;
+		atomic_inc(&peer->parallel_encryption_inflight);
 		ret = start_encryption(peer->device->parallel_send, &ctx->padata, cpu);
 		if (unlikely(ret < 0)) {
+			atomic_dec(&peer->parallel_encryption_inflight);
 			peer_put(ctx->peer);
 err_parallel:
 			skb_queue_splice(&ctx->queue, queue);
diff --git a/src/peer.h b/src/peer.h
index f68d7f0..895eb77 100644
--- a/src/peer.h
+++ b/src/peer.h
@@ -40,6 +40,9 @@ struct wireguard_peer {
 	struct rcu_head rcu;
 	struct list_head peer_list;
 	uint64_t internal_id;
+#ifdef CONFIG_WIREGUARD_PARALLEL
+	atomic_t parallel_encryption_inflight;
+#endif
 };
 
 struct wireguard_peer *peer_create(struct wireguard_device *wg, const u8 public_key[NOISE_PUBLIC_KEY_LEN]);