kernel: add support for threaded network backlog processing

This can improve load balancing by pushing backlog (and RPS) processing
to separate threads, allowing the scheduler to distribute the load.
It can be enabled with: echo 1 > /proc/sys/net/core/backlog_threaded

Signed-off-by: Felix Fietkau <nbd@nbd.name>

1 files changed, 224 insertions, 0 deletions

diff --git a/target/linux/generic/pending-5.15/760-net-core-add-optional-threading-for-backlog-processi.patch b/target/linux/generic/pending-5.15/760-net-core-add-optional-threading-for-backlog-processi.patch
new file mode 100644
index 0000000000..463f405f3a
--- /dev/null
+++ b/target/linux/generic/pending-5.15/760-net-core-add-optional-threading-for-backlog-processi.patch
@@ -0,0 +1,224 @@
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 16 Feb 2023 18:39:04 +0100
+Subject: [PATCH] net/core: add optional threading for backlog processing
+
+When dealing with few flows or an imbalance on CPU utilization, static RPS
+CPU assignment can be too inflexible. Add support for enabling threaded NAPI
+for backlog processing in order to allow the scheduler to better balance
+processing. This helps better spread the load across idle CPUs.
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+---
+
+--- a/include/linux/netdevice.h
++++ b/include/linux/netdevice.h
+@@ -500,6 +500,7 @@ static inline bool napi_complete(struct
+ }
+ 
+ int dev_set_threaded(struct net_device *dev, bool threaded);
++int backlog_set_threaded(bool threaded);
+ 
+ /**
+  *	napi_disable - prevent NAPI from scheduling
+@@ -3363,6 +3364,7 @@ struct softnet_data {
+ 	unsigned int		processed;
+ 	unsigned int		time_squeeze;
+ 	unsigned int		received_rps;
++	unsigned int		process_queue_empty;
+ #ifdef CONFIG_RPS
+ 	struct softnet_data	*rps_ipi_list;
+ #endif
+--- a/net/core/dev.c
++++ b/net/core/dev.c
+@@ -4574,7 +4574,7 @@ static int rps_ipi_queued(struct softnet
+ #ifdef CONFIG_RPS
+ 	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
+ 
+-	if (sd != mysd) {
++	if (sd != mysd && !test_bit(NAPI_STATE_THREADED, &sd->backlog.state)) {
+ 		sd->rps_ipi_next = mysd->rps_ipi_list;
+ 		mysd->rps_ipi_list = sd;
+ 
+@@ -5755,6 +5755,8 @@ static DEFINE_PER_CPU(struct work_struct
+ /* Network device is going away, flush any packets still pending */
+ static void flush_backlog(struct work_struct *work)
+ {
++	unsigned int process_queue_empty;
++	bool threaded, flush_processq;
+ 	struct sk_buff *skb, *tmp;
+ 	struct softnet_data *sd;
+ 
+@@ -5770,9 +5772,18 @@ static void flush_backlog(struct work_st
+ 			input_queue_head_incr(sd);
+ 		}
+ 	}
++
++	threaded = test_bit(NAPI_STATE_THREADED, &sd->backlog.state);
++	flush_processq = threaded &&
++			 !skb_queue_empty_lockless(&sd->process_queue);
++	if (flush_processq)
++		process_queue_empty = sd->process_queue_empty;
+ 	rps_unlock(sd);
+ 	local_irq_enable();
+ 
++	if (threaded)
++		goto out;
++
+ 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
+ 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
+ 			__skb_unlink(skb, &sd->process_queue);
+@@ -5780,7 +5791,18 @@ static void flush_backlog(struct work_st
+ 			input_queue_head_incr(sd);
+ 		}
+ 	}
++
++out:
+ 	local_bh_enable();
++
++	while (flush_processq) {
++		msleep(1);
++		local_irq_disable();
++		rps_lock(sd);
++		flush_processq = process_queue_empty == sd->process_queue_empty;
++		rps_unlock(sd);
++		local_irq_enable();
++	}
+ }
+ 
+ static bool flush_required(int cpu)
+@@ -6463,6 +6485,7 @@ static int process_backlog(struct napi_s
+ 
+ 		local_irq_disable();
+ 		rps_lock(sd);
++		sd->process_queue_empty++;
+ 		if (skb_queue_empty(&sd->input_pkt_queue)) {
+ 			/*
+ 			 * Inline a custom version of __napi_complete().
+@@ -6472,7 +6495,8 @@ static int process_backlog(struct napi_s
+ 			 * We can use a plain write instead of clear_bit(),
+ 			 * and we dont need an smp_mb() memory barrier.
+ 			 */
+-			napi->state = 0;
++			napi->state &= ~(NAPIF_STATE_SCHED |
++					 NAPIF_STATE_SCHED_THREADED);
+ 			again = false;
+ 		} else {
+ 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
+@@ -6889,6 +6913,57 @@ int dev_set_threaded(struct net_device *
+ }
+ EXPORT_SYMBOL(dev_set_threaded);
+ 
++int backlog_set_threaded(bool threaded)
++{
++	static bool backlog_threaded;
++	int err = 0;
++	int i;
++
++	if (backlog_threaded == threaded)
++		return 0;
++
++	for_each_possible_cpu(i) {
++		struct softnet_data *sd = &per_cpu(softnet_data, i);
++		struct napi_struct *n = &sd->backlog;
++
++		if (n->thread)
++			continue;
++		n->thread = kthread_run(napi_threaded_poll, n, "napi/backlog-%d", i);
++		if (IS_ERR(n->thread)) {
++			err = PTR_ERR(n->thread);
++			pr_err("kthread_run failed with err %d\n", err);
++			n->thread = NULL;
++			threaded = false;
++			break;
++		}
++
++	}
++
++	backlog_threaded = threaded;
++
++	/* Make sure kthread is created before THREADED bit
++	 * is set.
++	 */
++	smp_mb__before_atomic();
++
++	for_each_possible_cpu(i) {
++		struct softnet_data *sd = &per_cpu(softnet_data, i);
++		struct napi_struct *n = &sd->backlog;
++		unsigned long flags;
++
++		local_irq_save(flags);
++		rps_lock(sd);
++		if (threaded)
++			n->state |= NAPIF_STATE_THREADED;
++		else
++			n->state &= ~NAPIF_STATE_THREADED;
++		rps_unlock(sd);
++		local_irq_restore(flags);
++	}
++
++	return err;
++}
++
+ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
+ 		    int (*poll)(struct napi_struct *, int), int weight)
+ {
+@@ -11367,6 +11442,9 @@ static int dev_cpu_dead(unsigned int old
+ 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
+ 	local_irq_enable();
+ 
++	if (test_bit(NAPI_STATE_THREADED, &oldsd->backlog.state))
++		return 0;
++
+ #ifdef CONFIG_RPS
+ 	remsd = oldsd->rps_ipi_list;
+ 	oldsd->rps_ipi_list = NULL;
+--- a/net/core/sysctl_net_core.c
++++ b/net/core/sysctl_net_core.c
+@@ -28,6 +28,7 @@ static int int_3600 = 3600;
+ static int min_sndbuf = SOCK_MIN_SNDBUF;
+ static int min_rcvbuf = SOCK_MIN_RCVBUF;
+ static int max_skb_frags = MAX_SKB_FRAGS;
++static int backlog_threaded;
+ static long long_one __maybe_unused = 1;
+ static long long_max __maybe_unused = LONG_MAX;
+ 
+@@ -114,6 +115,23 @@ static int rps_sock_flow_sysctl(struct c
+ }
+ #endif /* CONFIG_RPS */
+ 
++static int backlog_threaded_sysctl(struct ctl_table *table, int write,
++			       void *buffer, size_t *lenp, loff_t *ppos)
++{
++	static DEFINE_MUTEX(backlog_threaded_mutex);
++	int ret;
++
++	mutex_lock(&backlog_threaded_mutex);
++
++	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
++	if (write && !ret)
++		ret = backlog_set_threaded(backlog_threaded);
++
++	mutex_unlock(&backlog_threaded_mutex);
++
++	return ret;
++}
++
+ #ifdef CONFIG_NET_FLOW_LIMIT
+ static DEFINE_MUTEX(flow_limit_update_mutex);
+ 
+@@ -470,6 +488,15 @@ static struct ctl_table net_core_table[]
+ 		.proc_handler	= rps_sock_flow_sysctl
+ 	},
+ #endif
++	{
++		.procname	= "backlog_threaded",
++		.data		= &backlog_threaded,
++		.maxlen		= sizeof(unsigned int),
++		.mode		= 0644,
++		.proc_handler	= backlog_threaded_sysctl,
++		.extra1		= SYSCTL_ZERO,
++		.extra2		= SYSCTL_ONE
++	},
+ #ifdef CONFIG_NET_FLOW_LIMIT
+ 	{
+ 		.procname	= "flow_limit_cpu_bitmap",