generic: net: tcp: backport tcp tx performance patches

An overall throughput gain of 22 % for heavy TCP use over a single TX queue.

Original patchset comment
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v4.13&id=3f4888adae7c1619b990d98a9b967536f71822b8

Signed-off-by: Pavel Kubelun <be.dissent@gmail.com>

1 files changed, 157 insertions, 0 deletions

diff --git a/target/linux/generic/backport-4.9/024-7-net-reorganize-struct-sock-for-better-data-locality.patch b/target/linux/generic/backport-4.9/024-7-net-reorganize-struct-sock-for-better-data-locality.patch
new file mode 100644
index 0000000000..e8c1915e18
--- /dev/null
+++ b/target/linux/generic/backport-4.9/024-7-net-reorganize-struct-sock-for-better-data-locality.patch
@@ -0,0 +1,157 @@
+From 9115e8cd2a0c6eaaa900c462721f12e1d45f326c Mon Sep 17 00:00:00 2001
+From: Eric Dumazet <edumazet@google.com>
+Date: Sat, 3 Dec 2016 11:14:56 -0800
+Subject: [PATCH 07/10] net: reorganize struct sock for better data locality
+
+Group fields used in TX path, and keep some cache lines mostly read
+to permit sharing among cpus.
+
+Gained two 4 bytes holes on 64bit arches.
+
+Added a place holder for tcp tsq_flags, next to sk_wmem_alloc
+to speed up tcp_wfree() in the following patch.
+
+I have not added ____cacheline_aligned_in_smp, this might be done later.
+I prefer doing this once inet and tcp/udp sockets reorg is also done.
+
+Tested with both TCP and UDP.
+
+UDP receiver performance under flood increased by ~20 % :
+Accessing sk_filter/sk_wq/sk_napi_id no longer stalls because sk_drops
+was moved away from a critical cache line, now mostly read and shared.
+
+	/* --- cacheline 4 boundary (256 bytes) --- */
+	unsigned int               sk_napi_id;           /* 0x100   0x4 */
+	int                        sk_rcvbuf;            /* 0x104   0x4 */
+	struct sk_filter *         sk_filter;            /* 0x108   0x8 */
+	union {
+		struct socket_wq * sk_wq;                /*         0x8 */
+		struct socket_wq * sk_wq_raw;            /*         0x8 */
+	};                                               /* 0x110   0x8 */
+	struct xfrm_policy *       sk_policy[2];         /* 0x118  0x10 */
+	struct dst_entry *         sk_rx_dst;            /* 0x128   0x8 */
+	struct dst_entry *         sk_dst_cache;         /* 0x130   0x8 */
+	atomic_t                   sk_omem_alloc;        /* 0x138   0x4 */
+	int                        sk_sndbuf;            /* 0x13c   0x4 */
+	/* --- cacheline 5 boundary (320 bytes) --- */
+	int                        sk_wmem_queued;       /* 0x140   0x4 */
+	atomic_t                   sk_wmem_alloc;        /* 0x144   0x4 */
+	long unsigned int          sk_tsq_flags;         /* 0x148   0x8 */
+	struct sk_buff *           sk_send_head;         /* 0x150   0x8 */
+	struct sk_buff_head        sk_write_queue;       /* 0x158  0x18 */
+	__s32                      sk_peek_off;          /* 0x170   0x4 */
+	int                        sk_write_pending;     /* 0x174   0x4 */
+	long int                   sk_sndtimeo;          /* 0x178   0x8 */
+
+Signed-off-by: Eric Dumazet <edumazet@google.com>
+Tested-by: Paolo Abeni <pabeni@redhat.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ include/net/sock.h | 51 +++++++++++++++++++++++++++------------------------
+ 1 file changed, 27 insertions(+), 24 deletions(-)
+
+--- a/include/net/sock.h
++++ b/include/net/sock.h
+@@ -343,6 +343,9 @@ struct sock {
+ #define sk_rxhash		__sk_common.skc_rxhash
+ 
+ 	socket_lock_t		sk_lock;
++	atomic_t		sk_drops;
++	int			sk_rcvlowat;
++	struct sk_buff_head	sk_error_queue;
+ 	struct sk_buff_head	sk_receive_queue;
+ 	/*
+ 	 * The backlog queue is special, it is always used with
+@@ -359,14 +362,13 @@ struct sock {
+ 		struct sk_buff	*tail;
+ 	} sk_backlog;
+ #define sk_rmem_alloc sk_backlog.rmem_alloc
+-	int			sk_forward_alloc;
+ 
+-	__u32			sk_txhash;
++	int			sk_forward_alloc;
+ #ifdef CONFIG_NET_RX_BUSY_POLL
+-	unsigned int		sk_napi_id;
+ 	unsigned int		sk_ll_usec;
++	/* ===== mostly read cache line ===== */
++	unsigned int		sk_napi_id;
+ #endif
+-	atomic_t		sk_drops;
+ 	int			sk_rcvbuf;
+ 
+ 	struct sk_filter __rcu	*sk_filter;
+@@ -379,11 +381,30 @@ struct sock {
+ #endif
+ 	struct dst_entry	*sk_rx_dst;
+ 	struct dst_entry __rcu	*sk_dst_cache;
+-	/* Note: 32bit hole on 64bit arches */
+-	atomic_t		sk_wmem_alloc;
+ 	atomic_t		sk_omem_alloc;
+ 	int			sk_sndbuf;
++
++	/* ===== cache line for TX ===== */
++	int			sk_wmem_queued;
++	atomic_t		sk_wmem_alloc;
++	unsigned long		sk_tsq_flags;
++	struct sk_buff		*sk_send_head;
+ 	struct sk_buff_head	sk_write_queue;
++	__s32			sk_peek_off;
++	int			sk_write_pending;
++	long			sk_sndtimeo;
++	struct timer_list	sk_timer;
++	__u32			sk_priority;
++	__u32			sk_mark;
++	u32			sk_pacing_rate; /* bytes per second */
++	u32			sk_max_pacing_rate;
++	struct page_frag	sk_frag;
++	netdev_features_t	sk_route_caps;
++	netdev_features_t	sk_route_nocaps;
++	int			sk_gso_type;
++	unsigned int		sk_gso_max_size;
++	gfp_t			sk_allocation;
++	__u32			sk_txhash;
+ 
+ 	/*
+ 	 * Because of non atomicity rules, all
+@@ -399,41 +420,23 @@ struct sock {
+ #define SK_PROTOCOL_MAX U8_MAX
+ 	kmemcheck_bitfield_end(flags);
+ 
+-	int			sk_wmem_queued;
+-	gfp_t			sk_allocation;
+-	u32			sk_pacing_rate; /* bytes per second */
+-	u32			sk_max_pacing_rate;
+-	netdev_features_t	sk_route_caps;
+-	netdev_features_t	sk_route_nocaps;
+-	int			sk_gso_type;
+-	unsigned int		sk_gso_max_size;
+ 	u16			sk_gso_max_segs;
+-	int			sk_rcvlowat;
+ 	unsigned long	        sk_lingertime;
+-	struct sk_buff_head	sk_error_queue;
+ 	struct proto		*sk_prot_creator;
+ 	rwlock_t		sk_callback_lock;
+ 	int			sk_err,
+ 				sk_err_soft;
+ 	u32			sk_ack_backlog;
+ 	u32			sk_max_ack_backlog;
+-	__u32			sk_priority;
+-	__u32			sk_mark;
+ 	struct pid		*sk_peer_pid;
+ 	const struct cred	*sk_peer_cred;
+ 	long			sk_rcvtimeo;
+-	long			sk_sndtimeo;
+-	struct timer_list	sk_timer;
+ 	ktime_t			sk_stamp;
+ 	u16			sk_tsflags;
+ 	u8			sk_shutdown;
+ 	u32			sk_tskey;
+ 	struct socket		*sk_socket;
+ 	void			*sk_user_data;
+-	struct page_frag	sk_frag;
+-	struct sk_buff		*sk_send_head;
+-	__s32			sk_peek_off;
+-	int			sk_write_pending;
+ #ifdef CONFIG_SECURITY
+ 	void			*sk_security;
+ #endif