diff options
| author | Jakub Kicinski <kuba@kernel.org> | 2020-11-15 03:09:59 +0300 |
|---|---|---|
| committer | Jakub Kicinski <kuba@kernel.org> | 2020-11-15 03:09:59 +0300 |
| commit | 405ac7fd394c07f57d7cc9b85aec33e1d9b5c24e (patch) | |
| tree | 24d9730863bc25b558c6ae1ca8e08672b5a29dfc | |
| parent | ae8cb93286e50ece1e45674661cbefcafe8b811c (diff) | |
| parent | 1c5f2ced136a7196c41ca2c16b8c2646e9b042ce (diff) | |
| download | linux-405ac7fd394c07f57d7cc9b85aec33e1d9b5c24e.tar.xz | |
Merge branch 'tcp-avoid-indirect-call-in-__sk_stream_memory_free'
Eric Dumazet says:
====================
tcp: avoid indirect call in __sk_stream_memory_free()
Small improvement for CONFIG_RETPOLINE=y, when dealing with TCP sockets.
====================
Link: https://lore.kernel.org/r/20201113150809.3443527-1-eric.dumazet@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
| -rw-r--r-- | include/net/sock.h | 8 | ||||
| -rw-r--r-- | include/net/tcp.h | 13 | ||||
| -rw-r--r-- | net/ipv4/tcp_ipv4.c | 14 |
3 files changed, 21 insertions, 14 deletions
diff --git a/include/net/sock.h b/include/net/sock.h index a5c6ae78df77..1d29aeae74fd 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -60,7 +60,7 @@ #include <linux/rculist_nulls.h> #include <linux/poll.h> #include <linux/sockptr.h> - +#include <linux/indirect_call_wrapper.h> #include <linux/atomic.h> #include <linux/refcount.h> #include <net/dst.h> @@ -1264,13 +1264,17 @@ static inline void sk_refcnt_debug_release(const struct sock *sk) #define sk_refcnt_debug_release(sk) do { } while (0) #endif /* SOCK_REFCNT_DEBUG */ +INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake)); + static inline bool __sk_stream_memory_free(const struct sock *sk, int wake) { if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf)) return false; return sk->sk_prot->stream_memory_free ? - sk->sk_prot->stream_memory_free(sk, wake) : true; + INDIRECT_CALL_1(sk->sk_prot->stream_memory_free, + tcp_stream_memory_free, + sk, wake) : true; } static inline bool sk_stream_memory_free(const struct sock *sk) diff --git a/include/net/tcp.h b/include/net/tcp.h index 4aba0f069b05..d643ee4e4249 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1965,18 +1965,7 @@ static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat; } -/* @wake is one when sk_stream_write_space() calls us. - * This sends EPOLLOUT only if notsent_bytes is half the limit. - * This mimics the strategy used in sock_def_write_space(). - */ -static inline bool tcp_stream_memory_free(const struct sock *sk, int wake) -{ - const struct tcp_sock *tp = tcp_sk(sk); - u32 notsent_bytes = READ_ONCE(tp->write_seq) - - READ_ONCE(tp->snd_nxt); - - return (notsent_bytes << wake) < tcp_notsent_lowat(tp); -} +bool tcp_stream_memory_free(const struct sock *sk, int wake); #ifdef CONFIG_PROC_FS int tcp4_proc_init(void); diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 7352c097ae48..c2d5132c523c 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2740,6 +2740,20 @@ void tcp4_proc_exit(void) } #endif /* CONFIG_PROC_FS */ +/* @wake is one when sk_stream_write_space() calls us. + * This sends EPOLLOUT only if notsent_bytes is half the limit. + * This mimics the strategy used in sock_def_write_space(). + */ +bool tcp_stream_memory_free(const struct sock *sk, int wake) +{ + const struct tcp_sock *tp = tcp_sk(sk); + u32 notsent_bytes = READ_ONCE(tp->write_seq) - + READ_ONCE(tp->snd_nxt); + + return (notsent_bytes << wake) < tcp_notsent_lowat(tp); +} +EXPORT_SYMBOL(tcp_stream_memory_free); + struct proto tcp_prot = { .name = "TCP", .owner = THIS_MODULE, |