From 2a12ae5d433df3d3c3f1a930799ec09cb2b8058f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:11:47 -0800 Subject: net: inline sock_prot_inuse_add() sock_prot_inuse_add() is very small, we can inline it. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sock.h | 14 +++++++++++--- net/core/sock.c | 11 ----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/include/net/sock.h b/include/net/sock.h index 95cc03bd3fac..5a1e1df3cefd 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1429,13 +1429,21 @@ proto_memory_pressure(struct proto *prot) #ifdef CONFIG_PROC_FS +#define PROTO_INUSE_NR 64 /* should be enough for the first time */ +struct prot_inuse { + int val[PROTO_INUSE_NR]; +}; /* Called with local bh disabled */ -void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc); +static inline void sock_prot_inuse_add(const struct net *net, + const struct proto *prot, int val) +{ + __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); +} int sock_prot_inuse_get(struct net *net, struct proto *proto); int sock_inuse_get(struct net *net); #else -static inline void sock_prot_inuse_add(struct net *net, struct proto *prot, - int inc) +static inline void sock_prot_inuse_add(const struct net *net, + const struct proto *prot, int val) { } #endif diff --git a/net/core/sock.c b/net/core/sock.c index c57d9883f62c..c0f9bdd5cf68 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -3536,19 +3536,8 @@ void sk_get_meminfo(const struct sock *sk, u32 *mem) } #ifdef CONFIG_PROC_FS -#define PROTO_INUSE_NR 64 /* should be enough for the first time */ -struct prot_inuse { - int val[PROTO_INUSE_NR]; -}; - static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); -void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) -{ - __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); -} -EXPORT_SYMBOL_GPL(sock_prot_inuse_add); - int sock_prot_inuse_get(struct net *net, struct proto *prot) { int cpu, idx = prot->inuse_idx; -- cgit v1.2.3 From d477eb9004845cb2dc92ad5eed79a437738a868a Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:11:48 -0800 Subject: net: make sock_inuse_add() available MPTCP hard codes it, let us instead provide this helper. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sock.h | 10 ++++++++++ net/core/sock.c | 10 ---------- net/mptcp/subflow.c | 4 +--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/include/net/sock.h b/include/net/sock.h index 5a1e1df3cefd..c4c981a51797 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1439,6 +1439,12 @@ static inline void sock_prot_inuse_add(const struct net *net, { __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); } + +static inline void sock_inuse_add(const struct net *net, int val) +{ + this_cpu_add(*net->core.sock_inuse, val); +} + int sock_prot_inuse_get(struct net *net, struct proto *proto); int sock_inuse_get(struct net *net); #else @@ -1446,6 +1452,10 @@ static inline void sock_prot_inuse_add(const struct net *net, const struct proto *prot, int val) { } + +static inline void sock_inuse_add(const struct net *net, int val) +{ +} #endif diff --git a/net/core/sock.c b/net/core/sock.c index c0f9bdd5cf68..a9bd22b883b9 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -144,8 +144,6 @@ static DEFINE_MUTEX(proto_list_mutex); static LIST_HEAD(proto_list); -static void sock_inuse_add(struct net *net, int val); - /** * sk_ns_capable - General socket capability test * @sk: Socket to use a capability on or through @@ -3550,11 +3548,6 @@ int sock_prot_inuse_get(struct net *net, struct proto *prot) } EXPORT_SYMBOL_GPL(sock_prot_inuse_get); -static void sock_inuse_add(struct net *net, int val) -{ - this_cpu_add(*net->core.sock_inuse, val); -} - int sock_inuse_get(struct net *net) { int cpu, res = 0; @@ -3633,9 +3626,6 @@ static inline void release_proto_idx(struct proto *prot) { } -static void sock_inuse_add(struct net *net, int val) -{ -} #endif static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 6172f380dfb7..49787a1d7b34 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -1534,9 +1534,7 @@ int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock) */ sf->sk->sk_net_refcnt = 1; get_net(net); -#ifdef CONFIG_PROC_FS - this_cpu_add(*net->core.sock_inuse, 1); -#endif + sock_inuse_add(net, 1); err = tcp_set_ulp(sf->sk, "mptcp"); release_sock(sf->sk); -- cgit v1.2.3 From 4199bae10c49e24bc2c5d8c06a68820d56640000 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:11:49 -0800 Subject: net: merge net->core.prot_inuse and net->core.sock_inuse net->core.sock_inuse is a per cpu variable (int), while net->core.prot_inuse is another per cpu variable of 64 integers. per cpu allocator tend to place them in very different places. Grouping them together makes sense, since it makes updates potentially faster, if hitting the same cache line. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/netns/core.h | 1 - include/net/sock.h | 3 ++- net/core/sock.c | 12 +----------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/include/net/netns/core.h b/include/net/netns/core.h index 36c2d998a43c..552bc25b1933 100644 --- a/include/net/netns/core.h +++ b/include/net/netns/core.h @@ -12,7 +12,6 @@ struct netns_core { int sysctl_somaxconn; #ifdef CONFIG_PROC_FS - int __percpu *sock_inuse; struct prot_inuse __percpu *prot_inuse; #endif }; diff --git a/include/net/sock.h b/include/net/sock.h index c4c981a51797..5589312531df 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1431,6 +1431,7 @@ proto_memory_pressure(struct proto *prot) #ifdef CONFIG_PROC_FS #define PROTO_INUSE_NR 64 /* should be enough for the first time */ struct prot_inuse { + int all; int val[PROTO_INUSE_NR]; }; /* Called with local bh disabled */ @@ -1442,7 +1443,7 @@ static inline void sock_prot_inuse_add(const struct net *net, static inline void sock_inuse_add(const struct net *net, int val) { - this_cpu_add(*net->core.sock_inuse, val); + this_cpu_add(net->core.prot_inuse->all, val); } int sock_prot_inuse_get(struct net *net, struct proto *proto); diff --git a/net/core/sock.c b/net/core/sock.c index a9bd22b883b9..d7fc8b5e2569 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -3553,7 +3553,7 @@ int sock_inuse_get(struct net *net) int cpu, res = 0; for_each_possible_cpu(cpu) - res += *per_cpu_ptr(net->core.sock_inuse, cpu); + res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; return res; } @@ -3565,22 +3565,12 @@ static int __net_init sock_inuse_init_net(struct net *net) net->core.prot_inuse = alloc_percpu(struct prot_inuse); if (net->core.prot_inuse == NULL) return -ENOMEM; - - net->core.sock_inuse = alloc_percpu(int); - if (net->core.sock_inuse == NULL) - goto out; - return 0; - -out: - free_percpu(net->core.prot_inuse); - return -ENOMEM; } static void __net_exit sock_inuse_exit_net(struct net *net) { free_percpu(net->core.prot_inuse); - free_percpu(net->core.sock_inuse); } static struct pernet_operations net_inuse_ops = { -- cgit v1.2.3 From b3cb764aa1d753cf6a58858f9e2097ba71e8100b Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:11:50 -0800 Subject: net: drop nopreempt requirement on sock_prot_inuse_add() This is distracting really, let's make this simpler, because many callers had to take care of this by themselves, even if on x86 this adds more code than really needed. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sock.h | 4 ++-- net/ieee802154/socket.c | 4 ++-- net/ipv4/raw.c | 2 +- net/ipv6/ipv6_sockglue.c | 8 ++++---- net/netlink/af_netlink.c | 4 ---- net/packet/af_packet.c | 4 ---- net/sctp/socket.c | 5 ----- net/smc/af_smc.c | 2 +- net/unix/af_unix.c | 4 ---- net/xdp/xsk.c | 4 ---- 10 files changed, 10 insertions(+), 31 deletions(-) diff --git a/include/net/sock.h b/include/net/sock.h index 5589312531df..f09c0c4736c4 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1434,11 +1434,11 @@ struct prot_inuse { int all; int val[PROTO_INUSE_NR]; }; -/* Called with local bh disabled */ + static inline void sock_prot_inuse_add(const struct net *net, const struct proto *prot, int val) { - __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); + this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); } static inline void sock_inuse_add(const struct net *net, int val) diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c index 7bb9ef35c570..3b2366a88c3c 100644 --- a/net/ieee802154/socket.c +++ b/net/ieee802154/socket.c @@ -174,8 +174,8 @@ static int raw_hash(struct sock *sk) { write_lock_bh(&raw_lock); sk_add_node(sk, &raw_head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&raw_lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } @@ -453,8 +453,8 @@ static int dgram_hash(struct sock *sk) { write_lock_bh(&dgram_lock); sk_add_node(sk, &dgram_head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&dgram_lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index bb446e60cf58..3910bb08a986 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -99,8 +99,8 @@ int raw_hash_sk(struct sock *sk) write_lock_bh(&h->lock); sk_add_node(sk, head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&h->lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c index 41efca817db4..04683ad6d3df 100644 --- a/net/ipv6/ipv6_sockglue.c +++ b/net/ipv6/ipv6_sockglue.c @@ -471,10 +471,10 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, if (sk->sk_protocol == IPPROTO_TCP) { struct inet_connection_sock *icsk = inet_csk(sk); - local_bh_disable(); + sock_prot_inuse_add(net, sk->sk_prot, -1); sock_prot_inuse_add(net, &tcp_prot, 1); - local_bh_enable(); + sk->sk_prot = &tcp_prot; icsk->icsk_af_ops = &ipv4_specific; sk->sk_socket->ops = &inet_stream_ops; @@ -485,10 +485,10 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, if (sk->sk_protocol == IPPROTO_UDPLITE) prot = &udplite_prot; - local_bh_disable(); + sock_prot_inuse_add(net, sk->sk_prot, -1); sock_prot_inuse_add(net, prot, 1); - local_bh_enable(); + sk->sk_prot = prot; sk->sk_socket->ops = &inet_dgram_ops; sk->sk_family = PF_INET; diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 4c575324a985..1a19d179e913 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -707,9 +707,7 @@ static int netlink_create(struct net *net, struct socket *sock, int protocol, if (err < 0) goto out_module; - local_bh_disable(); sock_prot_inuse_add(net, &netlink_proto, 1); - local_bh_enable(); nlk = nlk_sk(sock->sk); nlk->module = module; @@ -809,9 +807,7 @@ static int netlink_release(struct socket *sock) netlink_table_ungrab(); } - local_bh_disable(); sock_prot_inuse_add(sock_net(sk), &netlink_proto, -1); - local_bh_enable(); call_rcu(&nlk->rcu, deferred_put_nlk_sk); return 0; } diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 46943a18a10d..a1ffdb48cc47 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -3102,9 +3102,7 @@ static int packet_release(struct socket *sock) sk_del_node_init_rcu(sk); mutex_unlock(&net->packet.sklist_lock); - preempt_disable(); sock_prot_inuse_add(net, sk->sk_prot, -1); - preempt_enable(); spin_lock(&po->bind_lock); unregister_prot_hook(sk, false); @@ -3368,9 +3366,7 @@ static int packet_create(struct net *net, struct socket *sock, int protocol, sk_add_node_tail_rcu(sk, &net->packet.sklist); mutex_unlock(&net->packet.sklist_lock); - preempt_disable(); sock_prot_inuse_add(net, &packet_proto, 1); - preempt_enable(); return 0; out2: diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 33391254fa82..055a6d3ec6e2 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -5068,12 +5068,9 @@ static int sctp_init_sock(struct sock *sk) SCTP_DBG_OBJCNT_INC(sock); - local_bh_disable(); sk_sockets_allocated_inc(sk); sock_prot_inuse_add(net, sk->sk_prot, 1); - local_bh_enable(); - return 0; } @@ -5099,10 +5096,8 @@ static void sctp_destroy_sock(struct sock *sk) list_del(&sp->auto_asconf_list); } sctp_endpoint_free(sp->ep); - local_bh_disable(); sk_sockets_allocated_dec(sk); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); - local_bh_enable(); } /* Triggered when there are no references on the socket anymore */ diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 59284da9116d..ff5cd0c30741 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -89,8 +89,8 @@ int smc_hash_sk(struct sock *sk) write_lock_bh(&h->lock); sk_add_node(sk, head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&h->lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 78e08e82c08c..54e5553a150e 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -522,9 +522,7 @@ static void unix_sock_destructor(struct sock *sk) unix_release_addr(u->addr); atomic_long_dec(&unix_nr_socks); - local_bh_disable(); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); - local_bh_enable(); #ifdef UNIX_REFCNT_DEBUG pr_debug("UNIX %p is destroyed, %ld are still alive.\n", sk, atomic_long_read(&unix_nr_socks)); @@ -889,9 +887,7 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, memset(&u->scm_stat, 0, sizeof(struct scm_stat)); unix_insert_socket(unix_sockets_unbound(sk), sk); - local_bh_disable(); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); - local_bh_enable(); return sk; diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index f16074eb53c7..28ef3f4465ae 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -794,9 +794,7 @@ static int xsk_release(struct socket *sock) sk_del_node_init_rcu(sk); mutex_unlock(&net->xdp.lock); - local_bh_disable(); sock_prot_inuse_add(net, sk->sk_prot, -1); - local_bh_enable(); xsk_delete_from_maps(xs); mutex_lock(&xs->mutex); @@ -1396,9 +1394,7 @@ static int xsk_create(struct net *net, struct socket *sock, int protocol, sk_add_node_rcu(sk, &net->xdp.list); mutex_unlock(&net->xdp.lock); - local_bh_disable(); sock_prot_inuse_add(net, &xsk_proto, 1); - local_bh_enable(); return 0; } -- cgit v1.2.3