From fe5886852601fb2593cbc5a7549ef9fd2ef481ba Mon Sep 17 00:00:00 2001 From: Magnus Karlsson Date: Fri, 29 Jun 2018 09:48:18 +0200 Subject: xsk: frame could be completed more than once in SKB path Fixed a bug in which a frame could be completed more than once when an error was returned from dev_direct_xmit(). The code erroneously retried sending the message leading to multiple calls to the SKB destructor and therefore multiple completions of the same buffer to user space. The error code in this case has been changed from EAGAIN to EBUSY in order to tell user space that the sending of the packet failed and the buffer has been return to user space through the completion queue. Fixes: 35fcde7f8deb ("xsk: support for Tx") Signed-off-by: Magnus Karlsson Reported-by: Pavel Odintsov Signed-off-by: Alexei Starovoitov --- net/xdp/xsk.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'net/xdp/xsk.c') diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 59fb7d3c36a3..15aca73805fc 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -268,15 +268,15 @@ static int xsk_generic_xmit(struct sock *sk, struct msghdr *m, skb->destructor = xsk_destruct_skb; err = dev_direct_xmit(skb, xs->queue_id); + xskq_discard_desc(xs->tx); /* Ignore NET_XMIT_CN as packet might have been sent */ if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { - err = -EAGAIN; - /* SKB consumed by dev_direct_xmit() */ + /* SKB completed but not sent */ + err = -EBUSY; goto out; } sent_frame = true; - xskq_discard_desc(xs->tx); } out: -- cgit v1.2.3 From a9744f7ca200c756e6f8c65b633770a2da711651 Mon Sep 17 00:00:00 2001 From: Magnus Karlsson Date: Fri, 29 Jun 2018 09:48:20 +0200 Subject: xsk: fix potential race in SKB TX completion code There is a potential race in the TX completion code for the SKB case. One process enters the sendmsg code of an AF_XDP socket in order to send a frame. The execution eventually trickles down to the driver that is told to send the packet. However, it decides to drop the packet due to some error condition (e.g., rings full) and frees the SKB. This will trigger the SKB destructor and a completion will be sent to the AF_XDP user space through its single-producer/single-consumer queues. At the same time a TX interrupt has fired on another core and it dispatches the TX completion code in the driver. It does its HW specific things and ends up freeing the SKB associated with the transmitted packet. This will trigger the SKB destructor and a completion will be sent to the AF_XDP user space through its single-producer/single-consumer queues. With a pseudo call stack, it would look like this: Core 1: sendmsg() being called in the application netdev_start_xmit() Driver entered through ndo_start_xmit Driver decides to free the SKB for some reason (e.g., rings full) Destructor of SKB called xskq_produce_addr() is called to signal completion to user space Core 2: TX completion irq NAPI loop Driver irq handler for TX completions Frees the SKB Destructor of SKB called xskq_produce_addr() is called to signal completion to user space We now have a violation of the single-producer/single-consumer principle for our queues as there are two threads trying to produce at the same time on the same queue. Fixed by introducing a spin_lock in the destructor. In regards to the performance, I get around 1.74 Mpps for txonly before and after the introduction of the spinlock. There is of course some impact due to the spin lock but it is in the less significant digits that are too noisy for me to measure. But let us say that the version without the spin lock got 1.745 Mpps in the best case and the version with 1.735 Mpps in the worst case, then that would mean a maximum drop in performance of 0.5%. Fixes: 35fcde7f8deb ("xsk: support for Tx") Signed-off-by: Magnus Karlsson Signed-off-by: Alexei Starovoitov --- include/net/xdp_sock.h | 4 ++++ net/xdp/xsk.c | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'net/xdp/xsk.c') diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 9fe472f2ac95..7161856bcf9c 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -60,6 +60,10 @@ struct xdp_sock { bool zc; /* Protects multiple processes in the control path */ struct mutex mutex; + /* Mutual exclusion of NAPI TX thread and sendmsg error paths + * in the SKB destructor callback. + */ + spinlock_t tx_completion_lock; u64 rx_dropped; }; diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 15aca73805fc..7d220cbd09b6 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -199,8 +199,11 @@ static void xsk_destruct_skb(struct sk_buff *skb) { u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg; struct xdp_sock *xs = xdp_sk(skb->sk); + unsigned long flags; + spin_lock_irqsave(&xs->tx_completion_lock, flags); WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr)); + spin_unlock_irqrestore(&xs->tx_completion_lock, flags); sock_wfree(skb); } @@ -755,6 +758,7 @@ static int xsk_create(struct net *net, struct socket *sock, int protocol, xs = xdp_sk(sk); mutex_init(&xs->mutex); + spin_lock_init(&xs->tx_completion_lock); local_bh_disable(); sock_prot_inuse_add(net, &xsk_proto, 1); -- cgit v1.2.3 From 509d7648135f914a3dd64c17484b33df5dd0a19c Mon Sep 17 00:00:00 2001 From: Magnus Karlsson Date: Wed, 11 Jul 2018 10:12:49 +0200 Subject: xsk: do not return ENXIO from TX copy mode This patch removes the ENXIO return code from TX copy-mode when someone has forcefully changed the number of queues on the device so that the queue bound to the socket is no longer available. Just silently stop sending anything as in zero-copy mode so the error reporting gets consistent between the two modes. Fixes: 35fcde7f8deb ("xsk: support for Tx") Signed-off-by: Magnus Karlsson Signed-off-by: Daniel Borkmann --- net/xdp/xsk.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'net/xdp/xsk.c') diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 7d220cbd09b6..08d09115093e 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -244,10 +244,8 @@ static int xsk_generic_xmit(struct sock *sk, struct msghdr *m, goto out; } - if (xs->queue_id >= xs->dev->real_num_tx_queues) { - err = -ENXIO; + if (xs->queue_id >= xs->dev->real_num_tx_queues) goto out; - } skb = sock_alloc_send_skb(sk, len, 1, &err); if (unlikely(!skb)) { -- cgit v1.2.3 From 9684f5e7c8cdf076aeec81344d4893a30f7aa6a1 Mon Sep 17 00:00:00 2001 From: Magnus Karlsson Date: Wed, 11 Jul 2018 10:12:50 +0200 Subject: xsk: do not return EAGAIN from sendmsg when completion queue is full This patch stops returning EAGAIN in TX copy mode when the completion queue is full as zero-copy does not do this. Instead this situation can be detected by comparing the head and tail pointers of the completion queue in both modes. In any case, EAGAIN was not the correct error code here since no amount of calling sendmsg will solve the problem. Only consuming one or more messages on the completion queue will fix this. With this patch, the error reporting becomes consistent between copy mode and zero-copy mode. Fixes: 35fcde7f8deb ("xsk: support for Tx") Signed-off-by: Magnus Karlsson Signed-off-by: Daniel Borkmann --- net/xdp/xsk.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'net/xdp/xsk.c') diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 08d09115093e..87567232d0f8 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -233,10 +233,8 @@ static int xsk_generic_xmit(struct sock *sk, struct msghdr *m, goto out; } - if (xskq_reserve_addr(xs->umem->cq)) { - err = -EAGAIN; + if (xskq_reserve_addr(xs->umem->cq)) goto out; - } len = desc.len; if (unlikely(len > xs->dev->mtu)) { -- cgit v1.2.3 From 6efb4436f7fcc50cc3fb9a113d0f16e3968172b1 Mon Sep 17 00:00:00 2001 From: Magnus Karlsson Date: Wed, 11 Jul 2018 10:12:51 +0200 Subject: xsk: always return ENOBUFS from sendmsg if there is no TX queue This patch makes sure ENOBUFS is always returned from sendmsg if there is no TX queue configured. This was not the case for zero-copy mode. With this patch this error reporting is consistent between copy mode and zero-copy mode. Fixes: ac98d8aab61b ("xsk: wire upp Tx zero-copy functions") Signed-off-by: Magnus Karlsson Signed-off-by: Daniel Borkmann --- net/xdp/xsk.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'net/xdp/xsk.c') diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 87567232d0f8..9c784307f7b0 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -218,9 +218,6 @@ static int xsk_generic_xmit(struct sock *sk, struct msghdr *m, struct sk_buff *skb; int err = 0; - if (unlikely(!xs->tx)) - return -ENOBUFS; - mutex_lock(&xs->mutex); while (xskq_peek_desc(xs->tx, &desc)) { @@ -296,6 +293,8 @@ static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) return -ENXIO; if (unlikely(!(xs->dev->flags & IFF_UP))) return -ENETDOWN; + if (unlikely(!xs->tx)) + return -ENOBUFS; if (need_wait) return -EOPNOTSUPP; -- cgit v1.2.3 From 09210c4bcc065d9d91ef3c051902ad18252cd3c0 Mon Sep 17 00:00:00 2001 From: Magnus Karlsson Date: Wed, 11 Jul 2018 10:12:52 +0200 Subject: xsk: do not return EMSGSIZE in copy mode for packets larger than MTU This patch stops returning EMSGSIZE from sendmsg in copy mode when the size of the packet is larger than the MTU. Just send it to the device so that it will drop it as in zero-copy mode. This makes the error reporting consistent between copy mode and zero-copy mode. Fixes: 35fcde7f8deb ("xsk: support for Tx") Signed-off-by: Magnus Karlsson Signed-off-by: Daniel Borkmann --- net/xdp/xsk.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'net/xdp/xsk.c') diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 9c784307f7b0..72335c2e8108 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -233,15 +233,10 @@ static int xsk_generic_xmit(struct sock *sk, struct msghdr *m, if (xskq_reserve_addr(xs->umem->cq)) goto out; - len = desc.len; - if (unlikely(len > xs->dev->mtu)) { - err = -EMSGSIZE; - goto out; - } - if (xs->queue_id >= xs->dev->real_num_tx_queues) goto out; + len = desc.len; skb = sock_alloc_send_skb(sk, len, 1, &err); if (unlikely(!skb)) { err = -EAGAIN; -- cgit v1.2.3