summaryrefslogtreecommitdiff
path: root/net/xfrm
diff options
context:
space:
mode:
Diffstat (limited to 'net/xfrm')
-rw-r--r--net/xfrm/espintcp.c4
-rw-r--r--net/xfrm/xfrm_algo.c7
-rw-r--r--net/xfrm/xfrm_device.c50
-rw-r--r--net/xfrm/xfrm_interface_core.c56
-rw-r--r--net/xfrm/xfrm_ipcomp.c433
-rw-r--r--net/xfrm/xfrm_iptfs.c6
-rw-r--r--net/xfrm/xfrm_nat_keepalive.c30
-rw-r--r--net/xfrm/xfrm_output.c6
-rw-r--r--net/xfrm/xfrm_policy.c23
-rw-r--r--net/xfrm/xfrm_state.c227
-rw-r--r--net/xfrm/xfrm_user.c74
11 files changed, 494 insertions, 422 deletions
diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c
index fe82e2d07300..fc7a603b04f1 100644
--- a/net/xfrm/espintcp.c
+++ b/net/xfrm/espintcp.c
@@ -171,8 +171,10 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb)
struct espintcp_ctx *ctx = espintcp_getctx(sk);
if (skb_queue_len(&ctx->out_queue) >=
- READ_ONCE(net_hotdata.max_backlog))
+ READ_ONCE(net_hotdata.max_backlog)) {
+ kfree_skb(skb);
return -ENOBUFS;
+ }
__skb_queue_tail(&ctx->out_queue, skb);
diff --git a/net/xfrm/xfrm_algo.c b/net/xfrm/xfrm_algo.c
index e6da7e8495c9..749011e031c0 100644
--- a/net/xfrm/xfrm_algo.c
+++ b/net/xfrm/xfrm_algo.c
@@ -5,13 +5,13 @@
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
*/
+#include <crypto/acompress.h>
#include <crypto/aead.h>
#include <crypto/hash.h>
#include <crypto/skcipher.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pfkeyv2.h>
-#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <net/xfrm.h>
#if IS_ENABLED(CONFIG_INET_ESP) || IS_ENABLED(CONFIG_INET6_ESP)
@@ -669,7 +669,7 @@ static const struct xfrm_algo_list xfrm_ealg_list = {
};
static const struct xfrm_algo_list xfrm_calg_list = {
- .find = crypto_has_comp,
+ .find = crypto_has_acomp,
.algs = calg_list,
.entries = ARRAY_SIZE(calg_list),
};
@@ -828,8 +828,7 @@ void xfrm_probe_algs(void)
}
for (i = 0; i < calg_entries(); i++) {
- status = crypto_has_comp(calg_list[i].name, 0,
- CRYPTO_ALG_ASYNC);
+ status = crypto_has_acomp(calg_list[i].name, 0, 0);
if (calg_list[i].available != status)
calg_list[i].available = status;
}
diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c
index 97c8030cc417..c7a1f080d2de 100644
--- a/net/xfrm/xfrm_device.c
+++ b/net/xfrm/xfrm_device.c
@@ -145,10 +145,6 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur
return NULL;
}
- /* This skb was already validated on the upper/virtual dev */
- if ((x->xso.dev != dev) && (x->xso.real_dev == dev))
- return skb;
-
local_irq_save(flags);
sd = this_cpu_ptr(&softnet_data);
err = !skb_queue_empty(&sd->xfrm_backlog);
@@ -256,6 +252,11 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
return -EINVAL;
}
+ if (xuo->flags & XFRM_OFFLOAD_INBOUND && x->if_id) {
+ NL_SET_ERR_MSG(extack, "XFRM if_id is not supported in RX path");
+ return -EINVAL;
+ }
+
is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET;
/* We don't yet support TFC padding. */
@@ -305,7 +306,6 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
return -EINVAL;
}
- xfrm_set_type_offload(x);
if (!x->type_offload) {
NL_SET_ERR_MSG(extack, "Type doesn't support offload");
dev_put(dev);
@@ -314,7 +314,6 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
xso->dev = dev;
netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
- xso->real_dev = dev;
if (xuo->flags & XFRM_OFFLOAD_INBOUND)
xso->dir = XFRM_DEV_OFFLOAD_IN;
@@ -326,11 +325,10 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
else
xso->type = XFRM_DEV_OFFLOAD_CRYPTO;
- err = dev->xfrmdev_ops->xdo_dev_state_add(x, extack);
+ err = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, extack);
if (err) {
xso->dev = NULL;
xso->dir = 0;
- xso->real_dev = NULL;
netdev_put(dev, &xso->dev_tracker);
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
@@ -378,7 +376,6 @@ int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
xdo->dev = dev;
netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC);
- xdo->real_dev = dev;
xdo->type = XFRM_DEV_OFFLOAD_PACKET;
switch (dir) {
case XFRM_POLICY_IN:
@@ -400,7 +397,6 @@ int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
err = dev->xfrmdev_ops->xdo_dev_policy_add(xp, extack);
if (err) {
xdo->dev = NULL;
- xdo->real_dev = NULL;
xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
xdo->dir = 0;
netdev_put(dev, &xdo->dev_tracker);
@@ -418,14 +414,14 @@ bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
struct dst_entry *dst = skb_dst(skb);
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
struct net_device *dev = x->xso.dev;
+ bool check_tunnel_size;
if (!x->type_offload ||
(x->xso.type == XFRM_DEV_OFFLOAD_UNSPECIFIED && x->encap))
return false;
- if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET ||
- ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
- !xdst->child->xfrm)) {
+ if ((!dev || dev == xfrm_dst_path(dst)->dev) &&
+ !xdst->child->xfrm) {
mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
if (skb->len <= mtu)
goto ok;
@@ -437,8 +433,32 @@ bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
return false;
ok:
- if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
- return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
+ if (!dev)
+ return true;
+
+ check_tunnel_size = x->xso.type == XFRM_DEV_OFFLOAD_PACKET &&
+ x->props.mode == XFRM_MODE_TUNNEL;
+ switch (x->props.family) {
+ case AF_INET:
+ /* Check for IPv4 options */
+ if (ip_hdr(skb)->ihl != 5)
+ return false;
+ if (check_tunnel_size && xfrm4_tunnel_check_size(skb))
+ return false;
+ break;
+ case AF_INET6:
+ /* Check for IPv6 extensions */
+ if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
+ return false;
+ if (check_tunnel_size && xfrm6_tunnel_check_size(skb))
+ return false;
+ break;
+ default:
+ break;
+ }
+
+ if (dev->xfrmdev_ops->xdo_dev_offload_ok)
+ return dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
return true;
}
diff --git a/net/xfrm/xfrm_interface_core.c b/net/xfrm/xfrm_interface_core.c
index c397eb99d867..330a05286a56 100644
--- a/net/xfrm/xfrm_interface_core.c
+++ b/net/xfrm/xfrm_interface_core.c
@@ -242,10 +242,9 @@ static void xfrmi_dev_free(struct net_device *dev)
gro_cells_destroy(&xi->gro_cells);
}
-static int xfrmi_create(struct net_device *dev)
+static int xfrmi_create(struct net *net, struct net_device *dev)
{
struct xfrm_if *xi = netdev_priv(dev);
- struct net *net = dev_net(dev);
struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
int err;
@@ -814,15 +813,17 @@ static void xfrmi_netlink_parms(struct nlattr *data[],
parms->collect_md = true;
}
-static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
- struct nlattr *tb[], struct nlattr *data[],
- struct netlink_ext_ack *extack)
+static int xfrmi_newlink(struct net_device *dev,
+ struct rtnl_newlink_params *params,
+ struct netlink_ext_ack *extack)
{
- struct net *net = dev_net(dev);
+ struct nlattr **data = params->data;
struct xfrm_if_parms p = {};
struct xfrm_if *xi;
+ struct net *net;
int err;
+ net = params->link_net ? : dev_net(dev);
xfrmi_netlink_parms(data, &p);
if (p.collect_md) {
struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
@@ -851,7 +852,7 @@ static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
xi->net = net;
xi->dev = dev;
- err = xfrmi_create(dev);
+ err = xfrmi_create(net, dev);
return err;
}
@@ -874,7 +875,7 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[],
return -EINVAL;
}
- if (p.collect_md) {
+ if (p.collect_md || xi->p.collect_md) {
NL_SET_ERR_MSG(extack, "collect_md can't be changed");
return -EINVAL;
}
@@ -885,11 +886,6 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[],
} else {
if (xi->dev != dev)
return -EEXIST;
- if (xi->p.collect_md) {
- NL_SET_ERR_MSG(extack,
- "device can't be changed to collect_md");
- return -EINVAL;
- }
}
return xfrmi_update(xi, &p);
@@ -951,32 +947,28 @@ static struct rtnl_link_ops xfrmi_link_ops __read_mostly = {
.get_link_net = xfrmi_get_link_net,
};
-static void __net_exit xfrmi_exit_batch_rtnl(struct list_head *net_exit_list,
- struct list_head *dev_to_kill)
+static void __net_exit xfrmi_exit_rtnl(struct net *net,
+ struct list_head *dev_to_kill)
{
- struct net *net;
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+ struct xfrm_if __rcu **xip;
+ struct xfrm_if *xi;
+ int i;
- ASSERT_RTNL();
- list_for_each_entry(net, net_exit_list, exit_list) {
- struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
- struct xfrm_if __rcu **xip;
- struct xfrm_if *xi;
- int i;
-
- for (i = 0; i < XFRMI_HASH_SIZE; i++) {
- for (xip = &xfrmn->xfrmi[i];
- (xi = rtnl_dereference(*xip)) != NULL;
- xip = &xi->next)
- unregister_netdevice_queue(xi->dev, dev_to_kill);
- }
- xi = rtnl_dereference(xfrmn->collect_md_xfrmi);
- if (xi)
+ for (i = 0; i < XFRMI_HASH_SIZE; i++) {
+ for (xip = &xfrmn->xfrmi[i];
+ (xi = rtnl_net_dereference(net, *xip)) != NULL;
+ xip = &xi->next)
unregister_netdevice_queue(xi->dev, dev_to_kill);
}
+
+ xi = rtnl_net_dereference(net, xfrmn->collect_md_xfrmi);
+ if (xi)
+ unregister_netdevice_queue(xi->dev, dev_to_kill);
}
static struct pernet_operations xfrmi_net_ops = {
- .exit_batch_rtnl = xfrmi_exit_batch_rtnl,
+ .exit_rtnl = xfrmi_exit_rtnl,
.id = &xfrmi_net_id,
.size = sizeof(struct xfrmi_net),
};
diff --git a/net/xfrm/xfrm_ipcomp.c b/net/xfrm/xfrm_ipcomp.c
index 9c0fa0e1786a..43fdc6ed8dd1 100644
--- a/net/xfrm/xfrm_ipcomp.c
+++ b/net/xfrm/xfrm_ipcomp.c
@@ -3,7 +3,7 @@
* IP Payload Compression Protocol (IPComp) - RFC3173.
*
* Copyright (c) 2003 James Morris <jmorris@intercode.com.au>
- * Copyright (c) 2003-2008 Herbert Xu <herbert@gondor.apana.org.au>
+ * Copyright (c) 2003-2025 Herbert Xu <herbert@gondor.apana.org.au>
*
* Todo:
* - Tunable compression parameters.
@@ -11,303 +11,301 @@
* - Adaptive compression.
*/
-#include <linux/crypto.h>
+#include <crypto/acompress.h>
#include <linux/err.h>
-#include <linux/list.h>
#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/percpu.h>
+#include <linux/skbuff_ref.h>
#include <linux/slab.h>
-#include <linux/smp.h>
-#include <linux/vmalloc.h>
-#include <net/ip.h>
#include <net/ipcomp.h>
#include <net/xfrm.h>
-struct ipcomp_tfms {
- struct list_head list;
- struct crypto_comp * __percpu *tfms;
- int users;
+#define IPCOMP_SCRATCH_SIZE 65400
+
+struct ipcomp_skb_cb {
+ struct xfrm_skb_cb xfrm;
+ struct acomp_req *req;
};
-static DEFINE_MUTEX(ipcomp_resource_mutex);
-static void * __percpu *ipcomp_scratches;
-static int ipcomp_scratch_users;
-static LIST_HEAD(ipcomp_tfms_list);
+struct ipcomp_data {
+ u16 threshold;
+ struct crypto_acomp *tfm;
+};
-static int ipcomp_decompress(struct xfrm_state *x, struct sk_buff *skb)
+struct ipcomp_req_extra {
+ struct xfrm_state *x;
+ struct scatterlist sg[];
+};
+
+static inline struct ipcomp_skb_cb *ipcomp_cb(struct sk_buff *skb)
{
- struct ipcomp_data *ipcd = x->data;
- const int plen = skb->len;
- int dlen = IPCOMP_SCRATCH_SIZE;
- const u8 *start = skb->data;
- u8 *scratch = *this_cpu_ptr(ipcomp_scratches);
- struct crypto_comp *tfm = *this_cpu_ptr(ipcd->tfms);
- int err = crypto_comp_decompress(tfm, start, plen, scratch, &dlen);
- int len;
+ struct ipcomp_skb_cb *cb = (void *)skb->cb;
- if (err)
- return err;
+ BUILD_BUG_ON(sizeof(*cb) > sizeof(skb->cb));
+ return cb;
+}
- if (dlen < (plen + sizeof(struct ip_comp_hdr)))
- return -EINVAL;
+static int ipcomp_post_acomp(struct sk_buff *skb, int err, int hlen)
+{
+ struct acomp_req *req = ipcomp_cb(skb)->req;
+ struct ipcomp_req_extra *extra;
+ struct scatterlist *dsg;
+ int len, dlen;
- len = dlen - plen;
- if (len > skb_tailroom(skb))
- len = skb_tailroom(skb);
+ if (unlikely(err))
+ goto out_free_req;
- __skb_put(skb, len);
+ extra = acomp_request_extra(req);
+ dsg = extra->sg;
+ dlen = req->dlen;
- len += plen;
- skb_copy_to_linear_data(skb, scratch, len);
+ pskb_trim_unique(skb, 0);
+ __skb_put(skb, hlen);
- while ((scratch += len, dlen -= len) > 0) {
+ /* Only update truesize on input. */
+ if (!hlen)
+ skb->truesize += dlen;
+ skb->data_len = dlen;
+ skb->len += dlen;
+
+ do {
skb_frag_t *frag;
struct page *page;
- if (WARN_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS))
- return -EMSGSIZE;
-
frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
- page = alloc_page(GFP_ATOMIC);
-
- if (!page)
- return -ENOMEM;
+ page = sg_page(dsg);
+ dsg = sg_next(dsg);
len = PAGE_SIZE;
if (dlen < len)
len = dlen;
skb_frag_fill_page_desc(frag, page, 0, len);
- memcpy(skb_frag_address(frag), scratch, len);
-
- skb->truesize += len;
- skb->data_len += len;
- skb->len += len;
skb_shinfo(skb)->nr_frags++;
- }
+ } while ((dlen -= len));
- return 0;
+ for (; dsg; dsg = sg_next(dsg))
+ __free_page(sg_page(dsg));
+
+out_free_req:
+ acomp_request_free(req);
+ return err;
}
-int ipcomp_input(struct xfrm_state *x, struct sk_buff *skb)
+static int ipcomp_input_done2(struct sk_buff *skb, int err)
{
- int nexthdr;
- int err = -ENOMEM;
- struct ip_comp_hdr *ipch;
-
- if (skb_linearize_cow(skb))
- goto out;
-
- skb->ip_summed = CHECKSUM_NONE;
-
- /* Remove ipcomp header and decompress original payload */
- ipch = (void *)skb->data;
- nexthdr = ipch->nexthdr;
+ struct ip_comp_hdr *ipch = ip_comp_hdr(skb);
+ const int plen = skb->len;
skb->transport_header = skb->network_header + sizeof(*ipch);
- __skb_pull(skb, sizeof(*ipch));
- err = ipcomp_decompress(x, skb);
- if (err)
- goto out;
- err = nexthdr;
+ return ipcomp_post_acomp(skb, err, 0) ?:
+ skb->len < (plen + sizeof(ip_comp_hdr)) ? -EINVAL :
+ ipch->nexthdr;
+}
-out:
- return err;
+static void ipcomp_input_done(void *data, int err)
+{
+ struct sk_buff *skb = data;
+
+ xfrm_input_resume(skb, ipcomp_input_done2(skb, err));
}
-EXPORT_SYMBOL_GPL(ipcomp_input);
-static int ipcomp_compress(struct xfrm_state *x, struct sk_buff *skb)
+static struct acomp_req *ipcomp_setup_req(struct xfrm_state *x,
+ struct sk_buff *skb, int minhead,
+ int dlen)
{
+ const int dnfrags = min(MAX_SKB_FRAGS, 16);
struct ipcomp_data *ipcd = x->data;
+ struct ipcomp_req_extra *extra;
+ struct scatterlist *sg, *dsg;
const int plen = skb->len;
- int dlen = IPCOMP_SCRATCH_SIZE;
- u8 *start = skb->data;
- struct crypto_comp *tfm;
- u8 *scratch;
+ struct crypto_acomp *tfm;
+ struct acomp_req *req;
+ int nfrags;
+ int total;
int err;
+ int i;
- local_bh_disable();
- scratch = *this_cpu_ptr(ipcomp_scratches);
- tfm = *this_cpu_ptr(ipcd->tfms);
- err = crypto_comp_compress(tfm, start, plen, scratch, &dlen);
- if (err)
- goto out;
-
- if ((dlen + sizeof(struct ip_comp_hdr)) >= plen) {
- err = -EMSGSIZE;
- goto out;
- }
+ ipcomp_cb(skb)->req = NULL;
- memcpy(start + sizeof(struct ip_comp_hdr), scratch, dlen);
- local_bh_enable();
+ do {
+ struct sk_buff *trailer;
- pskb_trim(skb, dlen + sizeof(struct ip_comp_hdr));
- return 0;
+ if (skb->len > PAGE_SIZE) {
+ if (skb_linearize_cow(skb))
+ return ERR_PTR(-ENOMEM);
+ nfrags = 1;
+ break;
+ }
-out:
- local_bh_enable();
- return err;
-}
+ if (!skb_cloned(skb) && skb_headlen(skb) >= minhead) {
+ if (!skb_is_nonlinear(skb)) {
+ nfrags = 1;
+ break;
+ } else if (!skb_has_frag_list(skb)) {
+ nfrags = skb_shinfo(skb)->nr_frags;
+ nfrags++;
+ break;
+ }
+ }
-int ipcomp_output(struct xfrm_state *x, struct sk_buff *skb)
-{
- int err;
- struct ip_comp_hdr *ipch;
- struct ipcomp_data *ipcd = x->data;
+ nfrags = skb_cow_data(skb, skb_headlen(skb) < minhead ?
+ minhead - skb_headlen(skb) : 0,
+ &trailer);
+ if (nfrags < 0)
+ return ERR_PTR(nfrags);
+ } while (0);
+
+ tfm = ipcd->tfm;
+ req = acomp_request_alloc_extra(
+ tfm, sizeof(*extra) + sizeof(*sg) * (nfrags + dnfrags),
+ GFP_ATOMIC);
+ ipcomp_cb(skb)->req = req;
+ if (!req)
+ return ERR_PTR(-ENOMEM);
+
+ extra = acomp_request_extra(req);
+ extra->x = x;
+
+ dsg = extra->sg;
+ sg = dsg + dnfrags;
+ sg_init_table(sg, nfrags);
+ err = skb_to_sgvec(skb, sg, 0, plen);
+ if (unlikely(err < 0))
+ return ERR_PTR(err);
+
+ sg_init_table(dsg, dnfrags);
+ total = 0;
+ for (i = 0; i < dnfrags && total < dlen; i++) {
+ struct page *page;
- if (skb->len < ipcd->threshold) {
- /* Don't bother compressing */
- goto out_ok;
+ page = alloc_page(GFP_ATOMIC);
+ if (!page)
+ break;
+ sg_set_page(dsg + i, page, PAGE_SIZE, 0);
+ total += PAGE_SIZE;
}
+ if (!i)
+ return ERR_PTR(-ENOMEM);
+ sg_mark_end(dsg + i - 1);
+ dlen = min(dlen, total);
- if (skb_linearize_cow(skb))
- goto out_ok;
-
- err = ipcomp_compress(x, skb);
+ acomp_request_set_params(req, sg, dsg, plen, dlen);
- if (err) {
- goto out_ok;
- }
-
- /* Install ipcomp header, convert into ipcomp datagram. */
- ipch = ip_comp_hdr(skb);
- ipch->nexthdr = *skb_mac_header(skb);
- ipch->flags = 0;
- ipch->cpi = htons((u16 )ntohl(x->id.spi));
- *skb_mac_header(skb) = IPPROTO_COMP;
-out_ok:
- skb_push(skb, -skb_network_offset(skb));
- return 0;
+ return req;
}
-EXPORT_SYMBOL_GPL(ipcomp_output);
-static void ipcomp_free_scratches(void)
+static int ipcomp_decompress(struct xfrm_state *x, struct sk_buff *skb)
{
- int i;
- void * __percpu *scratches;
-
- if (--ipcomp_scratch_users)
- return;
+ struct acomp_req *req;
+ int err;
- scratches = ipcomp_scratches;
- if (!scratches)
- return;
+ req = ipcomp_setup_req(x, skb, 0, IPCOMP_SCRATCH_SIZE);
+ err = PTR_ERR(req);
+ if (IS_ERR(req))
+ goto out;
- for_each_possible_cpu(i)
- vfree(*per_cpu_ptr(scratches, i));
+ acomp_request_set_callback(req, 0, ipcomp_input_done, skb);
+ err = crypto_acomp_decompress(req);
+ if (err == -EINPROGRESS)
+ return err;
- free_percpu(scratches);
- ipcomp_scratches = NULL;
+out:
+ return ipcomp_input_done2(skb, err);
}
-static void * __percpu *ipcomp_alloc_scratches(void)
+int ipcomp_input(struct xfrm_state *x, struct sk_buff *skb)
{
- void * __percpu *scratches;
- int i;
-
- if (ipcomp_scratch_users++)
- return ipcomp_scratches;
+ struct ip_comp_hdr *ipch __maybe_unused;
- scratches = alloc_percpu(void *);
- if (!scratches)
- return NULL;
-
- ipcomp_scratches = scratches;
+ if (!pskb_may_pull(skb, sizeof(*ipch)))
+ return -EINVAL;
- for_each_possible_cpu(i) {
- void *scratch;
+ skb->ip_summed = CHECKSUM_NONE;
- scratch = vmalloc_node(IPCOMP_SCRATCH_SIZE, cpu_to_node(i));
- if (!scratch)
- return NULL;
- *per_cpu_ptr(scratches, i) = scratch;
- }
+ /* Remove ipcomp header and decompress original payload */
+ __skb_pull(skb, sizeof(*ipch));
- return scratches;
+ return ipcomp_decompress(x, skb);
}
+EXPORT_SYMBOL_GPL(ipcomp_input);
-static void ipcomp_free_tfms(struct crypto_comp * __percpu *tfms)
+static int ipcomp_output_push(struct sk_buff *skb)
{
- struct ipcomp_tfms *pos;
- int cpu;
-
- list_for_each_entry(pos, &ipcomp_tfms_list, list) {
- if (pos->tfms == tfms)
- break;
- }
-
- WARN_ON(list_entry_is_head(pos, &ipcomp_tfms_list, list));
-
- if (--pos->users)
- return;
+ skb_push(skb, -skb_network_offset(skb));
+ return 0;
+}
- list_del(&pos->list);
- kfree(pos);
+static int ipcomp_output_done2(struct xfrm_state *x, struct sk_buff *skb,
+ int err)
+{
+ struct ip_comp_hdr *ipch;
- if (!tfms)
- return;
+ err = ipcomp_post_acomp(skb, err, sizeof(*ipch));
+ if (err)
+ goto out_ok;
- for_each_possible_cpu(cpu) {
- struct crypto_comp *tfm = *per_cpu_ptr(tfms, cpu);
- crypto_free_comp(tfm);
- }
- free_percpu(tfms);
+ /* Install ipcomp header, convert into ipcomp datagram. */
+ ipch = ip_comp_hdr(skb);
+ ipch->nexthdr = *skb_mac_header(skb);
+ ipch->flags = 0;
+ ipch->cpi = htons((u16 )ntohl(x->id.spi));
+ *skb_mac_header(skb) = IPPROTO_COMP;
+out_ok:
+ return ipcomp_output_push(skb);
}
-static struct crypto_comp * __percpu *ipcomp_alloc_tfms(const char *alg_name)
+static void ipcomp_output_done(void *data, int err)
{
- struct ipcomp_tfms *pos;
- struct crypto_comp * __percpu *tfms;
- int cpu;
+ struct ipcomp_req_extra *extra;
+ struct sk_buff *skb = data;
+ struct acomp_req *req;
+ req = ipcomp_cb(skb)->req;
+ extra = acomp_request_extra(req);
- list_for_each_entry(pos, &ipcomp_tfms_list, list) {
- struct crypto_comp *tfm;
+ xfrm_output_resume(skb_to_full_sk(skb), skb,
+ ipcomp_output_done2(extra->x, skb, err));
+}
- /* This can be any valid CPU ID so we don't need locking. */
- tfm = this_cpu_read(*pos->tfms);
+static int ipcomp_compress(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ip_comp_hdr *ipch __maybe_unused;
+ struct acomp_req *req;
+ int err;
- if (!strcmp(crypto_comp_name(tfm), alg_name)) {
- pos->users++;
- return pos->tfms;
- }
- }
+ req = ipcomp_setup_req(x, skb, sizeof(*ipch),
+ skb->len - sizeof(*ipch));
+ err = PTR_ERR(req);
+ if (IS_ERR(req))
+ goto out;
- pos = kmalloc(sizeof(*pos), GFP_KERNEL);
- if (!pos)
- return NULL;
+ acomp_request_set_callback(req, 0, ipcomp_output_done, skb);
+ err = crypto_acomp_compress(req);
+ if (err == -EINPROGRESS)
+ return err;
- pos->users = 1;
- INIT_LIST_HEAD(&pos->list);
- list_add(&pos->list, &ipcomp_tfms_list);
+out:
+ return ipcomp_output_done2(x, skb, err);
+}
- pos->tfms = tfms = alloc_percpu(struct crypto_comp *);
- if (!tfms)
- goto error;
+int ipcomp_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ipcomp_data *ipcd = x->data;
- for_each_possible_cpu(cpu) {
- struct crypto_comp *tfm = crypto_alloc_comp(alg_name, 0,
- CRYPTO_ALG_ASYNC);
- if (IS_ERR(tfm))
- goto error;
- *per_cpu_ptr(tfms, cpu) = tfm;
+ if (skb->len < ipcd->threshold) {
+ /* Don't bother compressing */
+ return ipcomp_output_push(skb);
}
- return tfms;
-
-error:
- ipcomp_free_tfms(tfms);
- return NULL;
+ return ipcomp_compress(x, skb);
}
+EXPORT_SYMBOL_GPL(ipcomp_output);
static void ipcomp_free_data(struct ipcomp_data *ipcd)
{
- if (ipcd->tfms)
- ipcomp_free_tfms(ipcd->tfms);
- ipcomp_free_scratches();
+ crypto_free_acomp(ipcd->tfm);
}
void ipcomp_destroy(struct xfrm_state *x)
@@ -315,10 +313,7 @@ void ipcomp_destroy(struct xfrm_state *x)
struct ipcomp_data *ipcd = x->data;
if (!ipcd)
return;
- xfrm_state_delete_tunnel(x);
- mutex_lock(&ipcomp_resource_mutex);
ipcomp_free_data(ipcd);
- mutex_unlock(&ipcomp_resource_mutex);
kfree(ipcd);
}
EXPORT_SYMBOL_GPL(ipcomp_destroy);
@@ -345,14 +340,9 @@ int ipcomp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
if (!ipcd)
goto out;
- mutex_lock(&ipcomp_resource_mutex);
- if (!ipcomp_alloc_scratches())
- goto error;
-
- ipcd->tfms = ipcomp_alloc_tfms(x->calg->alg_name);
- if (!ipcd->tfms)
+ ipcd->tfm = crypto_alloc_acomp(x->calg->alg_name, 0, 0);
+ if (IS_ERR(ipcd->tfm))
goto error;
- mutex_unlock(&ipcomp_resource_mutex);
calg_desc = xfrm_calg_get_byname(x->calg->alg_name, 0);
BUG_ON(!calg_desc);
@@ -364,7 +354,6 @@ out:
error:
ipcomp_free_data(ipcd);
- mutex_unlock(&ipcomp_resource_mutex);
kfree(ipcd);
goto out;
}
diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c
index 755f1eea8bfa..3b6d7284fc70 100644
--- a/net/xfrm/xfrm_iptfs.c
+++ b/net/xfrm/xfrm_iptfs.c
@@ -2625,12 +2625,10 @@ static void __iptfs_init_state(struct xfrm_state *x,
struct xfrm_iptfs_data *xtfs)
{
__skb_queue_head_init(&xtfs->queue);
- hrtimer_init(&xtfs->iptfs_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
- xtfs->iptfs_timer.function = iptfs_delay_timer;
+ hrtimer_setup(&xtfs->iptfs_timer, iptfs_delay_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
spin_lock_init(&xtfs->drop_lock);
- hrtimer_init(&xtfs->drop_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
- xtfs->drop_timer.function = iptfs_drop_timer;
+ hrtimer_setup(&xtfs->drop_timer, iptfs_drop_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
/* Modify type (esp) adjustment values */
diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c
index 82f0a301683f..ebf95d48e86c 100644
--- a/net/xfrm/xfrm_nat_keepalive.c
+++ b/net/xfrm/xfrm_nat_keepalive.c
@@ -9,9 +9,13 @@
#include <net/ip6_checksum.h>
#include <net/xfrm.h>
-static DEFINE_PER_CPU(struct sock *, nat_keepalive_sk_ipv4);
+static DEFINE_PER_CPU(struct sock_bh_locked, nat_keepalive_sk_ipv4) = {
+ .bh_lock = INIT_LOCAL_LOCK(bh_lock),
+};
#if IS_ENABLED(CONFIG_IPV6)
-static DEFINE_PER_CPU(struct sock *, nat_keepalive_sk_ipv6);
+static DEFINE_PER_CPU(struct sock_bh_locked, nat_keepalive_sk_ipv6) = {
+ .bh_lock = INIT_LOCAL_LOCK(bh_lock),
+};
#endif
struct nat_keepalive {
@@ -56,10 +60,12 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb,
skb_dst_set(skb, &rt->dst);
- sk = *this_cpu_ptr(&nat_keepalive_sk_ipv4);
+ local_lock_nested_bh(&nat_keepalive_sk_ipv4.bh_lock);
+ sk = this_cpu_read(nat_keepalive_sk_ipv4.sock);
sock_net_set(sk, net);
err = ip_build_and_send_pkt(skb, sk, fl4.saddr, fl4.daddr, NULL, tos);
sock_net_set(sk, &init_net);
+ local_unlock_nested_bh(&nat_keepalive_sk_ipv4.bh_lock);
return err;
}
@@ -89,15 +95,19 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb,
fl6.fl6_sport = ka->encap_sport;
fl6.fl6_dport = ka->encap_dport;
- sk = *this_cpu_ptr(&nat_keepalive_sk_ipv6);
+ local_lock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock);
+ sk = this_cpu_read(nat_keepalive_sk_ipv6.sock);
sock_net_set(sk, net);
dst = ipv6_stub->ipv6_dst_lookup_flow(net, sk, &fl6, NULL);
- if (IS_ERR(dst))
+ if (IS_ERR(dst)) {
+ local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock);
return PTR_ERR(dst);
+ }
skb_dst_set(skb, dst);
err = ipv6_stub->ip6_xmit(sk, skb, &fl6, skb->mark, NULL, 0, 0);
sock_net_set(sk, &init_net);
+ local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock);
return err;
}
#endif
@@ -202,7 +212,7 @@ static void nat_keepalive_work(struct work_struct *work)
(ctx.next_run - ctx.now) * HZ);
}
-static int nat_keepalive_sk_init(struct sock * __percpu *socks,
+static int nat_keepalive_sk_init(struct sock_bh_locked __percpu *socks,
unsigned short family)
{
struct sock *sk;
@@ -214,22 +224,22 @@ static int nat_keepalive_sk_init(struct sock * __percpu *socks,
if (err < 0)
goto err;
- *per_cpu_ptr(socks, i) = sk;
+ per_cpu_ptr(socks, i)->sock = sk;
}
return 0;
err:
for_each_possible_cpu(i)
- inet_ctl_sock_destroy(*per_cpu_ptr(socks, i));
+ inet_ctl_sock_destroy(per_cpu_ptr(socks, i)->sock);
return err;
}
-static void nat_keepalive_sk_fini(struct sock * __percpu *socks)
+static void nat_keepalive_sk_fini(struct sock_bh_locked __percpu *socks)
{
int i;
for_each_possible_cpu(i)
- inet_ctl_sock_destroy(*per_cpu_ptr(socks, i));
+ inet_ctl_sock_destroy(per_cpu_ptr(socks, i)->sock);
}
void xfrm_nat_keepalive_state_updated(struct xfrm_state *x)
diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c
index 3cabc87978dd..9077730ff7d0 100644
--- a/net/xfrm/xfrm_output.c
+++ b/net/xfrm/xfrm_output.c
@@ -827,7 +827,7 @@ out:
}
EXPORT_SYMBOL_GPL(xfrm_output);
-static int xfrm4_tunnel_check_size(struct sk_buff *skb)
+int xfrm4_tunnel_check_size(struct sk_buff *skb)
{
int mtu, ret = 0;
@@ -853,6 +853,7 @@ static int xfrm4_tunnel_check_size(struct sk_buff *skb)
out:
return ret;
}
+EXPORT_SYMBOL_GPL(xfrm4_tunnel_check_size);
static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
{
@@ -875,7 +876,7 @@ static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
}
#if IS_ENABLED(CONFIG_IPV6)
-static int xfrm6_tunnel_check_size(struct sk_buff *skb)
+int xfrm6_tunnel_check_size(struct sk_buff *skb)
{
int mtu, ret = 0;
struct dst_entry *dst = skb_dst(skb);
@@ -905,6 +906,7 @@ static int xfrm6_tunnel_check_size(struct sk_buff *skb)
out:
return ret;
}
+EXPORT_SYMBOL_GPL(xfrm6_tunnel_check_size);
#endif
static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 6551e588fe52..094d2454602e 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -353,7 +353,7 @@ static inline unsigned long make_jiffies(long secs)
static void xfrm_policy_timer(struct timer_list *t)
{
- struct xfrm_policy *xp = from_timer(xp, t, timer);
+ struct xfrm_policy *xp = timer_container_of(xp, t, timer);
time64_t now = ktime_get_real_seconds();
time64_t next = TIME64_MAX;
int warn = 0;
@@ -462,7 +462,7 @@ void xfrm_policy_destroy(struct xfrm_policy *policy)
{
BUG_ON(!policy->walk.dead);
- if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
+ if (timer_delete(&policy->timer) || timer_delete(&policy->polq.hold_timer))
BUG();
xfrm_dev_policy_free(policy);
@@ -487,11 +487,11 @@ static void xfrm_policy_kill(struct xfrm_policy *policy)
atomic_inc(&policy->genid);
- if (del_timer(&policy->polq.hold_timer))
+ if (timer_delete(&policy->polq.hold_timer))
xfrm_pol_put(policy);
skb_queue_purge(&policy->polq.hold_queue);
- if (del_timer(&policy->timer))
+ if (timer_delete(&policy->timer))
xfrm_pol_put(policy);
/* XXX: Flush state cache */
@@ -1469,7 +1469,7 @@ static void xfrm_policy_requeue(struct xfrm_policy *old,
spin_lock_bh(&pq->hold_queue.lock);
skb_queue_splice_init(&pq->hold_queue, &list);
- if (del_timer(&pq->hold_timer))
+ if (timer_delete(&pq->hold_timer))
xfrm_pol_put(old);
spin_unlock_bh(&pq->hold_queue.lock);
@@ -1581,6 +1581,9 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
struct xfrm_policy *delpol;
struct hlist_head *chain;
+ /* Sanitize mark before store */
+ policy->mark.v &= policy->mark.m;
+
spin_lock_bh(&net->xfrm.xfrm_policy_lock);
chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
if (chain)
@@ -2895,7 +2898,7 @@ static void xfrm_policy_queue_process(struct timer_list *t)
struct sk_buff *skb;
struct sock *sk;
struct dst_entry *dst;
- struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);
+ struct xfrm_policy *pol = timer_container_of(pol, t, polq.hold_timer);
struct net *net = xp_net(pol);
struct xfrm_policy_queue *pq = &pol->polq;
struct flowi fl;
@@ -3004,7 +3007,7 @@ static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *s
sched_next = jiffies + pq->timeout;
- if (del_timer(&pq->hold_timer)) {
+ if (timer_delete(&pq->hold_timer)) {
if (time_before(pq->hold_timer.expires, sched_next))
sched_next = pq->hold_timer.expires;
xfrm_pol_put(pol);
@@ -3294,7 +3297,7 @@ no_transform:
ok:
xfrm_pols_put(pols, drop_pols);
- if (dst && dst->xfrm &&
+ if (dst->xfrm &&
(dst->xfrm->props.mode == XFRM_MODE_TUNNEL ||
dst->xfrm->props.mode == XFRM_MODE_IPTFS))
dst->flags |= DST_XFRM_TUNNEL;
@@ -4630,7 +4633,7 @@ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
struct xfrm_migrate *m, int num_migrate,
struct xfrm_kmaddress *k, struct net *net,
struct xfrm_encap_tmpl *encap, u32 if_id,
- struct netlink_ext_ack *extack)
+ struct netlink_ext_ack *extack, struct xfrm_user_offload *xuo)
{
int i, err, nx_cur = 0, nx_new = 0;
struct xfrm_policy *pol = NULL;
@@ -4663,7 +4666,7 @@ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
x_cur[nx_cur] = x;
nx_cur++;
- xc = xfrm_state_migrate(x, mp, encap);
+ xc = xfrm_state_migrate(x, mp, encap, net, xuo, extack);
if (xc) {
x_new[nx_new] = xc;
nx_new++;
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 69af5964c886..86337453709b 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -424,11 +424,10 @@ void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
}
EXPORT_SYMBOL(xfrm_unregister_type_offload);
-void xfrm_set_type_offload(struct xfrm_state *x)
+void xfrm_set_type_offload(struct xfrm_state *x, bool try_load)
{
const struct xfrm_type_offload *type = NULL;
struct xfrm_state_afinfo *afinfo;
- bool try_load = true;
retry:
afinfo = xfrm_state_get_afinfo(x->props.family);
@@ -593,20 +592,21 @@ void xfrm_state_free(struct xfrm_state *x)
}
EXPORT_SYMBOL(xfrm_state_free);
-static void ___xfrm_state_destroy(struct xfrm_state *x)
+static void xfrm_state_gc_destroy(struct xfrm_state *x)
{
if (x->mode_cbs && x->mode_cbs->destroy_state)
x->mode_cbs->destroy_state(x);
hrtimer_cancel(&x->mtimer);
- del_timer_sync(&x->rtimer);
- kfree(x->aead);
- kfree(x->aalg);
- kfree(x->ealg);
+ timer_delete_sync(&x->rtimer);
+ kfree_sensitive(x->aead);
+ kfree_sensitive(x->aalg);
+ kfree_sensitive(x->ealg);
kfree(x->calg);
kfree(x->encap);
kfree(x->coaddr);
kfree(x->replay_esn);
kfree(x->preplay_esn);
+ xfrm_unset_type_offload(x);
if (x->type) {
x->type->destructor(x);
xfrm_put_type(x->type);
@@ -631,7 +631,7 @@ static void xfrm_state_gc_task(struct work_struct *work)
synchronize_rcu();
hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
- ___xfrm_state_destroy(x);
+ xfrm_state_gc_destroy(x);
}
static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
@@ -742,8 +742,8 @@ struct xfrm_state *xfrm_state_alloc(struct net *net)
INIT_HLIST_NODE(&x->bysrc);
INIT_HLIST_NODE(&x->byspi);
INIT_HLIST_NODE(&x->byseq);
- hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
- x->mtimer.function = xfrm_timer_handler;
+ hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME,
+ HRTIMER_MODE_ABS_SOFT);
timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
x->curlft.add_time = ktime_get_real_seconds();
x->lft.soft_byte_limit = XFRM_INF;
@@ -767,7 +767,7 @@ void xfrm_dev_state_delete(struct xfrm_state *x)
struct net_device *dev = READ_ONCE(xso->dev);
if (dev) {
- dev->xfrmdev_ops->xdo_dev_state_delete(x);
+ dev->xfrmdev_ops->xdo_dev_state_delete(dev, x);
spin_lock_bh(&xfrm_state_dev_gc_lock);
hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list);
spin_unlock_bh(&xfrm_state_dev_gc_lock);
@@ -780,8 +780,6 @@ void xfrm_dev_state_free(struct xfrm_state *x)
struct xfrm_dev_offload *xso = &x->xso;
struct net_device *dev = READ_ONCE(xso->dev);
- xfrm_unset_type_offload(x);
-
if (dev && dev->xfrmdev_ops) {
spin_lock_bh(&xfrm_state_dev_gc_lock);
if (!hlist_unhashed(&x->dev_gclist))
@@ -789,7 +787,7 @@ void xfrm_dev_state_free(struct xfrm_state *x)
spin_unlock_bh(&xfrm_state_dev_gc_lock);
if (dev->xfrmdev_ops->xdo_dev_state_free)
- dev->xfrmdev_ops->xdo_dev_state_free(x);
+ dev->xfrmdev_ops->xdo_dev_state_free(dev, x);
WRITE_ONCE(xso->dev, NULL);
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
netdev_put(dev, &xso->dev_tracker);
@@ -797,22 +795,18 @@ void xfrm_dev_state_free(struct xfrm_state *x)
}
#endif
-void __xfrm_state_destroy(struct xfrm_state *x, bool sync)
+void __xfrm_state_destroy(struct xfrm_state *x)
{
WARN_ON(x->km.state != XFRM_STATE_DEAD);
- if (sync) {
- synchronize_rcu();
- ___xfrm_state_destroy(x);
- } else {
- spin_lock_bh(&xfrm_state_gc_lock);
- hlist_add_head(&x->gclist, &xfrm_state_gc_list);
- spin_unlock_bh(&xfrm_state_gc_lock);
- schedule_work(&xfrm_state_gc_work);
- }
+ spin_lock_bh(&xfrm_state_gc_lock);
+ hlist_add_head(&x->gclist, &xfrm_state_gc_list);
+ spin_unlock_bh(&xfrm_state_gc_lock);
+ schedule_work(&xfrm_state_gc_work);
}
EXPORT_SYMBOL(__xfrm_state_destroy);
+static void xfrm_state_delete_tunnel(struct xfrm_state *x);
int __xfrm_state_delete(struct xfrm_state *x)
{
struct net *net = xs_net(x);
@@ -838,11 +832,10 @@ int __xfrm_state_delete(struct xfrm_state *x)
xfrm_nat_keepalive_state_updated(x);
spin_unlock(&net->xfrm.xfrm_state_lock);
- if (x->encap_sk)
- sock_put(rcu_dereference_raw(x->encap_sk));
-
xfrm_dev_state_delete(x);
+ xfrm_state_delete_tunnel(x);
+
/* All xfrm_state objects are created by xfrm_state_alloc.
* The xfrm_state_alloc call gives a reference, and that
* is what we are dropping here.
@@ -924,7 +917,7 @@ xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool
}
#endif
-int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync)
+int xfrm_state_flush(struct net *net, u8 proto, bool task_valid)
{
int i, err = 0, cnt = 0;
@@ -946,10 +939,7 @@ restart:
err = xfrm_state_delete(x);
xfrm_audit_state_delete(x, err ? 0 : 1,
task_valid);
- if (sync)
- xfrm_state_put_sync(x);
- else
- xfrm_state_put(x);
+ xfrm_state_put(x);
if (!err)
cnt++;
@@ -1310,14 +1300,8 @@ static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
const struct flowi *fl, unsigned short family,
struct xfrm_state **best, int *acq_in_progress,
- int *error)
+ int *error, unsigned int pcpu_id)
{
- /* We need the cpu id just as a lookup key,
- * we don't require it to be stable.
- */
- unsigned int pcpu_id = get_cpu();
- put_cpu();
-
/* Resolution logic:
* 1. There is a valid state with matching selector. Done.
* 2. Valid state with inappropriate selector. Skip.
@@ -1384,14 +1368,15 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
/* We need the cpu id just as a lookup key,
* we don't require it to be stable.
*/
- pcpu_id = get_cpu();
- put_cpu();
+ pcpu_id = raw_smp_processor_id();
to_put = NULL;
sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
rcu_read_lock();
+ xfrm_hash_ptrs_get(net, &state_ptrs);
+
hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {
if (x->props.family == encap_family &&
x->props.reqid == tmpl->reqid &&
@@ -1403,7 +1388,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
tmpl->id.proto == x->id.proto &&
(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
xfrm_state_look_at(pol, x, fl, encap_family,
- &best, &acquire_in_progress, &error);
+ &best, &acquire_in_progress, &error, pcpu_id);
}
if (best)
@@ -1420,7 +1405,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
tmpl->id.proto == x->id.proto &&
(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
xfrm_state_look_at(pol, x, fl, family,
- &best, &acquire_in_progress, &error);
+ &best, &acquire_in_progress, &error, pcpu_id);
}
cached:
@@ -1432,8 +1417,6 @@ cached:
else if (acquire_in_progress) /* XXX: acquire_in_progress should not happen */
WARN_ON(1);
- xfrm_hash_ptrs_get(net, &state_ptrs);
-
h = __xfrm_dst_hash(daddr, saddr, tmpl->reqid, encap_family, state_ptrs.hmask);
hlist_for_each_entry_rcu(x, state_ptrs.bydst + h, bydst) {
#ifdef CONFIG_XFRM_OFFLOAD
@@ -1463,7 +1446,7 @@ cached:
tmpl->id.proto == x->id.proto &&
(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
xfrm_state_look_at(pol, x, fl, family,
- &best, &acquire_in_progress, &error);
+ &best, &acquire_in_progress, &error, pcpu_id);
}
if (best || acquire_in_progress)
goto found;
@@ -1498,7 +1481,7 @@ cached:
tmpl->id.proto == x->id.proto &&
(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
xfrm_state_look_at(pol, x, fl, family,
- &best, &acquire_in_progress, &error);
+ &best, &acquire_in_progress, &error, pcpu_id);
}
found:
@@ -1551,19 +1534,19 @@ found:
if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
struct xfrm_dev_offload *xdo = &pol->xdo;
struct xfrm_dev_offload *xso = &x->xso;
+ struct net_device *dev = xdo->dev;
xso->type = XFRM_DEV_OFFLOAD_PACKET;
xso->dir = xdo->dir;
- xso->dev = xdo->dev;
- xso->real_dev = xdo->real_dev;
+ xso->dev = dev;
xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
- netdev_hold(xso->dev, &xso->dev_tracker, GFP_ATOMIC);
- error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL);
+ netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC);
+ error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x,
+ NULL);
if (error) {
xso->dir = 0;
- netdev_put(xso->dev, &xso->dev_tracker);
+ netdev_put(dev, &xso->dev_tracker);
xso->dev = NULL;
- xso->real_dev = NULL;
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
x->km.state = XFRM_STATE_DEAD;
to_put = x;
@@ -1714,6 +1697,26 @@ struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
}
EXPORT_SYMBOL(xfrm_state_lookup_byspi);
+static struct xfrm_state *xfrm_state_lookup_spi_proto(struct net *net, __be32 spi, u8 proto)
+{
+ struct xfrm_state *x;
+ unsigned int i;
+
+ rcu_read_lock();
+ for (i = 0; i <= net->xfrm.state_hmask; i++) {
+ hlist_for_each_entry_rcu(x, &net->xfrm.state_byspi[i], byspi) {
+ if (x->id.spi == spi && x->id.proto == proto) {
+ if (!xfrm_state_hold_rcu(x))
+ continue;
+ rcu_read_unlock();
+ return x;
+ }
+ }
+ }
+ rcu_read_unlock();
+ return NULL;
+}
+
static void __xfrm_state_insert(struct xfrm_state *x)
{
struct net *net = xs_net(x);
@@ -1721,6 +1724,9 @@ static void __xfrm_state_insert(struct xfrm_state *x)
list_add(&x->km.all, &net->xfrm.state_all);
+ /* Sanitize mark before store */
+ x->mark.v &= x->mark.m;
+
h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
x->props.reqid, x->props.family);
XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
@@ -1958,8 +1964,9 @@ static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *secu
return 0;
}
-static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
- struct xfrm_encap_tmpl *encap)
+static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig,
+ struct xfrm_encap_tmpl *encap,
+ struct xfrm_migrate *m)
{
struct net *net = xs_net(orig);
struct xfrm_state *x = xfrm_state_alloc(net);
@@ -2058,6 +2065,11 @@ static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
goto error;
}
+
+ x->props.family = m->new_family;
+ memcpy(&x->id.daddr, &m->new_daddr, sizeof(x->id.daddr));
+ memcpy(&x->props.saddr, &m->new_saddr, sizeof(x->props.saddr));
+
return x;
error:
@@ -2120,21 +2132,23 @@ EXPORT_SYMBOL(xfrm_migrate_state_find);
struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
struct xfrm_migrate *m,
- struct xfrm_encap_tmpl *encap)
+ struct xfrm_encap_tmpl *encap,
+ struct net *net,
+ struct xfrm_user_offload *xuo,
+ struct netlink_ext_ack *extack)
{
struct xfrm_state *xc;
- xc = xfrm_state_clone(x, encap);
+ xc = xfrm_state_clone_and_setup(x, encap, m);
if (!xc)
return NULL;
- xc->props.family = m->new_family;
-
if (xfrm_init_state(xc) < 0)
goto error;
- memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
- memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
+ /* configure the hardware if offload is requested */
+ if (xuo && xfrm_dev_state_add(net, xc, xuo, extack))
+ goto error;
/* add state */
if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {
@@ -2313,12 +2327,12 @@ xfrm_state_lookup_byaddr(struct net *net, u32 mark,
struct xfrm_hash_state_ptrs state_ptrs;
struct xfrm_state *x;
- spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ rcu_read_lock();
xfrm_hash_ptrs_get(net, &state_ptrs);
x = __xfrm_state_lookup_byaddr(&state_ptrs, mark, daddr, saddr, proto, family);
- spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ rcu_read_unlock();
return x;
}
EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
@@ -2547,10 +2561,8 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
unsigned int h;
struct xfrm_state *x0;
int err = -ENOENT;
- __be32 minspi = htonl(low);
- __be32 maxspi = htonl(high);
+ u32 range = high - low + 1;
__be32 newspi = 0;
- u32 mark = x->mark.v & x->mark.m;
spin_lock_bh(&x->lock);
if (x->km.state == XFRM_STATE_DEAD) {
@@ -2564,38 +2576,34 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
err = -ENOENT;
- if (minspi == maxspi) {
- x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);
- if (x0) {
- NL_SET_ERR_MSG(extack, "Requested SPI is already in use");
- xfrm_state_put(x0);
+ for (h = 0; h < range; h++) {
+ u32 spi = (low == high) ? low : get_random_u32_inclusive(low, high);
+ newspi = htonl(spi);
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x0 = xfrm_state_lookup_spi_proto(net, newspi, x->id.proto);
+ if (!x0) {
+ x->id.spi = newspi;
+ h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family);
+ XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, x->xso.type);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ err = 0;
goto unlock;
}
- newspi = minspi;
- } else {
- u32 spi = 0;
- for (h = 0; h < high-low+1; h++) {
- spi = get_random_u32_inclusive(low, high);
- x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);
- if (x0 == NULL) {
- newspi = htonl(spi);
- break;
- }
- xfrm_state_put(x0);
+ xfrm_state_put(x0);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ if (signal_pending(current)) {
+ err = -ERESTARTSYS;
+ goto unlock;
}
+
+ if (low == high)
+ break;
}
- if (newspi) {
- spin_lock_bh(&net->xfrm.xfrm_state_lock);
- x->id.spi = newspi;
- h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);
- XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
- x->xso.type);
- spin_unlock_bh(&net->xfrm.xfrm_state_lock);
- err = 0;
- } else {
+ if (err)
NL_SET_ERR_MSG(extack, "No SPI available in the requested range");
- }
unlock:
spin_unlock_bh(&x->lock);
@@ -2689,7 +2697,7 @@ EXPORT_SYMBOL(xfrm_state_walk_done);
static void xfrm_replay_timer_handler(struct timer_list *t)
{
- struct xfrm_state *x = from_timer(x, t, rtimer);
+ struct xfrm_state *x = timer_container_of(x, t, rtimer);
spin_lock(&x->lock);
@@ -3069,20 +3077,17 @@ void xfrm_flush_gc(void)
}
EXPORT_SYMBOL(xfrm_flush_gc);
-/* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
-void xfrm_state_delete_tunnel(struct xfrm_state *x)
+static void xfrm_state_delete_tunnel(struct xfrm_state *x)
{
if (x->tunnel) {
struct xfrm_state *t = x->tunnel;
- if (atomic_read(&t->tunnel_users) == 2)
+ if (atomic_dec_return(&t->tunnel_users) == 1)
xfrm_state_delete(t);
- atomic_dec(&t->tunnel_users);
- xfrm_state_put_sync(t);
+ xfrm_state_put(t);
x->tunnel = NULL;
}
}
-EXPORT_SYMBOL(xfrm_state_delete_tunnel);
u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
{
@@ -3120,8 +3125,7 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
}
EXPORT_SYMBOL_GPL(xfrm_state_mtu);
-int __xfrm_init_state(struct xfrm_state *x, bool init_replay,
- struct netlink_ext_ack *extack)
+int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
{
const struct xfrm_mode *inner_mode;
const struct xfrm_mode *outer_mode;
@@ -3188,12 +3192,6 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay,
}
x->outer_mode = *outer_mode;
- if (init_replay) {
- err = xfrm_init_replay(x, extack);
- if (err)
- goto error;
- }
-
if (x->nat_keepalive_interval) {
if (x->dir != XFRM_SA_DIR_OUT) {
NL_SET_ERR_MSG(extack, "NAT keepalive is only supported for outbound SAs");
@@ -3225,11 +3223,16 @@ int xfrm_init_state(struct xfrm_state *x)
{
int err;
- err = __xfrm_init_state(x, true, NULL);
- if (!err)
- x->km.state = XFRM_STATE_VALID;
+ err = __xfrm_init_state(x, NULL);
+ if (err)
+ return err;
- return err;
+ err = xfrm_init_replay(x, NULL);
+ if (err)
+ return err;
+
+ x->km.state = XFRM_STATE_VALID;
+ return 0;
}
EXPORT_SYMBOL(xfrm_init_state);
@@ -3289,8 +3292,8 @@ void xfrm_state_fini(struct net *net)
unsigned int sz;
flush_work(&net->xfrm.state_hash_work);
+ xfrm_state_flush(net, 0, false);
flush_work(&xfrm_state_gc_work);
- xfrm_state_flush(net, 0, false, true);
WARN_ON(!list_empty(&net->xfrm.state_all));
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 82a768500999..684239018bec 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -178,6 +178,28 @@ static inline int verify_replay(struct xfrm_usersa_info *p,
"Replay seq and seq_hi should be 0 for output SA");
return -EINVAL;
}
+
+ if (!(p->flags & XFRM_STATE_ESN)) {
+ if (rs->oseq_hi) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay oseq_hi should be 0 in non-ESN mode for output SA");
+ return -EINVAL;
+ }
+ if (rs->oseq == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay oseq should be less than 0xFFFFFFFF in non-ESN mode for output SA");
+ return -EINVAL;
+ }
+ } else {
+ if (rs->oseq == U32_MAX && rs->oseq_hi == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay oseq and oseq_hi should be less than 0xFFFFFFFF for output SA");
+ return -EINVAL;
+ }
+ }
if (rs->bmp_len) {
NL_SET_ERR_MSG(extack, "Replay bmp_len should 0 for output SA");
return -EINVAL;
@@ -190,6 +212,28 @@ static inline int verify_replay(struct xfrm_usersa_info *p,
"Replay oseq and oseq_hi should be 0 for input SA");
return -EINVAL;
}
+ if (!(p->flags & XFRM_STATE_ESN)) {
+ if (rs->seq_hi) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay seq_hi should be 0 in non-ESN mode for input SA");
+ return -EINVAL;
+ }
+
+ if (rs->seq == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay seq should be less than 0xFFFFFFFF in non-ESN mode for input SA");
+ return -EINVAL;
+ }
+ } else {
+ if (rs->seq == U32_MAX && rs->seq_hi == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay seq and seq_hi should be less than 0xFFFFFFFF for input SA");
+ return -EINVAL;
+ }
+ }
}
return 0;
@@ -907,7 +951,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
goto error;
}
- err = __xfrm_init_state(x, false, extack);
+ err = __xfrm_init_state(x, extack);
if (err)
goto error;
@@ -933,6 +977,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
/* override default values from above */
xfrm_update_ae_params(x, attrs, 0);
+ xfrm_set_type_offload(x, attrs[XFRMA_OFFLOAD_DEV]);
/* configure the hardware if offload is requested */
if (attrs[XFRMA_OFFLOAD_DEV]) {
err = xfrm_dev_state_add(net, x,
@@ -1161,7 +1206,7 @@ static int copy_to_user_auth(struct xfrm_algo_auth *auth, struct sk_buff *skb)
if (!nla)
return -EMSGSIZE;
algo = nla_data(nla);
- strscpy_pad(algo->alg_name, auth->alg_name, sizeof(algo->alg_name));
+ strscpy_pad(algo->alg_name, auth->alg_name);
if (redact_secret && auth->alg_key_len)
memset(algo->alg_key, 0, (auth->alg_key_len + 7) / 8);
@@ -1174,7 +1219,7 @@ static int copy_to_user_auth(struct xfrm_algo_auth *auth, struct sk_buff *skb)
if (!nla)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, auth->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, auth->alg_name);
ap->alg_key_len = auth->alg_key_len;
ap->alg_trunc_len = auth->alg_trunc_len;
if (redact_secret && auth->alg_key_len)
@@ -1195,7 +1240,7 @@ static int copy_to_user_aead(struct xfrm_algo_aead *aead, struct sk_buff *skb)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, aead->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, aead->alg_name);
ap->alg_key_len = aead->alg_key_len;
ap->alg_icv_len = aead->alg_icv_len;
@@ -1217,7 +1262,7 @@ static int copy_to_user_ealg(struct xfrm_algo *ealg, struct sk_buff *skb)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, ealg->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, ealg->alg_name);
ap->alg_key_len = ealg->alg_key_len;
if (redact_secret && ealg->alg_key_len)
@@ -1238,7 +1283,7 @@ static int copy_to_user_calg(struct xfrm_algo *calg, struct sk_buff *skb)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, calg->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, calg->alg_name);
ap->alg_key_len = 0;
return 0;
@@ -2590,7 +2635,7 @@ static int xfrm_flush_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
struct xfrm_usersa_flush *p = nlmsg_data(nlh);
int err;
- err = xfrm_state_flush(net, p->proto, true, false);
+ err = xfrm_state_flush(net, p->proto, true);
if (err) {
if (err == -ESRCH) /* empty table */
return 0;
@@ -3057,6 +3102,7 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
int n = 0;
struct net *net = sock_net(skb->sk);
struct xfrm_encap_tmpl *encap = NULL;
+ struct xfrm_user_offload *xuo = NULL;
u32 if_id = 0;
if (!attrs[XFRMA_MIGRATE]) {
@@ -3087,11 +3133,19 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
if (attrs[XFRMA_IF_ID])
if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+ if (attrs[XFRMA_OFFLOAD_DEV]) {
+ xuo = kmemdup(nla_data(attrs[XFRMA_OFFLOAD_DEV]),
+ sizeof(*xuo), GFP_KERNEL);
+ if (!xuo) {
+ err = -ENOMEM;
+ goto error;
+ }
+ }
err = xfrm_migrate(&pi->sel, pi->dir, type, m, n, kmp, net, encap,
- if_id, extack);
-
+ if_id, extack, xuo);
+error:
kfree(encap);
-
+ kfree(xuo);
return err;
}
#else
generated by cgit v1.2.3 (git 2.39.0) at 2025-08-21 05:01:17 +0300