summaryrefslogtreecommitdiff
path: root/net/core
diff options
context:
space:
mode:
Diffstat (limited to 'net/core')
-rw-r--r--net/core/bpf_sk_storage.c14
-rw-r--r--net/core/dev.c97
-rw-r--r--net/core/dev.h4
-rw-r--r--net/core/dev_addr_lists.c385
-rw-r--r--net/core/dev_addr_lists_test.c387
-rw-r--r--net/core/dev_api.c3
-rw-r--r--net/core/dev_ioctl.c6
-rw-r--r--net/core/devmem.c3
-rw-r--r--net/core/failover.c6
-rw-r--r--net/core/filter.c59
-rw-r--r--net/core/flow_dissector.c13
-rw-r--r--net/core/neighbour.c10
-rw-r--r--net/core/netpoll.c42
-rw-r--r--net/core/page_pool.c10
-rw-r--r--net/core/rtnetlink.c2
-rw-r--r--net/core/sock_map.c43
16 files changed, 946 insertions, 138 deletions
diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index 14eb7812bda4..ecd659f79fd4 100644
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -172,7 +172,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
struct bpf_map *map;
smap = rcu_dereference(SDATA(selem)->smap);
- if (!(smap->map.map_flags & BPF_F_CLONE))
+ if (!smap || !(smap->map.map_flags & BPF_F_CLONE))
continue;
/* Note that for lockless listeners adding new element
@@ -531,10 +531,10 @@ err_free:
}
EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
-static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
+static int diag_get(struct bpf_local_storage_map *smap,
+ struct bpf_local_storage_data *sdata, struct sk_buff *skb)
{
struct nlattr *nla_stg, *nla_value;
- struct bpf_local_storage_map *smap;
/* It cannot exceed max nlattr's payload */
BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
@@ -543,7 +543,6 @@ static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
if (!nla_stg)
return -EMSGSIZE;
- smap = rcu_dereference(sdata->smap);
if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
goto errout;
@@ -558,6 +557,7 @@ static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
sdata->data, true);
else
copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
+ check_and_init_map_value(&smap->map, nla_data(nla_value));
nla_nest_end(skb, nla_stg);
return 0;
@@ -596,9 +596,11 @@ static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
saved_len = skb->len;
hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
smap = rcu_dereference(SDATA(selem)->smap);
+ if (!smap)
+ continue;
diag_size += nla_value_size(smap->map.value_size);
- if (nla_stgs && diag_get(SDATA(selem), skb))
+ if (nla_stgs && diag_get(smap, SDATA(selem), skb))
/* Continue to learn diag_size */
err = -EMSGSIZE;
}
@@ -665,7 +667,7 @@ int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
diag_size += nla_value_size(diag->maps[i]->value_size);
- if (nla_stgs && diag_get(sdata, skb))
+ if (nla_stgs && diag_get((struct bpf_local_storage_map *)diag->maps[i], sdata, skb))
/* Continue to learn diag_size */
err = -EMSGSIZE;
}
diff --git a/net/core/dev.c b/net/core/dev.c
index e59f6025067c..0c6c270d9f7d 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -371,7 +371,7 @@ static void netdev_name_node_alt_free(struct rcu_head *head)
static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
{
netdev_name_node_del(name_node);
- list_del(&name_node->list);
+ list_del_rcu(&name_node->list);
call_rcu(&name_node->rcu, netdev_name_node_alt_free);
}
@@ -5862,13 +5862,6 @@ static __latent_entropy void net_tx_action(void)
xfrm_dev_backlog(sd);
}
-#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
-/* This hook is defined here for ATM LANE */
-int (*br_fdb_test_addr_hook)(struct net_device *dev,
- unsigned char *addr) __read_mostly;
-EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
-#endif
-
/**
* netdev_is_rx_handler_busy - check if receive handler is registered
* @dev: device to check
@@ -6869,9 +6862,9 @@ static void skb_defer_free_flush(void)
#if defined(CONFIG_NET_RX_BUSY_POLL)
-static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
+static void __busy_poll_stop(struct napi_struct *napi, unsigned long timeout)
{
- if (!skip_schedule) {
+ if (!timeout) {
gro_normal_list(&napi->gro);
__napi_schedule(napi);
return;
@@ -6881,6 +6874,8 @@ static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
gro_flush_normal(&napi->gro, HZ >= 1000);
clear_bit(NAPI_STATE_SCHED, &napi->state);
+ hrtimer_start(&napi->timer, ns_to_ktime(timeout),
+ HRTIMER_MODE_REL_PINNED);
}
enum {
@@ -6892,8 +6887,7 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
unsigned flags, u16 budget)
{
struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- bool skip_schedule = false;
- unsigned long timeout;
+ unsigned long timeout = 0;
int rc;
/* Busy polling means there is a high chance device driver hard irq
@@ -6913,10 +6907,12 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
if (flags & NAPI_F_PREFER_BUSY_POLL) {
napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi);
- timeout = napi_get_gro_flush_timeout(napi);
- if (napi->defer_hard_irqs_count && timeout) {
- hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
- skip_schedule = true;
+ if (napi->defer_hard_irqs_count) {
+ /* A short enough gro flush timeout and long enough
+ * poll can result in timer firing too early.
+ * Timer will be armed later if necessary.
+ */
+ timeout = napi_get_gro_flush_timeout(napi);
}
}
@@ -6931,7 +6927,7 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
trace_napi_poll(napi, rc, budget);
netpoll_poll_unlock(have_poll_lock);
if (rc == budget)
- __busy_poll_stop(napi, skip_schedule);
+ __busy_poll_stop(napi, timeout);
bpf_net_ctx_clear(bpf_net_ctx);
local_bh_enable();
}
@@ -9593,14 +9589,14 @@ static void dev_change_rx_flags(struct net_device *dev, int flags)
ops->ndo_change_rx_flags(dev, flags);
}
-static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
+int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
{
unsigned int old_flags = dev->flags;
unsigned int promiscuity, flags;
kuid_t uid;
kgid_t gid;
- ASSERT_RTNL();
+ netdev_ops_assert_locked(dev);
promiscuity = dev->promiscuity + inc;
if (promiscuity == 0) {
@@ -9636,16 +9632,8 @@ static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
dev_change_rx_flags(dev, IFF_PROMISC);
}
- if (notify) {
- /* The ops lock is only required to ensure consistent locking
- * for `NETDEV_CHANGE` notifiers. This function is sometimes
- * called without the lock, even for devices that are ops
- * locked, such as in `dev_uc_sync_multiple` when using
- * bonding or teaming.
- */
- netdev_ops_assert_locked(dev);
+ if (notify)
__dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
- }
return 0;
}
@@ -9667,7 +9655,7 @@ int netif_set_allmulti(struct net_device *dev, int inc, bool notify)
unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
unsigned int allmulti, flags;
- ASSERT_RTNL();
+ netdev_ops_assert_locked(dev);
allmulti = dev->allmulti + inc;
if (allmulti == 0) {
@@ -9697,46 +9685,6 @@ int netif_set_allmulti(struct net_device *dev, int inc, bool notify)
return 0;
}
-/*
- * Upload unicast and multicast address lists to device and
- * configure RX filtering. When the device doesn't support unicast
- * filtering it is put in promiscuous mode while unicast addresses
- * are present.
- */
-void __dev_set_rx_mode(struct net_device *dev)
-{
- const struct net_device_ops *ops = dev->netdev_ops;
-
- /* dev_open will call this function so the list will stay sane. */
- if (!(dev->flags&IFF_UP))
- return;
-
- if (!netif_device_present(dev))
- return;
-
- if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
- /* Unicast addresses changes may only happen under the rtnl,
- * therefore calling __dev_set_promiscuity here is safe.
- */
- if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
- __dev_set_promiscuity(dev, 1, false);
- dev->uc_promisc = true;
- } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
- __dev_set_promiscuity(dev, -1, false);
- dev->uc_promisc = false;
- }
- }
-
- if (ops->ndo_set_rx_mode)
- ops->ndo_set_rx_mode(dev);
-}
-
-void dev_set_rx_mode(struct net_device *dev)
-{
- netif_addr_lock_bh(dev);
- __dev_set_rx_mode(dev);
- netif_addr_unlock_bh(dev);
-}
/**
* netif_get_flags() - get flags reported to userspace
@@ -9775,7 +9723,7 @@ int __dev_change_flags(struct net_device *dev, unsigned int flags,
unsigned int old_flags = dev->flags;
int ret;
- ASSERT_RTNL();
+ netdev_ops_assert_locked(dev);
/*
* Set the flags on our device.
@@ -11408,6 +11356,11 @@ int register_netdevice(struct net_device *dev)
goto err_uninit;
}
+ if (netdev_need_ops_lock(dev) &&
+ dev->netdev_ops->ndo_set_rx_mode &&
+ !dev->netdev_ops->ndo_set_rx_mode_async)
+ netdev_WARN(dev, "ops-locked drivers should use ndo_set_rx_mode_async\n");
+
ret = netdev_do_alloc_pcpu_stats(dev);
if (ret)
goto err_uninit;
@@ -12127,6 +12080,8 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
#endif
mutex_init(&dev->lock);
+ INIT_LIST_HEAD(&dev->rx_mode_node);
+ __hw_addr_init(&dev->rx_mode_addr_cache);
dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
setup(dev);
@@ -12231,6 +12186,8 @@ void free_netdev(struct net_device *dev)
kfree(rcu_dereference_protected(dev->ingress_queue, 1));
+ __hw_addr_flush(&dev->rx_mode_addr_cache);
+
/* Flush device addresses */
dev_addr_flush(dev);
diff --git a/net/core/dev.h b/net/core/dev.h
index 628bdaebf0ca..0cf24b8f5008 100644
--- a/net/core/dev.h
+++ b/net/core/dev.h
@@ -78,6 +78,7 @@ void linkwatch_run_queue(void);
void dev_addr_flush(struct net_device *dev);
int dev_addr_init(struct net_device *dev);
void dev_addr_check(struct net_device *dev);
+void __hw_addr_flush(struct netdev_hw_addr_list *list);
#if IS_ENABLED(CONFIG_NET_SHAPER)
void net_shaper_flush_netdev(struct net_device *dev);
@@ -164,6 +165,9 @@ int netif_change_carrier(struct net_device *dev, bool new_carrier);
int dev_change_carrier(struct net_device *dev, bool new_carrier);
void __dev_set_rx_mode(struct net_device *dev);
+int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify);
+bool netif_rx_mode_clean(struct net_device *dev);
+void netif_rx_mode_sync(struct net_device *dev);
void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
unsigned int gchanges, u32 portid,
diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c
index 76c91f224886..d73fcb0c6785 100644
--- a/net/core/dev_addr_lists.c
+++ b/net/core/dev_addr_lists.c
@@ -11,9 +11,18 @@
#include <linux/rtnetlink.h>
#include <linux/export.h>
#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <kunit/visibility.h>
#include "dev.h"
+static void netdev_rx_mode_work(struct work_struct *work);
+
+static LIST_HEAD(rx_mode_list);
+static DEFINE_SPINLOCK(rx_mode_lock);
+static DECLARE_WORK(rx_mode_work, netdev_rx_mode_work);
+
/*
* General list handling functions
*/
@@ -481,7 +490,7 @@ void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
}
EXPORT_SYMBOL(__hw_addr_unsync_dev);
-static void __hw_addr_flush(struct netdev_hw_addr_list *list)
+void __hw_addr_flush(struct netdev_hw_addr_list *list)
{
struct netdev_hw_addr *ha, *tmp;
@@ -492,6 +501,7 @@ static void __hw_addr_flush(struct netdev_hw_addr_list *list)
}
list->count = 0;
}
+EXPORT_SYMBOL_IF_KUNIT(__hw_addr_flush);
void __hw_addr_init(struct netdev_hw_addr_list *list)
{
@@ -501,6 +511,133 @@ void __hw_addr_init(struct netdev_hw_addr_list *list)
}
EXPORT_SYMBOL(__hw_addr_init);
+static void __hw_addr_splice(struct netdev_hw_addr_list *dst,
+ struct netdev_hw_addr_list *src)
+{
+ src->tree = RB_ROOT;
+ list_splice_init(&src->list, &dst->list);
+ dst->count += src->count;
+ src->count = 0;
+}
+
+/**
+ * __hw_addr_list_snapshot - create a snapshot copy of an address list
+ * @snap: destination snapshot list (needs to be __hw_addr_init-initialized)
+ * @list: source address list to snapshot
+ * @addr_len: length of addresses
+ * @cache: entry cache to reuse entries from; falls back to GFP_ATOMIC
+ *
+ * Creates a copy of @list reusing entries from @cache when available.
+ * Must be called under a spinlock.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+int __hw_addr_list_snapshot(struct netdev_hw_addr_list *snap,
+ const struct netdev_hw_addr_list *list,
+ int addr_len, struct netdev_hw_addr_list *cache)
+{
+ struct netdev_hw_addr *ha, *entry;
+
+ list_for_each_entry(ha, &list->list, list) {
+ if (cache->count) {
+ entry = list_first_entry(&cache->list,
+ struct netdev_hw_addr, list);
+ list_del(&entry->list);
+ cache->count--;
+ memcpy(entry->addr, ha->addr, addr_len);
+ entry->type = ha->type;
+ entry->global_use = false;
+ entry->synced = 0;
+ } else {
+ entry = __hw_addr_create(ha->addr, addr_len, ha->type,
+ false, false);
+ if (!entry) {
+ __hw_addr_flush(snap);
+ return -ENOMEM;
+ }
+ }
+ entry->sync_cnt = ha->sync_cnt;
+ entry->refcount = ha->refcount;
+
+ list_add_tail(&entry->list, &snap->list);
+ __hw_addr_insert(snap, entry, addr_len);
+ snap->count++;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_IF_KUNIT(__hw_addr_list_snapshot);
+
+/**
+ * __hw_addr_list_reconcile - sync snapshot changes back and free snapshots
+ * @real_list: the real address list to update
+ * @work: the working snapshot (modified by driver via __hw_addr_sync_dev)
+ * @ref: the reference snapshot (untouched copy of original state)
+ * @addr_len: length of addresses
+ * @cache: entry cache to return snapshot entries to for reuse
+ *
+ * Walks the reference snapshot and compares each entry against the work
+ * snapshot to compute sync_cnt deltas. Applies those deltas to @real_list.
+ * Returns snapshot entries to @cache for reuse; frees both snapshots.
+ * Caller must hold netif_addr_lock_bh.
+ */
+void __hw_addr_list_reconcile(struct netdev_hw_addr_list *real_list,
+ struct netdev_hw_addr_list *work,
+ struct netdev_hw_addr_list *ref, int addr_len,
+ struct netdev_hw_addr_list *cache)
+{
+ struct netdev_hw_addr *ref_ha, *tmp, *work_ha, *real_ha;
+ int delta;
+
+ list_for_each_entry_safe(ref_ha, tmp, &ref->list, list) {
+ work_ha = __hw_addr_lookup(work, ref_ha->addr, addr_len,
+ ref_ha->type);
+ if (work_ha)
+ delta = work_ha->sync_cnt - ref_ha->sync_cnt;
+ else
+ delta = -1;
+
+ if (delta == 0)
+ continue;
+
+ real_ha = __hw_addr_lookup(real_list, ref_ha->addr, addr_len,
+ ref_ha->type);
+ if (!real_ha) {
+ /* The real entry was concurrently removed. If the
+ * driver synced this addr to hardware (delta > 0),
+ * re-insert it as a stale entry so the next work
+ * run unsyncs it from hardware.
+ */
+ if (delta > 0) {
+ rb_erase(&ref_ha->node, &ref->tree);
+ list_del(&ref_ha->list);
+ ref->count--;
+ ref_ha->sync_cnt = delta;
+ ref_ha->refcount = delta;
+ list_add_tail_rcu(&ref_ha->list,
+ &real_list->list);
+ __hw_addr_insert(real_list, ref_ha,
+ addr_len);
+ real_list->count++;
+ }
+ continue;
+ }
+
+ real_ha->sync_cnt += delta;
+ real_ha->refcount += delta;
+ if (!real_ha->refcount) {
+ rb_erase(&real_ha->node, &real_list->tree);
+ list_del_rcu(&real_ha->list);
+ kfree_rcu(real_ha, rcu_head);
+ real_list->count--;
+ }
+ }
+
+ __hw_addr_splice(cache, work);
+ __hw_addr_splice(cache, ref);
+}
+EXPORT_SYMBOL_IF_KUNIT(__hw_addr_list_reconcile);
+
/*
* Device addresses handling functions
*/
@@ -1049,3 +1186,249 @@ void dev_mc_init(struct net_device *dev)
__hw_addr_init(&dev->mc);
}
EXPORT_SYMBOL(dev_mc_init);
+
+static int netif_addr_lists_snapshot(struct net_device *dev,
+ struct netdev_hw_addr_list *uc_snap,
+ struct netdev_hw_addr_list *mc_snap,
+ struct netdev_hw_addr_list *uc_ref,
+ struct netdev_hw_addr_list *mc_ref)
+{
+ int err;
+
+ err = __hw_addr_list_snapshot(uc_snap, &dev->uc, dev->addr_len,
+ &dev->rx_mode_addr_cache);
+ if (!err)
+ err = __hw_addr_list_snapshot(uc_ref, &dev->uc, dev->addr_len,
+ &dev->rx_mode_addr_cache);
+ if (!err)
+ err = __hw_addr_list_snapshot(mc_snap, &dev->mc,
+ dev->addr_len,
+ &dev->rx_mode_addr_cache);
+ if (!err)
+ err = __hw_addr_list_snapshot(mc_ref, &dev->mc, dev->addr_len,
+ &dev->rx_mode_addr_cache);
+
+ if (err) {
+ __hw_addr_flush(uc_snap);
+ __hw_addr_flush(uc_ref);
+ __hw_addr_flush(mc_snap);
+ }
+
+ return err;
+}
+
+static void netif_addr_lists_reconcile(struct net_device *dev,
+ struct netdev_hw_addr_list *uc_snap,
+ struct netdev_hw_addr_list *mc_snap,
+ struct netdev_hw_addr_list *uc_ref,
+ struct netdev_hw_addr_list *mc_ref)
+{
+ __hw_addr_list_reconcile(&dev->uc, uc_snap, uc_ref, dev->addr_len,
+ &dev->rx_mode_addr_cache);
+ __hw_addr_list_reconcile(&dev->mc, mc_snap, mc_ref, dev->addr_len,
+ &dev->rx_mode_addr_cache);
+}
+
+/**
+ * netif_uc_promisc_update() - evaluate whether uc_promisc should be toggled.
+ * @dev: device
+ *
+ * Must be called under netif_addr_lock_bh.
+ * Return: +1 to enter promisc, -1 to leave, 0 for no change.
+ */
+static int netif_uc_promisc_update(struct net_device *dev)
+{
+ if (dev->priv_flags & IFF_UNICAST_FLT)
+ return 0;
+
+ if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
+ dev->uc_promisc = true;
+ return 1;
+ }
+ if (netdev_uc_empty(dev) && dev->uc_promisc) {
+ dev->uc_promisc = false;
+ return -1;
+ }
+ return 0;
+}
+
+static void netif_rx_mode_run(struct net_device *dev)
+{
+ struct netdev_hw_addr_list uc_snap, mc_snap, uc_ref, mc_ref;
+ const struct net_device_ops *ops = dev->netdev_ops;
+ int promisc_inc;
+ int err;
+
+ might_sleep();
+ netdev_ops_assert_locked(dev);
+
+ __hw_addr_init(&uc_snap);
+ __hw_addr_init(&mc_snap);
+ __hw_addr_init(&uc_ref);
+ __hw_addr_init(&mc_ref);
+
+ if (!(dev->flags & IFF_UP) || !netif_device_present(dev))
+ return;
+
+ if (ops->ndo_set_rx_mode_async) {
+ netif_addr_lock_bh(dev);
+ err = netif_addr_lists_snapshot(dev, &uc_snap, &mc_snap,
+ &uc_ref, &mc_ref);
+ if (err) {
+ netdev_WARN(dev, "failed to sync uc/mc addresses\n");
+ netif_addr_unlock_bh(dev);
+ return;
+ }
+
+ promisc_inc = netif_uc_promisc_update(dev);
+ netif_addr_unlock_bh(dev);
+ } else {
+ netif_addr_lock_bh(dev);
+ promisc_inc = netif_uc_promisc_update(dev);
+ netif_addr_unlock_bh(dev);
+ }
+
+ if (promisc_inc)
+ __dev_set_promiscuity(dev, promisc_inc, false);
+
+ if (ops->ndo_set_rx_mode_async) {
+ ops->ndo_set_rx_mode_async(dev, &uc_snap, &mc_snap);
+
+ netif_addr_lock_bh(dev);
+ netif_addr_lists_reconcile(dev, &uc_snap, &mc_snap,
+ &uc_ref, &mc_ref);
+ netif_addr_unlock_bh(dev);
+ } else if (ops->ndo_set_rx_mode) {
+ netif_addr_lock_bh(dev);
+ ops->ndo_set_rx_mode(dev);
+ netif_addr_unlock_bh(dev);
+ }
+}
+
+static void netdev_rx_mode_work(struct work_struct *work)
+{
+ struct net_device *dev;
+
+ rtnl_lock();
+
+ while (true) {
+ spin_lock_bh(&rx_mode_lock);
+ if (list_empty(&rx_mode_list)) {
+ spin_unlock_bh(&rx_mode_lock);
+ break;
+ }
+ dev = list_first_entry(&rx_mode_list, struct net_device,
+ rx_mode_node);
+ list_del_init(&dev->rx_mode_node);
+ /* We must free netdev tracker under
+ * the spinlock protection.
+ */
+ netdev_tracker_free(dev, &dev->rx_mode_tracker);
+ spin_unlock_bh(&rx_mode_lock);
+
+ netdev_lock_ops(dev);
+ netif_rx_mode_run(dev);
+ netdev_unlock_ops(dev);
+ /* Use __dev_put() because netdev_tracker_free() was already
+ * called above. Must be after netdev_unlock_ops() to prevent
+ * netdev_run_todo() from freeing the device while still in use.
+ */
+ __dev_put(dev);
+ }
+
+ rtnl_unlock();
+}
+
+static void netif_rx_mode_queue(struct net_device *dev)
+{
+ spin_lock_bh(&rx_mode_lock);
+ if (list_empty(&dev->rx_mode_node)) {
+ list_add_tail(&dev->rx_mode_node, &rx_mode_list);
+ netdev_hold(dev, &dev->rx_mode_tracker, GFP_ATOMIC);
+ }
+ spin_unlock_bh(&rx_mode_lock);
+ schedule_work(&rx_mode_work);
+}
+
+/**
+ * __dev_set_rx_mode() - upload unicast and multicast address lists to device
+ * and configure RX filtering.
+ * @dev: device
+ *
+ * When the device doesn't support unicast filtering it is put in promiscuous
+ * mode while unicast addresses are present.
+ */
+void __dev_set_rx_mode(struct net_device *dev)
+{
+ const struct net_device_ops *ops = dev->netdev_ops;
+ int promisc_inc;
+
+ /* dev_open will call this function so the list will stay sane. */
+ if (!(dev->flags & IFF_UP))
+ return;
+
+ if (!netif_device_present(dev))
+ return;
+
+ if (ops->ndo_set_rx_mode_async || ops->ndo_change_rx_flags ||
+ netdev_need_ops_lock(dev)) {
+ netif_rx_mode_queue(dev);
+ return;
+ }
+
+ /* Legacy path for non-ops-locked HW devices. */
+
+ promisc_inc = netif_uc_promisc_update(dev);
+ if (promisc_inc)
+ __dev_set_promiscuity(dev, promisc_inc, false);
+
+ if (ops->ndo_set_rx_mode)
+ ops->ndo_set_rx_mode(dev);
+}
+
+void dev_set_rx_mode(struct net_device *dev)
+{
+ netif_addr_lock_bh(dev);
+ __dev_set_rx_mode(dev);
+ netif_addr_unlock_bh(dev);
+}
+
+bool netif_rx_mode_clean(struct net_device *dev)
+{
+ bool clean = false;
+
+ spin_lock_bh(&rx_mode_lock);
+ if (!list_empty(&dev->rx_mode_node)) {
+ list_del_init(&dev->rx_mode_node);
+ clean = true;
+ /* We must release netdev tracker under
+ * the spinlock protection.
+ */
+ netdev_tracker_free(dev, &dev->rx_mode_tracker);
+ }
+ spin_unlock_bh(&rx_mode_lock);
+
+ return clean;
+}
+
+/**
+ * netif_rx_mode_sync() - sync rx mode inline
+ * @dev: network device
+ *
+ * Drivers implementing ndo_set_rx_mode_async() have their rx mode callback
+ * executed from a workqueue. This allows the callback to sleep, but means
+ * the hardware update is deferred and may not be visible to userspace
+ * by the time the initiating syscall returns. netif_rx_mode_sync() steals
+ * workqueue update and executes it inline. This preserves the atomicity of
+ * operations to the userspace.
+ */
+void netif_rx_mode_sync(struct net_device *dev)
+{
+ if (netif_rx_mode_clean(dev)) {
+ netif_rx_mode_run(dev);
+ /* Use __dev_put() because netdev_tracker_free() was already
+ * called inside netif_rx_mode_clean().
+ */
+ __dev_put(dev);
+ }
+}
diff --git a/net/core/dev_addr_lists_test.c b/net/core/dev_addr_lists_test.c
index 8e1dba825e94..260e71a2399f 100644
--- a/net/core/dev_addr_lists_test.c
+++ b/net/core/dev_addr_lists_test.c
@@ -2,22 +2,31 @@
#include <kunit/test.h>
#include <linux/etherdevice.h>
+#include <linux/math64.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
static const struct net_device_ops dummy_netdev_ops = {
};
+#define ADDR_A 1
+#define ADDR_B 2
+#define ADDR_C 3
+
struct dev_addr_test_priv {
u32 addr_seen;
+ u32 addr_synced;
+ u32 addr_unsynced;
};
static int dev_addr_test_sync(struct net_device *netdev, const unsigned char *a)
{
struct dev_addr_test_priv *datp = netdev_priv(netdev);
- if (a[0] < 31 && !memchr_inv(a, a[0], ETH_ALEN))
+ if (a[0] < 31 && !memchr_inv(a, a[0], ETH_ALEN)) {
datp->addr_seen |= 1 << a[0];
+ datp->addr_synced |= 1 << a[0];
+ }
return 0;
}
@@ -26,11 +35,22 @@ static int dev_addr_test_unsync(struct net_device *netdev,
{
struct dev_addr_test_priv *datp = netdev_priv(netdev);
- if (a[0] < 31 && !memchr_inv(a, a[0], ETH_ALEN))
+ if (a[0] < 31 && !memchr_inv(a, a[0], ETH_ALEN)) {
datp->addr_seen &= ~(1 << a[0]);
+ datp->addr_unsynced |= 1 << a[0];
+ }
return 0;
}
+static void dev_addr_test_reset(struct net_device *netdev)
+{
+ struct dev_addr_test_priv *datp = netdev_priv(netdev);
+
+ datp->addr_seen = 0;
+ datp->addr_synced = 0;
+ datp->addr_unsynced = 0;
+}
+
static int dev_addr_test_init(struct kunit *test)
{
struct dev_addr_test_priv *datp;
@@ -225,6 +245,363 @@ static void dev_addr_test_add_excl(struct kunit *test)
rtnl_unlock();
}
+/* Snapshot test: basic sync with no concurrent modifications.
+ * Add one address, snapshot, driver syncs it, reconcile propagates
+ * sync_cnt delta back to real list.
+ */
+static void dev_addr_test_snapshot_sync(struct kunit *test)
+{
+ struct netdev_hw_addr_list snap, ref, cache;
+ struct net_device *netdev = test->priv;
+ struct dev_addr_test_priv *datp;
+ struct netdev_hw_addr *ha;
+ u8 addr[ETH_ALEN];
+
+ datp = netdev_priv(netdev);
+
+ rtnl_lock();
+
+ memset(addr, ADDR_A, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+
+ /* Snapshot: ADDR_A has sync_cnt=0, refcount=1 (new) */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_init(&snap);
+ __hw_addr_init(&ref);
+ __hw_addr_init(&cache);
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&snap, &netdev->uc, ETH_ALEN,
+ &cache));
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&ref, &netdev->uc, ETH_ALEN,
+ &cache));
+ netif_addr_unlock_bh(netdev);
+
+ /* Driver syncs ADDR_A to hardware */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&snap, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_A, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_unsynced);
+
+ /* Reconcile: delta=+1 applied to real entry */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_list_reconcile(&netdev->uc, &snap, &ref, ETH_ALEN,
+ &cache);
+ netif_addr_unlock_bh(netdev);
+
+ /* Real entry should now reflect the sync: sync_cnt=1, refcount=2 */
+ KUNIT_EXPECT_EQ(test, 1, netdev->uc.count);
+ ha = list_first_entry(&netdev->uc.list, struct netdev_hw_addr, list);
+ KUNIT_EXPECT_MEMEQ(test, ha->addr, addr, ETH_ALEN);
+ KUNIT_EXPECT_EQ(test, 1, ha->sync_cnt);
+ KUNIT_EXPECT_EQ(test, 2, ha->refcount);
+
+ /* Second work run: already synced, nothing to do */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&netdev->uc, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_unsynced);
+ KUNIT_EXPECT_EQ(test, 1, netdev->uc.count);
+
+ __hw_addr_flush(&cache);
+ rtnl_unlock();
+}
+
+/* Snapshot test: ADDR_A synced to hardware, then concurrently removed
+ * from the real list before reconcile runs. Reconcile re-inserts ADDR_A as
+ * a stale entry so the next work run unsyncs it from hardware.
+ */
+static void dev_addr_test_snapshot_remove_during_sync(struct kunit *test)
+{
+ struct netdev_hw_addr_list snap, ref, cache;
+ struct net_device *netdev = test->priv;
+ struct dev_addr_test_priv *datp;
+ struct netdev_hw_addr *ha;
+ u8 addr[ETH_ALEN];
+
+ datp = netdev_priv(netdev);
+
+ rtnl_lock();
+
+ memset(addr, ADDR_A, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+
+ /* Snapshot: ADDR_A is new (sync_cnt=0, refcount=1) */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_init(&snap);
+ __hw_addr_init(&ref);
+ __hw_addr_init(&cache);
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&snap, &netdev->uc, ETH_ALEN,
+ &cache));
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&ref, &netdev->uc, ETH_ALEN,
+ &cache));
+ netif_addr_unlock_bh(netdev);
+
+ /* Driver syncs ADDR_A to hardware */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&snap, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_A, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_unsynced);
+
+ /* Concurrent removal: user deletes ADDR_A while driver was working */
+ memset(addr, ADDR_A, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_del(netdev, addr));
+ KUNIT_EXPECT_EQ(test, 0, netdev->uc.count);
+
+ /* Reconcile: ADDR_A gone from real list but driver synced it,
+ * so it gets re-inserted as stale (sync_cnt=1, refcount=1).
+ */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_list_reconcile(&netdev->uc, &snap, &ref, ETH_ALEN,
+ &cache);
+ netif_addr_unlock_bh(netdev);
+
+ KUNIT_EXPECT_EQ(test, 1, netdev->uc.count);
+ ha = list_first_entry(&netdev->uc.list, struct netdev_hw_addr, list);
+ KUNIT_EXPECT_MEMEQ(test, ha->addr, addr, ETH_ALEN);
+ KUNIT_EXPECT_EQ(test, 1, ha->sync_cnt);
+ KUNIT_EXPECT_EQ(test, 1, ha->refcount);
+
+ /* Second work run: stale entry gets unsynced from HW and removed */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&netdev->uc, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_A, datp->addr_unsynced);
+ KUNIT_EXPECT_EQ(test, 0, netdev->uc.count);
+
+ __hw_addr_flush(&cache);
+ rtnl_unlock();
+}
+
+/* Snapshot test: ADDR_A was stale (unsynced from hardware by driver),
+ * but concurrently re-added by the user. The re-add bumps refcount of
+ * the existing stale entry. Reconcile applies delta=-1, leaving ADDR_A
+ * as a fresh entry (sync_cnt=0, refcount=1) for the next work run.
+ */
+static void dev_addr_test_snapshot_readd_during_unsync(struct kunit *test)
+{
+ struct netdev_hw_addr_list snap, ref, cache;
+ struct net_device *netdev = test->priv;
+ struct dev_addr_test_priv *datp;
+ struct netdev_hw_addr *ha;
+ u8 addr[ETH_ALEN];
+
+ datp = netdev_priv(netdev);
+
+ rtnl_lock();
+
+ memset(addr, ADDR_A, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+
+ /* Sync ADDR_A to hardware: sync_cnt=1, refcount=2 */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&netdev->uc, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_A, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_unsynced);
+
+ /* User removes ADDR_A: refcount=1, sync_cnt=1 -> stale */
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_del(netdev, addr));
+
+ /* Snapshot: ADDR_A is stale (sync_cnt=1, refcount=1) */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_init(&snap);
+ __hw_addr_init(&ref);
+ __hw_addr_init(&cache);
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&snap, &netdev->uc, ETH_ALEN,
+ &cache));
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&ref, &netdev->uc, ETH_ALEN,
+ &cache));
+ netif_addr_unlock_bh(netdev);
+
+ /* Driver unsyncs stale ADDR_A from hardware */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&snap, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_A, datp->addr_unsynced);
+
+ /* Concurrent: user re-adds ADDR_A. dev_uc_add finds the existing
+ * stale entry and bumps refcount from 1 -> 2. sync_cnt stays 1.
+ */
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+ KUNIT_EXPECT_EQ(test, 1, netdev->uc.count);
+
+ /* Reconcile: ref sync_cnt=1 matches real sync_cnt=1, delta=-1
+ * applied. Result: sync_cnt=0, refcount=1 (fresh).
+ */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_list_reconcile(&netdev->uc, &snap, &ref, ETH_ALEN,
+ &cache);
+ netif_addr_unlock_bh(netdev);
+
+ /* Entry survives as fresh: needs re-sync to HW */
+ KUNIT_EXPECT_EQ(test, 1, netdev->uc.count);
+ ha = list_first_entry(&netdev->uc.list, struct netdev_hw_addr, list);
+ KUNIT_EXPECT_MEMEQ(test, ha->addr, addr, ETH_ALEN);
+ KUNIT_EXPECT_EQ(test, 0, ha->sync_cnt);
+ KUNIT_EXPECT_EQ(test, 1, ha->refcount);
+
+ /* Second work run: fresh entry gets synced to HW */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&netdev->uc, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_A, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_unsynced);
+
+ __hw_addr_flush(&cache);
+ rtnl_unlock();
+}
+
+/* Snapshot test: ADDR_A is new (synced by driver), and independent ADDR_B
+ * is concurrently removed from the real list. A's sync delta propagates
+ * normally; B's absence doesn't interfere.
+ */
+static void dev_addr_test_snapshot_add_and_remove(struct kunit *test)
+{
+ struct netdev_hw_addr_list snap, ref, cache;
+ struct net_device *netdev = test->priv;
+ struct dev_addr_test_priv *datp;
+ struct netdev_hw_addr *ha;
+ u8 addr[ETH_ALEN];
+
+ datp = netdev_priv(netdev);
+
+ rtnl_lock();
+
+ /* Add ADDR_A and ADDR_B (will be synced then removed) */
+ memset(addr, ADDR_A, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+ memset(addr, ADDR_B, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+
+ /* Sync both to hardware: sync_cnt=1, refcount=2 */
+ __hw_addr_sync_dev(&netdev->uc, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+
+ /* Add ADDR_C (new, will be synced by snapshot) */
+ memset(addr, ADDR_C, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+
+ /* Snapshot: A,B synced (sync_cnt=1,refcount=2); C new (0,1) */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_init(&snap);
+ __hw_addr_init(&ref);
+ __hw_addr_init(&cache);
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&snap, &netdev->uc, ETH_ALEN,
+ &cache));
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&ref, &netdev->uc, ETH_ALEN,
+ &cache));
+ netif_addr_unlock_bh(netdev);
+
+ /* Driver syncs snapshot: ADDR_C is new -> synced; A,B already synced */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&snap, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_C, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_unsynced);
+
+ /* Concurrent: user removes addr B while driver was working */
+ memset(addr, ADDR_B, sizeof(addr));
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_del(netdev, addr));
+
+ /* Reconcile: ADDR_C's delta=+1 applied to real list.
+ * ADDR_B's delta=0 (unchanged in snapshot),
+ * so nothing to apply to ADDR_B.
+ */
+ netif_addr_lock_bh(netdev);
+ __hw_addr_list_reconcile(&netdev->uc, &snap, &ref, ETH_ALEN,
+ &cache);
+ netif_addr_unlock_bh(netdev);
+
+ /* ADDR_A: unchanged (sync_cnt=1, refcount=2)
+ * ADDR_B: refcount went from 2->1 via dev_uc_del (still present, stale)
+ * ADDR_C: sync propagated (sync_cnt=1, refcount=2)
+ */
+ KUNIT_EXPECT_EQ(test, 3, netdev->uc.count);
+ netdev_hw_addr_list_for_each(ha, &netdev->uc) {
+ u8 id = ha->addr[0];
+
+ if (!memchr_inv(ha->addr, id, ETH_ALEN)) {
+ if (id == ADDR_A) {
+ KUNIT_EXPECT_EQ(test, 1, ha->sync_cnt);
+ KUNIT_EXPECT_EQ(test, 2, ha->refcount);
+ } else if (id == ADDR_B) {
+ /* B: still present but now stale */
+ KUNIT_EXPECT_EQ(test, 1, ha->sync_cnt);
+ KUNIT_EXPECT_EQ(test, 1, ha->refcount);
+ } else if (id == ADDR_C) {
+ KUNIT_EXPECT_EQ(test, 1, ha->sync_cnt);
+ KUNIT_EXPECT_EQ(test, 2, ha->refcount);
+ }
+ }
+ }
+
+ /* Second work run: ADDR_B is stale, gets unsynced and removed */
+ dev_addr_test_reset(netdev);
+ __hw_addr_sync_dev(&netdev->uc, netdev, dev_addr_test_sync,
+ dev_addr_test_unsync);
+ KUNIT_EXPECT_EQ(test, 0, datp->addr_synced);
+ KUNIT_EXPECT_EQ(test, 1 << ADDR_B, datp->addr_unsynced);
+ KUNIT_EXPECT_EQ(test, 2, netdev->uc.count);
+
+ __hw_addr_flush(&cache);
+ rtnl_unlock();
+}
+
+static void dev_addr_test_snapshot_benchmark(struct kunit *test)
+{
+ struct net_device *netdev = test->priv;
+ struct netdev_hw_addr_list snap, cache;
+ u8 addr[ETH_ALEN];
+ s64 duration = 0;
+ ktime_t start;
+ int i, iter;
+
+ rtnl_lock();
+
+ for (i = 0; i < 1024; i++) {
+ memset(addr, 0, sizeof(addr));
+ addr[0] = (i >> 8) & 0xff;
+ addr[1] = i & 0xff;
+ KUNIT_EXPECT_EQ(test, 0, dev_uc_add(netdev, addr));
+ }
+
+ __hw_addr_init(&cache);
+
+ for (iter = 0; iter < 1000; iter++) {
+ netif_addr_lock_bh(netdev);
+ __hw_addr_init(&snap);
+
+ start = ktime_get();
+ KUNIT_EXPECT_EQ(test, 0,
+ __hw_addr_list_snapshot(&snap, &netdev->uc,
+ ETH_ALEN, &cache));
+ duration += ktime_to_ns(ktime_sub(ktime_get(), start));
+
+ netif_addr_unlock_bh(netdev);
+ __hw_addr_flush(&snap);
+ }
+
+ __hw_addr_flush(&cache);
+
+ kunit_info(test,
+ "1024 addrs x 1000 snapshots: %lld ns total, %lld ns/iter",
+ duration, div_s64(duration, 1000));
+
+ rtnl_unlock();
+}
+
static struct kunit_case dev_addr_test_cases[] = {
KUNIT_CASE(dev_addr_test_basic),
KUNIT_CASE(dev_addr_test_sync_one),
@@ -232,6 +609,11 @@ static struct kunit_case dev_addr_test_cases[] = {
KUNIT_CASE(dev_addr_test_del_main),
KUNIT_CASE(dev_addr_test_add_set),
KUNIT_CASE(dev_addr_test_add_excl),
+ KUNIT_CASE(dev_addr_test_snapshot_sync),
+ KUNIT_CASE(dev_addr_test_snapshot_remove_during_sync),
+ KUNIT_CASE(dev_addr_test_snapshot_readd_during_unsync),
+ KUNIT_CASE(dev_addr_test_snapshot_add_and_remove),
+ KUNIT_CASE_SLOW(dev_addr_test_snapshot_benchmark),
{}
};
@@ -243,5 +625,6 @@ static struct kunit_suite dev_addr_test_suite = {
};
kunit_test_suite(dev_addr_test_suite);
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
MODULE_DESCRIPTION("KUnit tests for struct netdev_hw_addr_list");
MODULE_LICENSE("GPL");
diff --git a/net/core/dev_api.c b/net/core/dev_api.c
index f28852078aa6..437947dd08ed 100644
--- a/net/core/dev_api.c
+++ b/net/core/dev_api.c
@@ -66,6 +66,7 @@ int dev_change_flags(struct net_device *dev, unsigned int flags,
netdev_lock_ops(dev);
ret = netif_change_flags(dev, flags, extack);
+ netif_rx_mode_sync(dev);
netdev_unlock_ops(dev);
return ret;
@@ -285,6 +286,7 @@ int dev_set_promiscuity(struct net_device *dev, int inc)
netdev_lock_ops(dev);
ret = netif_set_promiscuity(dev, inc);
+ netif_rx_mode_sync(dev);
netdev_unlock_ops(dev);
return ret;
@@ -311,6 +313,7 @@ int dev_set_allmulti(struct net_device *dev, int inc)
netdev_lock_ops(dev);
ret = netif_set_allmulti(dev, inc, true);
+ netif_rx_mode_sync(dev);
netdev_unlock_ops(dev);
return ret;
diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c
index 7a8966544c9d..f3979b276090 100644
--- a/net/core/dev_ioctl.c
+++ b/net/core/dev_ioctl.c
@@ -586,24 +586,26 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, void __user *data,
return err;
case SIOCADDMULTI:
- if (!ops->ndo_set_rx_mode ||
+ if ((!ops->ndo_set_rx_mode && !ops->ndo_set_rx_mode_async) ||
ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
return -EINVAL;
if (!netif_device_present(dev))
return -ENODEV;
netdev_lock_ops(dev);
err = dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
+ netif_rx_mode_sync(dev);
netdev_unlock_ops(dev);
return err;
case SIOCDELMULTI:
- if (!ops->ndo_set_rx_mode ||
+ if ((!ops->ndo_set_rx_mode && !ops->ndo_set_rx_mode_async) ||
ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
return -EINVAL;
if (!netif_device_present(dev))
return -ENODEV;
netdev_lock_ops(dev);
err = dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
+ netif_rx_mode_sync(dev);
netdev_unlock_ops(dev);
return err;
diff --git a/net/core/devmem.c b/net/core/devmem.c
index cde4c89bc146..468344739db2 100644
--- a/net/core/devmem.c
+++ b/net/core/devmem.c
@@ -297,8 +297,7 @@ net_devmem_bind_dmabuf(struct net_device *dev,
for (i = 0; i < owner->area.num_niovs; i++) {
niov = &owner->area.niovs[i];
- niov->type = NET_IOV_DMABUF;
- niov->owner = &owner->area;
+ net_iov_init(niov, &owner->area, NET_IOV_DMABUF);
page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
net_devmem_get_dma_addr(niov));
if (direction == DMA_TO_DEVICE)
diff --git a/net/core/failover.c b/net/core/failover.c
index 11bb183c7a1b..e43c59cd6868 100644
--- a/net/core/failover.c
+++ b/net/core/failover.c
@@ -12,6 +12,7 @@
#include <uapi/linux/if_arp.h>
#include <linux/rtnetlink.h>
#include <linux/if_vlan.h>
+#include <net/netdev_lock.h>
#include <net/failover.h>
static LIST_HEAD(failover_list);
@@ -221,8 +222,11 @@ failover_existing_slave_register(struct net_device *failover_dev)
for_each_netdev(net, dev) {
if (netif_is_failover(dev))
continue;
- if (ether_addr_equal(failover_dev->perm_addr, dev->perm_addr))
+ if (ether_addr_equal(failover_dev->perm_addr, dev->perm_addr)) {
+ netdev_lock_ops(dev);
failover_slave_register(dev);
+ netdev_unlock_ops(dev);
+ }
}
rtnl_unlock();
}
diff --git a/net/core/filter.c b/net/core/filter.c
index fcfcb72663ca..9590877b0714 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -503,7 +503,7 @@ static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
((unaligned_ok && offset >= 0) ||
(!unaligned_ok && offset >= 0 &&
offset + ip_align >= 0 &&
- offset + ip_align % size == 0))) {
+ (offset + ip_align) % size == 0))) {
bool ldx_off_ok = offset <= S16_MAX;
*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
@@ -1654,15 +1654,24 @@ err_prog_put:
return err;
}
+static void sk_reuseport_prog_free_rcu(struct rcu_head *rcu)
+{
+ struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
+ struct bpf_prog *prog = aux->prog;
+
+ bpf_release_orig_filter(prog);
+ bpf_prog_free(prog);
+}
+
void sk_reuseport_prog_free(struct bpf_prog *prog)
{
if (!prog)
return;
- if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
- bpf_prog_put(prog);
+ if (bpf_prog_was_classic(prog))
+ call_rcu(&prog->aux->rcu, sk_reuseport_prog_free_rcu);
else
- bpf_prog_destroy(prog);
+ bpf_prog_put(prog);
}
static inline int __bpf_try_make_writable(struct sk_buff *skb,
@@ -5396,7 +5405,7 @@ static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname,
if (val <= 0)
return -EINVAL;
tp->snd_cwnd_clamp = val;
- tp->snd_ssthresh = val;
+ WRITE_ONCE(tp->snd_ssthresh, val);
break;
case TCP_BPF_DELACK_MAX:
timeout = usecs_to_jiffies(val);
@@ -5481,7 +5490,7 @@ static int sol_tcp_sockopt(struct sock *sk, int optname,
char *optval, int *optlen,
bool getopt)
{
- if (sk->sk_protocol != IPPROTO_TCP)
+ if (!sk_is_tcp(sk))
return -EINVAL;
switch (optname) {
@@ -5688,6 +5697,30 @@ const struct bpf_func_proto bpf_sk_getsockopt_proto = {
.arg5_type = ARG_CONST_SIZE,
};
+BPF_CALL_5(bpf_sk_setsockopt_nodelay, struct sock *, sk, int, level,
+ int, optname, char *, optval, int, optlen)
+{
+ /*
+ * TCP_NODELAY triggers tcp_push_pending_frames() and re-enters
+ * CA_EVENT_TX_START in bpf_tcp_cc.
+ */
+ if (level == SOL_TCP && optname == TCP_NODELAY)
+ return -EOPNOTSUPP;
+
+ return _bpf_setsockopt(sk, level, optname, optval, optlen);
+}
+
+const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto = {
+ .func = bpf_sk_setsockopt_nodelay,
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
+ .arg2_type = ARG_ANYTHING,
+ .arg3_type = ARG_ANYTHING,
+ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE,
+};
+
BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level,
int, optname, char *, optval, int, optlen)
{
@@ -5833,6 +5866,12 @@ BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
if (!is_locked_tcp_sock_ops(bpf_sock))
return -EOPNOTSUPP;
+ /* TCP_NODELAY triggers tcp_push_pending_frames() and re-enters these callbacks. */
+ if ((bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB ||
+ bpf_sock->op == BPF_SOCK_OPS_WRITE_HDR_OPT_CB) &&
+ level == SOL_TCP && optname == TCP_NODELAY)
+ return -EOPNOTSUPP;
+
return _bpf_setsockopt(bpf_sock->sk, level, optname, optval, optlen);
}
@@ -6443,6 +6482,8 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb,
* against MTU of FIB lookup resulting net_device
*/
dev = dev_get_by_index_rcu(net, params->ifindex);
+ if (unlikely(!dev))
+ return -ENODEV;
if (!is_skb_forwardable(dev, skb))
rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;
@@ -7443,7 +7484,7 @@ u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
BPF_CALL_1(bpf_tcp_sock, struct sock *, sk)
{
- if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
+ if (sk_fullsock(sk) && sk_is_tcp(sk))
return (unsigned long)sk;
return (unsigned long)NULL;
@@ -11915,7 +11956,7 @@ BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk)
*/
BTF_TYPE_EMIT(struct tcp6_sock);
if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP &&
- sk->sk_family == AF_INET6)
+ sk->sk_type == SOCK_STREAM && sk->sk_family == AF_INET6)
return (unsigned long)sk;
return (unsigned long)NULL;
@@ -11931,7 +11972,7 @@ const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = {
BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk)
{
- if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
+ if (sk && sk_fullsock(sk) && sk_is_tcp(sk))
return (unsigned long)sk;
return (unsigned long)NULL;
diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
index 1b61bb25ba0e..2a98f5fa74eb 100644
--- a/net/core/flow_dissector.c
+++ b/net/core/flow_dissector.c
@@ -1374,16 +1374,13 @@ proto_again:
break;
}
- /* least significant bit of the most significant octet
- * indicates if protocol field was compressed
+ /* PFC (compressed 1-byte protocol) frames are not processed.
+ * A compressed protocol field has the least significant bit of
+ * the most significant octet set, which will fail the following
+ * ppp_proto_is_valid(), returning FLOW_DISSECT_RET_OUT_BAD.
*/
ppp_proto = ntohs(hdr->proto);
- if (ppp_proto & 0x0100) {
- ppp_proto = ppp_proto >> 8;
- nhoff += PPPOE_SES_HLEN - 1;
- } else {
- nhoff += PPPOE_SES_HLEN;
- }
+ nhoff += PPPOE_SES_HLEN;
if (ppp_proto == PPP_IP) {
proto = htons(ETH_P_IP);
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 9e12524b67fa..5d9216016507 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -3210,8 +3210,10 @@ int neigh_xmit(int index, struct net_device *dev,
rcu_read_lock();
tbl = rcu_dereference(neigh_tables[index]);
- if (!tbl)
- goto out_unlock;
+ if (!tbl) {
+ rcu_read_unlock();
+ goto out_kfree_skb;
+ }
if (index == NEIGH_ARP_TABLE) {
u32 key = *((u32 *)addr);
@@ -3227,7 +3229,6 @@ int neigh_xmit(int index, struct net_device *dev,
goto out_kfree_skb;
}
err = READ_ONCE(neigh->output)(neigh, skb);
-out_unlock:
rcu_read_unlock();
}
else if (index == NEIGH_LINK_TABLE) {
@@ -3237,11 +3238,10 @@ out_unlock:
goto out_kfree_skb;
err = dev_queue_xmit(skb);
}
-out:
return err;
out_kfree_skb:
kfree_skb(skb);
- goto out;
+ return err;
}
EXPORT_SYMBOL(neigh_xmit);
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index cd74beffd209..84faace50ac2 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -608,14 +608,16 @@ EXPORT_SYMBOL_GPL(__netpoll_setup);
/*
* Returns a pointer to a string representation of the identifier used
* to select the egress interface for the given netpoll instance. buf
- * must be a buffer of length at least MAC_ADDR_STR_LEN + 1.
+ * is used to format np->dev_mac when np->dev_name is empty; bufsz must
+ * be at least MAC_ADDR_STR_LEN + 1 to fit the formatted MAC address
+ * and its NUL terminator.
*/
-static char *egress_dev(struct netpoll *np, char *buf)
+static char *egress_dev(struct netpoll *np, char *buf, size_t bufsz)
{
if (np->dev_name[0])
return np->dev_name;
- snprintf(buf, MAC_ADDR_STR_LEN, "%pM", np->dev_mac);
+ snprintf(buf, bufsz, "%pM", np->dev_mac);
return buf;
}
@@ -645,7 +647,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev)
if (!IS_ENABLED(CONFIG_IPV6)) {
np_err(np, "IPv6 is not supported %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return -EINVAL;
}
@@ -667,7 +669,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev)
}
if (err) {
np_err(np, "no IPv6 address for %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return err;
}
@@ -687,14 +689,14 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev)
in_dev = __in_dev_get_rtnl(ndev);
if (!in_dev) {
np_err(np, "no IP address for %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return -EDESTADDRREQ;
}
ifa = rtnl_dereference(in_dev->ifa_list);
if (!ifa) {
np_err(np, "no IP address for %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return -EDESTADDRREQ;
}
@@ -704,6 +706,23 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev)
return 0;
}
+/*
+ * Test whether the caller left np->local_ip unset, so that
+ * netpoll_setup() should auto-populate it from the egress device.
+ *
+ * np->local_ip is a union of __be32 (IPv4) and struct in6_addr (IPv6),
+ * so an IPv6 address whose first 4 bytes are zero (e.g. ::1, ::2,
+ * IPv4-mapped ::ffff:a.b.c.d) must not be tested via the IPv4 arm —
+ * doing so would misclassify a caller-supplied address as unset and
+ * silently overwrite it with whatever address the device exposes.
+ */
+static bool netpoll_local_ip_unset(const struct netpoll *np)
+{
+ if (np->ipv6)
+ return ipv6_addr_any(&np->local_ip.in6);
+ return !np->local_ip.ip;
+}
+
int netpoll_setup(struct netpoll *np)
{
struct net *net = current->nsproxy->net_ns;
@@ -719,7 +738,8 @@ int netpoll_setup(struct netpoll *np)
ndev = dev_getbyhwaddr(net, ARPHRD_ETHER, np->dev_mac);
if (!ndev) {
- np_err(np, "%s doesn't exist, aborting\n", egress_dev(np, buf));
+ np_err(np, "%s doesn't exist, aborting\n",
+ egress_dev(np, buf, sizeof(buf)));
err = -ENODEV;
goto unlock;
}
@@ -727,14 +747,14 @@ int netpoll_setup(struct netpoll *np)
if (netdev_master_upper_dev_get(ndev)) {
np_err(np, "%s is a slave device, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
err = -EBUSY;
goto put;
}
if (!netif_running(ndev)) {
np_info(np, "device %s not up yet, forcing it\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
err = dev_open(ndev, NULL);
if (err) {
@@ -747,7 +767,7 @@ int netpoll_setup(struct netpoll *np)
rtnl_lock();
}
- if (!np->local_ip.ip) {
+ if (netpoll_local_ip_unset(np)) {
if (!np->ipv6) {
err = netpoll_take_ipv4(np, ndev);
if (err)
diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 877bbf7a1938..6e576dec80db 100644
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -327,6 +327,11 @@ static void page_pool_uninit(struct page_pool *pool)
if (!pool->system)
free_percpu(pool->recycle_stats);
#endif
+
+ if (pool->mp_ops) {
+ pool->mp_ops->destroy(pool);
+ static_branch_dec(&page_pool_mem_providers);
+ }
}
/**
@@ -1146,11 +1151,6 @@ static void __page_pool_destroy(struct page_pool *pool)
page_pool_unlist(pool);
page_pool_uninit(pool);
- if (pool->mp_ops) {
- pool->mp_ops->destroy(pool);
- static_branch_dec(&page_pool_mem_providers);
- }
-
kfree(pool);
}
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 69daba3ddaf0..df042da422ef 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -1572,6 +1572,7 @@ static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb,
port_guid.vf = ivi.vf;
memcpy(vf_mac.mac, ivi.mac, sizeof(ivi.mac));
+ memset(&vf_broadcast, 0, sizeof(vf_broadcast));
memcpy(vf_broadcast.broadcast, dev->broadcast, dev->addr_len);
vf_vlan.vlan = ivi.vlan;
vf_vlan.qos = ivi.qos;
@@ -3431,6 +3432,7 @@ errout:
dev->name);
}
+ netif_rx_mode_sync(dev);
netdev_unlock_ops(dev);
return err;
diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index b0e96337a269..99e3789492a0 100644
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -530,7 +530,7 @@ static bool sock_map_redirect_allowed(const struct sock *sk)
if (sk_is_tcp(sk))
return sk->sk_state != TCP_LISTEN;
else
- return sk->sk_state == TCP_ESTABLISHED;
+ return READ_ONCE(sk->sk_state) == TCP_ESTABLISHED;
}
static bool sock_map_sk_is_suitable(const struct sock *sk)
@@ -543,7 +543,7 @@ static bool sock_map_sk_state_allowed(const struct sock *sk)
if (sk_is_tcp(sk))
return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN);
if (sk_is_stream_unix(sk))
- return (1 << sk->sk_state) & TCPF_ESTABLISHED;
+ return (1 << READ_ONCE(sk->sk_state)) & TCPF_ESTABLISHED;
if (sk_is_vsock(sk) &&
(sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET))
return (1 << sk->sk_state) & TCPF_ESTABLISHED;
@@ -1630,18 +1630,23 @@ void sock_map_unhash(struct sock *sk)
void (*saved_unhash)(struct sock *sk);
struct sk_psock *psock;
+retry:
rcu_read_lock();
psock = sk_psock(sk);
if (unlikely(!psock)) {
rcu_read_unlock();
saved_unhash = READ_ONCE(sk->sk_prot)->unhash;
+ if (unlikely(saved_unhash == sock_map_unhash))
+ goto retry;
} else {
saved_unhash = psock->saved_unhash;
sock_map_remove_links(sk, psock);
rcu_read_unlock();
+
+ if (WARN_ON_ONCE(saved_unhash == sock_map_unhash))
+ return;
}
- if (WARN_ON_ONCE(saved_unhash == sock_map_unhash))
- return;
+
if (saved_unhash)
saved_unhash(sk);
}
@@ -1652,20 +1657,25 @@ void sock_map_destroy(struct sock *sk)
void (*saved_destroy)(struct sock *sk);
struct sk_psock *psock;
+retry:
rcu_read_lock();
psock = sk_psock_get(sk);
if (unlikely(!psock)) {
rcu_read_unlock();
saved_destroy = READ_ONCE(sk->sk_prot)->destroy;
+ if (unlikely(saved_destroy == sock_map_destroy))
+ goto retry;
} else {
saved_destroy = psock->saved_destroy;
sock_map_remove_links(sk, psock);
rcu_read_unlock();
sk_psock_stop(psock);
sk_psock_put(sk, psock);
+
+ if (WARN_ON_ONCE(saved_destroy == sock_map_destroy))
+ return;
}
- if (WARN_ON_ONCE(saved_destroy == sock_map_destroy))
- return;
+
if (saved_destroy)
saved_destroy(sk);
}
@@ -1676,32 +1686,33 @@ void sock_map_close(struct sock *sk, long timeout)
void (*saved_close)(struct sock *sk, long timeout);
struct sk_psock *psock;
+retry:
lock_sock(sk);
rcu_read_lock();
- psock = sk_psock(sk);
+ psock = sk_psock_get(sk);
if (likely(psock)) {
saved_close = psock->saved_close;
sock_map_remove_links(sk, psock);
- psock = sk_psock_get(sk);
- if (unlikely(!psock))
- goto no_psock;
rcu_read_unlock();
sk_psock_stop(psock);
release_sock(sk);
cancel_delayed_work_sync(&psock->work);
sk_psock_put(sk, psock);
+
+ /* Make sure we do not recurse. This is a bug.
+ * Leak the socket instead of crashing on a stack overflow.
+ */
+ if (WARN_ON_ONCE(saved_close == sock_map_close))
+ return;
} else {
saved_close = READ_ONCE(sk->sk_prot)->close;
-no_psock:
rcu_read_unlock();
release_sock(sk);
+
+ if (unlikely(saved_close == sock_map_close))
+ goto retry;
}
- /* Make sure we do not recurse. This is a bug.
- * Leak the socket instead of crashing on a stack overflow.
- */
- if (WARN_ON_ONCE(saved_close == sock_map_close))
- return;
saved_close(sk, timeout);
}
EXPORT_SYMBOL_GPL(sock_map_close);
generated by cgit v1.2.3 (git 2.39.0) at 2026-05-17 13:15:44 +0300