From c1da4ac752b8b0411791d26c678fcf23d2eed242 Mon Sep 17 00:00:00 2001 From: Or Gerlitz Date: Fri, 13 Jun 2008 18:12:00 -0700 Subject: net/core: add NETDEV_BONDING_FAILOVER event Add NETDEV_BONDING_FAILOVER event to be used in a successive patch by bonding to announce fail-over for the active-backup mode through the netdev events notifier chain mechanism. Such an event can be of use for the RDMA CM (communication manager) to let native RDMA ULPs (eg NFS-RDMA, iSER) always be aligned with the IP stack, in the sense that they use the same ports/links as the stack does. More usages can be done to allow monitoring tools based on netlink events being aware to bonding fail-over. Signed-off-by: Or Gerlitz Signed-off-by: Jay Vosburgh Signed-off-by: Jeff Garzik --- net/core/dev.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'net/core/dev.c') diff --git a/net/core/dev.c b/net/core/dev.c index 68d8df0992ab..0e45742e7158 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -961,6 +961,12 @@ void netdev_state_change(struct net_device *dev) } } +void netdev_bonding_change(struct net_device *dev) +{ + call_netdevice_notifiers(NETDEV_BONDING_FAILOVER, dev); +} +EXPORT_SYMBOL(netdev_bonding_change); + /** * dev_load - load a network module * @net: the applicable net namespace -- cgit v1.2.3 From dad9b335c6940de2746a9788eb456d09cf102f81 Mon Sep 17 00:00:00 2001 From: Wang Chen Date: Wed, 18 Jun 2008 01:48:28 -0700 Subject: netdevice: Fix promiscuity and allmulti overflow Max of promiscuity and allmulti plus positive @inc can cause overflow. Fox example: when allmulti=0xFFFFFFFF, any caller give dev_set_allmulti() a positive @inc will cause allmulti be off. This is not what we want, though it's rare case. The fix is that only negative @inc will cause allmulti or promiscuity be off and when any caller makes the counters touch the roof, we return error. Change of v2: Change void function dev_set_promiscuity/allmulti to return int. So callers can get the overflow error. Caller's fix will be done later. Change of v3: 1. Since we return error to caller, we don't need to print KERN_ERROR, KERN_WARNING is enough. 2. In dev_set_promiscuity(), if __dev_set_promiscuity() failed, we return at once. Signed-off-by: Wang Chen Acked-by: Patrick McHardy Signed-off-by: David S. Miller --- include/linux/netdevice.h | 4 ++-- net/core/dev.c | 55 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 12 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 4bf613cd9e2d..45dce2b58d4c 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1476,8 +1476,8 @@ extern int __dev_addr_delete(struct dev_addr_list **list, int *count, void *ad extern int __dev_addr_add(struct dev_addr_list **list, int *count, void *addr, int alen, int newonly); extern int __dev_addr_sync(struct dev_addr_list **to, int *to_count, struct dev_addr_list **from, int *from_count); extern void __dev_addr_unsync(struct dev_addr_list **to, int *to_count, struct dev_addr_list **from, int *from_count); -extern void dev_set_promiscuity(struct net_device *dev, int inc); -extern void dev_set_allmulti(struct net_device *dev, int inc); +extern int dev_set_promiscuity(struct net_device *dev, int inc); +extern int dev_set_allmulti(struct net_device *dev, int inc); extern void netdev_state_change(struct net_device *dev); extern void netdev_bonding_change(struct net_device *dev); extern void netdev_features_change(struct net_device *dev); diff --git a/net/core/dev.c b/net/core/dev.c index 0e45742e7158..a495f712d38c 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2771,16 +2771,29 @@ int netdev_set_master(struct net_device *slave, struct net_device *master) return 0; } -static void __dev_set_promiscuity(struct net_device *dev, int inc) +static int __dev_set_promiscuity(struct net_device *dev, int inc) { unsigned short old_flags = dev->flags; ASSERT_RTNL(); - if ((dev->promiscuity += inc) == 0) - dev->flags &= ~IFF_PROMISC; - else - dev->flags |= IFF_PROMISC; + dev->flags |= IFF_PROMISC; + dev->promiscuity += inc; + if (dev->promiscuity == 0) { + /* + * Avoid overflow. + * If inc causes overflow, untouch promisc and return error. + */ + if (inc < 0) + dev->flags &= ~IFF_PROMISC; + else { + dev->promiscuity -= inc; + printk(KERN_WARNING "%s: promiscuity touches roof, " + "set promiscuity failed, promiscuity feature " + "of device might be broken.\n", dev->name); + return -EOVERFLOW; + } + } if (dev->flags != old_flags) { printk(KERN_INFO "device %s %s promiscuous mode\n", dev->name, (dev->flags & IFF_PROMISC) ? "entered" : @@ -2798,6 +2811,7 @@ static void __dev_set_promiscuity(struct net_device *dev, int inc) if (dev->change_rx_flags) dev->change_rx_flags(dev, IFF_PROMISC); } + return 0; } /** @@ -2809,14 +2823,19 @@ static void __dev_set_promiscuity(struct net_device *dev, int inc) * remains above zero the interface remains promiscuous. Once it hits zero * the device reverts back to normal filtering operation. A negative inc * value is used to drop promiscuity on the device. + * Return 0 if successful or a negative errno code on error. */ -void dev_set_promiscuity(struct net_device *dev, int inc) +int dev_set_promiscuity(struct net_device *dev, int inc) { unsigned short old_flags = dev->flags; + int err; - __dev_set_promiscuity(dev, inc); + err = __dev_set_promiscuity(dev, inc); + if (!err) + return err; if (dev->flags != old_flags) dev_set_rx_mode(dev); + return err; } /** @@ -2829,22 +2848,38 @@ void dev_set_promiscuity(struct net_device *dev, int inc) * to all interfaces. Once it hits zero the device reverts back to normal * filtering operation. A negative @inc value is used to drop the counter * when releasing a resource needing all multicasts. + * Return 0 if successful or a negative errno code on error. */ -void dev_set_allmulti(struct net_device *dev, int inc) +int dev_set_allmulti(struct net_device *dev, int inc) { unsigned short old_flags = dev->flags; ASSERT_RTNL(); dev->flags |= IFF_ALLMULTI; - if ((dev->allmulti += inc) == 0) - dev->flags &= ~IFF_ALLMULTI; + dev->allmulti += inc; + if (dev->allmulti == 0) { + /* + * Avoid overflow. + * If inc causes overflow, untouch allmulti and return error. + */ + if (inc < 0) + dev->flags &= ~IFF_ALLMULTI; + else { + dev->allmulti -= inc; + printk(KERN_WARNING "%s: allmulti touches roof, " + "set allmulti failed, allmulti feature of " + "device might be broken.\n", dev->name); + return -EOVERFLOW; + } + } if (dev->flags ^ old_flags) { if (dev->change_rx_flags) dev->change_rx_flags(dev, IFF_ALLMULTI); dev_set_rx_mode(dev); } + return 0; } /* -- cgit v1.2.3 From 0187bdfb05674147774ca79a79942537f3ad54bd Mon Sep 17 00:00:00 2001 From: Ben Hutchings Date: Thu, 19 Jun 2008 16:15:47 -0700 Subject: net: Disable LRO on devices that are forwarding Large Receive Offload (LRO) is only appropriate for packets that are destined for the host, and should be disabled if received packets may be forwarded. It can also confuse the GSO on output. Add dev_disable_lro() function which uses the appropriate ethtool ops to disable LRO if enabled. Add calls to dev_disable_lro() in br_add_if() and functions that enable IPv4 and IPv6 forwarding. Signed-off-by: Ben Hutchings Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 + net/bridge/br_if.c | 1 + net/core/dev.c | 24 ++++++++++++++++++++++++ net/ipv4/devinet.c | 21 ++++++++++++++++----- net/ipv6/addrconf.c | 6 ++++++ 5 files changed, 48 insertions(+), 5 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 45dce2b58d4c..1304ad2d7105 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -886,6 +886,7 @@ extern struct net_device *__dev_get_by_name(struct net *net, const char *name); extern int dev_alloc_name(struct net_device *dev, const char *name); extern int dev_open(struct net_device *dev); extern int dev_close(struct net_device *dev); +extern void dev_disable_lro(struct net_device *dev); extern int dev_queue_xmit(struct sk_buff *skb); extern int register_netdevice(struct net_device *dev); extern void unregister_netdevice(struct net_device *dev); diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index 143c954681b8..832a561500d9 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -387,6 +387,7 @@ int br_add_if(struct net_bridge *br, struct net_device *dev) goto err2; rcu_assign_pointer(dev->br_port, p); + dev_disable_lro(dev); dev_set_promiscuity(dev, 1); list_add_rcu(&p->list, &br->port_list); diff --git a/net/core/dev.c b/net/core/dev.c index a495f712d38c..f6944ecd5b2e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -90,6 +90,7 @@ #include #include #include +#include #include #include #include @@ -1123,6 +1124,29 @@ int dev_close(struct net_device *dev) } +/** + * dev_disable_lro - disable Large Receive Offload on a device + * @dev: device + * + * Disable Large Receive Offload (LRO) on a net device. Must be + * called under RTNL. This is needed if received packets may be + * forwarded to another interface. + */ +void dev_disable_lro(struct net_device *dev) +{ + if (dev->ethtool_ops && dev->ethtool_ops->get_flags && + dev->ethtool_ops->set_flags) { + u32 flags = dev->ethtool_ops->get_flags(dev); + if (flags & ETH_FLAG_LRO) { + flags &= ~ETH_FLAG_LRO; + dev->ethtool_ops->set_flags(dev, flags); + } + } + WARN_ON(dev->features & NETIF_F_LRO); +} +EXPORT_SYMBOL(dev_disable_lro); + + static int dev_boot_phase = 1; /* diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index f8c0b0aea93a..9de2514946ca 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -168,6 +168,8 @@ static struct in_device *inetdev_init(struct net_device *dev) in_dev->dev = dev; if ((in_dev->arp_parms = neigh_parms_alloc(dev, &arp_tbl)) == NULL) goto out_kfree; + if (IPV4_DEVCONF(in_dev->cnf, FORWARDING)) + dev_disable_lro(dev); /* Reference in_dev->dev */ dev_hold(dev); /* Account for reference dev->ip_ptr (below) */ @@ -1241,6 +1243,8 @@ static void inet_forward_change(struct net *net) read_lock(&dev_base_lock); for_each_netdev(net, dev) { struct in_device *in_dev; + if (on) + dev_disable_lro(dev); rcu_read_lock(); in_dev = __in_dev_get_rcu(dev); if (in_dev) @@ -1248,8 +1252,6 @@ static void inet_forward_change(struct net *net) rcu_read_unlock(); } read_unlock(&dev_base_lock); - - rt_cache_flush(0); } static int devinet_conf_proc(ctl_table *ctl, int write, @@ -1335,10 +1337,19 @@ static int devinet_sysctl_forward(ctl_table *ctl, int write, if (write && *valp != val) { struct net *net = ctl->extra2; - if (valp == &IPV4_DEVCONF_ALL(net, FORWARDING)) - inet_forward_change(net); - else if (valp != &IPV4_DEVCONF_DFLT(net, FORWARDING)) + if (valp != &IPV4_DEVCONF_DFLT(net, FORWARDING)) { + rtnl_lock(); + if (valp == &IPV4_DEVCONF_ALL(net, FORWARDING)) { + inet_forward_change(net); + } else if (*valp) { + struct ipv4_devconf *cnf = ctl->extra1; + struct in_device *idev = + container_of(cnf, struct in_device, cnf); + dev_disable_lro(idev->dev); + } + rtnl_unlock(); rt_cache_flush(0); + } } return ret; diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 9be6be3a7ff3..84127d854cfc 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -348,6 +348,8 @@ static struct inet6_dev * ipv6_add_dev(struct net_device *dev) kfree(ndev); return NULL; } + if (ndev->cnf.forwarding) + dev_disable_lro(dev); /* We refer to the device */ dev_hold(dev); @@ -442,6 +444,8 @@ static void dev_forward_change(struct inet6_dev *idev) if (!idev) return; dev = idev->dev; + if (idev->cnf.forwarding) + dev_disable_lro(dev); if (dev && (dev->flags & IFF_MULTICAST)) { if (idev->cnf.forwarding) ipv6_dev_mc_inc(dev, &in6addr_linklocal_allrouters); @@ -487,12 +491,14 @@ static void addrconf_fixup_forwarding(struct ctl_table *table, int *p, int old) if (p == &net->ipv6.devconf_dflt->forwarding) return; + rtnl_lock(); if (p == &net->ipv6.devconf_all->forwarding) { __s32 newf = net->ipv6.devconf_all->forwarding; net->ipv6.devconf_dflt->forwarding = newf; addrconf_forward_change(net, newf); } else if ((!*p) ^ (!old)) dev_forward_change((struct inet6_dev *)table->extra1); + rtnl_unlock(); if (*p) rt6_purge_dflt_routers(net); -- cgit v1.2.3 From 4b5a698ef423eebc37cfacc6d3376d6dffd5bf83 Mon Sep 17 00:00:00 2001 From: Patrick McHardy Date: Sun, 6 Jul 2008 15:49:08 -0700 Subject: net: fix dev_set_promiscuity() breakage Commit dad9b335 (netdevice: Fix promiscuity and allmulti overflow) broke dev_set_promiscuity() by returning on success without reprogramming the device. Signed-off-by: Patrick McHardy Signed-off-by: David S. Miller --- net/core/dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/core/dev.c') diff --git a/net/core/dev.c b/net/core/dev.c index bfa9a6a951dd..75933932463d 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2859,7 +2859,7 @@ int dev_set_promiscuity(struct net_device *dev, int inc) int err; err = __dev_set_promiscuity(dev, inc); - if (!err) + if (err < 0) return err; if (dev->flags != old_flags) dev_set_rx_mode(dev); -- cgit v1.2.3 From bb949fbd1878973c3539d9aecff52f284482a937 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 16:55:56 -0700 Subject: netdev: Create netdev_queue abstraction. A netdev_queue is an entity managed by a qdisc. Currently there is one RX and one TX queue, and a netdev_queue merely contains a backpointer to the net_device. The Qdisc struct is augmented with a netdev_queue pointer as well. Eventually the 'dev' Qdisc member will go away and we will have the resulting hierarchy: net_device --> netdev_queue --> Qdisc Also, qdisc_alloc() and qdisc_create_dflt() now take a netdev_queue pointer argument. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 7 +++++++ include/net/sch_generic.h | 6 +++++- net/core/dev.c | 8 ++++++++ net/mac80211/wme.c | 6 ++++-- net/sched/sch_api.c | 12 +++++++----- net/sched/sch_atm.c | 6 ++++-- net/sched/sch_cbq.c | 9 ++++++--- net/sched/sch_dsmark.c | 6 ++++-- net/sched/sch_fifo.c | 3 ++- net/sched/sch_generic.c | 14 ++++++++++---- net/sched/sch_hfsc.c | 9 ++++++--- net/sched/sch_htb.c | 11 +++++++---- net/sched/sch_netem.c | 3 ++- net/sched/sch_prio.c | 3 ++- 14 files changed, 74 insertions(+), 29 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e009c6fbf5cd..515fd25bf0fc 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -448,6 +448,10 @@ static inline void napi_synchronize(const struct napi_struct *n) # define napi_synchronize(n) barrier() #endif +struct netdev_queue { + struct net_device *dev; +}; + /* * The DEVICE structure. * Actually, this whole structure is a big mistake. It mixes I/O @@ -624,6 +628,9 @@ struct net_device unsigned char broadcast[MAX_ADDR_LEN]; /* hw bcast add */ + struct netdev_queue rx_queue; + struct netdev_queue tx_queue; + /* ingress path synchronizer */ spinlock_t ingress_lock; struct Qdisc *qdisc_ingress; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 073f2580b83b..0ab53c575f87 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -37,6 +37,7 @@ struct Qdisc u32 parent; atomic_t refcnt; struct sk_buff_head q; + struct netdev_queue *dev_queue; struct net_device *dev; struct list_head list; @@ -216,8 +217,11 @@ extern void dev_deactivate(struct net_device *dev); extern void qdisc_reset(struct Qdisc *qdisc); extern void qdisc_destroy(struct Qdisc *qdisc); extern void qdisc_tree_decrease_qlen(struct Qdisc *qdisc, unsigned int n); -extern struct Qdisc *qdisc_alloc(struct net_device *dev, struct Qdisc_ops *ops); +extern struct Qdisc *qdisc_alloc(struct net_device *dev, + struct netdev_queue *dev_queue, + struct Qdisc_ops *ops); extern struct Qdisc *qdisc_create_dflt(struct net_device *dev, + struct netdev_queue *dev_queue, struct Qdisc_ops *ops, u32 parentid); extern void tcf_destroy(struct tcf_proto *tp); extern void tcf_destroy_chain(struct tcf_proto **fl); diff --git a/net/core/dev.c b/net/core/dev.c index 75933932463d..9b281c906eb0 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4072,6 +4072,12 @@ static struct net_device_stats *internal_stats(struct net_device *dev) return &dev->stats; } +static void netdev_init_queues(struct net_device *dev) +{ + dev->rx_queue.dev = dev; + dev->tx_queue.dev = dev; +} + /** * alloc_netdev_mq - allocate network device * @sizeof_priv: size of private data to allocate space for @@ -4124,6 +4130,8 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name, dev->egress_subqueue_count = queue_count; dev->gso_max_size = GSO_MAX_SIZE; + netdev_init_queues(dev); + dev->get_stats = internal_stats; netpoll_netdev_init(dev); setup(dev); diff --git a/net/mac80211/wme.c b/net/mac80211/wme.c index 5c666f7eda8f..770f1c09b793 100644 --- a/net/mac80211/wme.c +++ b/net/mac80211/wme.c @@ -359,7 +359,8 @@ static int wme_qdiscop_init(struct Qdisc *qd, struct nlattr *opt) /* create child queues */ for (i = 0; i < QD_NUM(hw); i++) { skb_queue_head_init(&q->requeued[i]); - q->queues[i] = qdisc_create_dflt(qd->dev, &pfifo_qdisc_ops, + q->queues[i] = qdisc_create_dflt(qd->dev, qd->dev_queue, + &pfifo_qdisc_ops, qd->handle); if (!q->queues[i]) { q->queues[i] = &noop_qdisc; @@ -575,7 +576,8 @@ void ieee80211_install_qdisc(struct net_device *dev) { struct Qdisc *qdisc; - qdisc = qdisc_create_dflt(dev, &wme_qdisc_ops, TC_H_ROOT); + qdisc = qdisc_create_dflt(dev, &dev->tx_queue, + &wme_qdisc_ops, TC_H_ROOT); if (!qdisc) { printk(KERN_ERR "%s: qdisc installation failed\n", dev->name); return; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 69e918bb4278..b86c98bd06a3 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -552,8 +552,8 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, */ static struct Qdisc * -qdisc_create(struct net_device *dev, u32 parent, u32 handle, - struct nlattr **tca, int *errp) +qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, + u32 parent, u32 handle, struct nlattr **tca, int *errp) { int err; struct nlattr *kind = tca[TCA_KIND]; @@ -593,7 +593,7 @@ qdisc_create(struct net_device *dev, u32 parent, u32 handle, if (ops == NULL) goto err_out; - sch = qdisc_alloc(dev, ops); + sch = qdisc_alloc(dev, dev_queue, ops); if (IS_ERR(sch)) { err = PTR_ERR(sch); goto err_out2; @@ -892,10 +892,12 @@ create_n_graft: if (!(n->nlmsg_flags&NLM_F_CREATE)) return -ENOENT; if (clid == TC_H_INGRESS) - q = qdisc_create(dev, tcm->tcm_parent, tcm->tcm_parent, + q = qdisc_create(dev, &dev->rx_queue, + tcm->tcm_parent, tcm->tcm_parent, tca, &err); else - q = qdisc_create(dev, tcm->tcm_parent, tcm->tcm_handle, + q = qdisc_create(dev, &dev->tx_queue, + tcm->tcm_parent, tcm->tcm_handle, tca, &err); if (q == NULL) { if (err == -EAGAIN) diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c index db0e23ae85f8..3dddab531d5a 100644 --- a/net/sched/sch_atm.c +++ b/net/sched/sch_atm.c @@ -296,7 +296,8 @@ static int atm_tc_change(struct Qdisc *sch, u32 classid, u32 parent, goto err_out; } flow->filter_list = NULL; - flow->q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, classid); + flow->q = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, classid); if (!flow->q) flow->q = &noop_qdisc; pr_debug("atm_tc_change: qdisc %p\n", flow->q); @@ -555,7 +556,8 @@ static int atm_tc_init(struct Qdisc *sch, struct nlattr *opt) pr_debug("atm_tc_init(sch %p,[qdisc %p],opt %p)\n", sch, p, opt); p->flows = &p->link; - p->link.q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, sch->handle); + p->link.q = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, sch->handle); if (!p->link.q) p->link.q = &noop_qdisc; pr_debug("atm_tc_init: link (%p) qdisc %p\n", &p->link, p->link.q); diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c index 968b4c73c9c1..d360dcd0818b 100644 --- a/net/sched/sch_cbq.c +++ b/net/sched/sch_cbq.c @@ -1401,7 +1401,8 @@ static int cbq_init(struct Qdisc *sch, struct nlattr *opt) q->link.sibling = &q->link; q->link.common.classid = sch->handle; q->link.qdisc = sch; - if (!(q->link.q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, + if (!(q->link.q = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, sch->handle))) q->link.q = &noop_qdisc; @@ -1645,7 +1646,8 @@ static int cbq_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, if (cl) { if (new == NULL) { - new = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, + new = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, cl->common.classid); if (new == NULL) return -ENOBUFS; @@ -1877,7 +1879,8 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t cl->R_tab = rtab; rtab = NULL; cl->refcnt = 1; - if (!(cl->q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, classid))) + if (!(cl->q = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, classid))) cl->q = &noop_qdisc; cl->common.classid = classid; cl->tparent = parent; diff --git a/net/sched/sch_dsmark.c b/net/sched/sch_dsmark.c index c4c1317cd47d..c955ba24e5cf 100644 --- a/net/sched/sch_dsmark.c +++ b/net/sched/sch_dsmark.c @@ -60,7 +60,8 @@ static int dsmark_graft(struct Qdisc *sch, unsigned long arg, sch, p, new, old); if (new == NULL) { - new = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, + new = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, sch->handle); if (new == NULL) new = &noop_qdisc; @@ -390,7 +391,8 @@ static int dsmark_init(struct Qdisc *sch, struct nlattr *opt) p->default_index = default_index; p->set_tc_index = nla_get_flag(tb[TCA_DSMARK_SET_TC_INDEX]); - p->q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, sch->handle); + p->q = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, sch->handle); if (p->q == NULL) p->q = &noop_qdisc; diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c index 82d7d7bbbb16..779eae85faf0 100644 --- a/net/sched/sch_fifo.c +++ b/net/sched/sch_fifo.c @@ -137,7 +137,8 @@ struct Qdisc *fifo_create_dflt(struct Qdisc *sch, struct Qdisc_ops *ops, struct Qdisc *q; int err = -ENOMEM; - q = qdisc_create_dflt(sch->dev, ops, TC_H_MAKE(sch->handle, 1)); + q = qdisc_create_dflt(sch->dev, sch->dev_queue, + ops, TC_H_MAKE(sch->handle, 1)); if (q) { err = fifo_set_limit(q, limit); if (err < 0) { diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 13afa7214392..d97086480893 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -440,7 +440,9 @@ static struct Qdisc_ops pfifo_fast_ops __read_mostly = { .owner = THIS_MODULE, }; -struct Qdisc *qdisc_alloc(struct net_device *dev, struct Qdisc_ops *ops) +struct Qdisc *qdisc_alloc(struct net_device *dev, + struct netdev_queue *dev_queue, + struct Qdisc_ops *ops) { void *p; struct Qdisc *sch; @@ -462,6 +464,7 @@ struct Qdisc *qdisc_alloc(struct net_device *dev, struct Qdisc_ops *ops) sch->ops = ops; sch->enqueue = ops->enqueue; sch->dequeue = ops->dequeue; + sch->dev_queue = dev_queue; sch->dev = dev; dev_hold(dev); atomic_set(&sch->refcnt, 1); @@ -471,12 +474,14 @@ errout: return ERR_PTR(err); } -struct Qdisc * qdisc_create_dflt(struct net_device *dev, struct Qdisc_ops *ops, +struct Qdisc * qdisc_create_dflt(struct net_device *dev, + struct netdev_queue *dev_queue, + struct Qdisc_ops *ops, unsigned int parentid) { struct Qdisc *sch; - sch = qdisc_alloc(dev, ops); + sch = qdisc_alloc(dev, dev_queue, ops); if (IS_ERR(sch)) goto errout; sch->stats_lock = &dev->queue_lock; @@ -545,7 +550,8 @@ void dev_activate(struct net_device *dev) if (dev->qdisc_sleeping == &noop_qdisc) { struct Qdisc *qdisc; if (dev->tx_queue_len) { - qdisc = qdisc_create_dflt(dev, &pfifo_fast_ops, + qdisc = qdisc_create_dflt(dev, &dev->tx_queue, + &pfifo_fast_ops, TC_H_ROOT); if (qdisc == NULL) { printk(KERN_INFO "%s: activation failed\n", dev->name); diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 3a8267246a4f..5a22fec4eadd 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -1083,7 +1083,8 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid, cl->refcnt = 1; cl->sched = q; cl->cl_parent = parent; - cl->qdisc = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, classid); + cl->qdisc = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, classid); if (cl->qdisc == NULL) cl->qdisc = &noop_qdisc; INIT_LIST_HEAD(&cl->children); @@ -1201,7 +1202,8 @@ hfsc_graft_class(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, if (cl->level > 0) return -EINVAL; if (new == NULL) { - new = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, + new = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, cl->cl_common.classid); if (new == NULL) new = &noop_qdisc; @@ -1443,7 +1445,8 @@ hfsc_init_qdisc(struct Qdisc *sch, struct nlattr *opt) q->root.cl_common.classid = sch->handle; q->root.refcnt = 1; q->root.sched = q; - q->root.qdisc = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, + q->root.qdisc = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, sch->handle); if (q->root.qdisc == NULL) q->root.qdisc = &noop_qdisc; diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index ee8b4ffe110c..956a67f66b9c 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -1129,7 +1129,8 @@ static int htb_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, if (cl && !cl->level) { if (new == NULL && - (new = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, + (new = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, cl->common.classid)) == NULL) return -ENOBUFS; @@ -1256,8 +1257,9 @@ static int htb_delete(struct Qdisc *sch, unsigned long arg) return -EBUSY; if (!cl->level && htb_parent_last_child(cl)) { - new_q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, - cl->parent->common.classid); + new_q = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, + cl->parent->common.classid); last_child = 1; } @@ -1376,7 +1378,8 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, /* create leaf qdisc early because it uses kmalloc(GFP_KERNEL) so that can't be used inside of sch_tree_lock -- thanks to Karlis Peisenieks */ - new_q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, classid); + new_q = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, classid); sch_tree_lock(sch); if (parent && !parent->level) { unsigned int qlen = parent->un.leaf.q->q.qlen; diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 24697667247c..aa7a04e32ae9 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -536,7 +536,8 @@ static int netem_init(struct Qdisc *sch, struct nlattr *opt) qdisc_watchdog_init(&q->watchdog, sch); - q->qdisc = qdisc_create_dflt(sch->dev, &tfifo_qdisc_ops, + q->qdisc = qdisc_create_dflt(sch->dev, sch->dev_queue, + &tfifo_qdisc_ops, TC_H_MAKE(sch->handle, 1)); if (!q->qdisc) { pr_debug("netem: qdisc create failed\n"); diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 5532f1031ab5..ca58a039208e 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -281,7 +281,8 @@ static int prio_tune(struct Qdisc *sch, struct nlattr *opt) for (i=0; ibands; i++) { if (q->queues[i] == &noop_qdisc) { struct Qdisc *child; - child = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, + child = qdisc_create_dflt(sch->dev, sch->dev_queue, + &pfifo_qdisc_ops, TC_H_MAKE(sch->handle, i + 1)); if (child) { sch_tree_lock(sch); -- cgit v1.2.3 From dc2b48475a0a36f8b3bbb2da60d3a006dc5c2c84 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 17:18:23 -0700 Subject: netdev: Move queue_lock into struct netdev_queue. The lock is now an attribute of the device queue. One thing to notice is that "suspicious" places emerge which will need specific training about multiple queue handling. They are so marked with explicit "netdev->rx_queue" and "netdev->tx_queue" references. Signed-off-by: David S. Miller --- drivers/net/ifb.c | 8 ++++---- include/linux/netdevice.h | 4 ++-- net/core/dev.c | 33 ++++++++++++++++++++++----------- net/mac80211/main.c | 10 +++++----- net/mac80211/wme.c | 2 +- net/sched/sch_api.c | 2 +- net/sched/sch_cbq.c | 8 ++++---- net/sched/sch_generic.c | 40 ++++++++++++++++++++-------------------- net/sched/sch_hfsc.c | 4 ++-- net/sched/sch_htb.c | 16 ++++++++-------- net/sched/sch_netem.c | 4 ++-- net/sched/sch_teql.c | 4 ++-- 12 files changed, 73 insertions(+), 62 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c index af233b591534..bc3de272a829 100644 --- a/drivers/net/ifb.c +++ b/drivers/net/ifb.c @@ -229,12 +229,12 @@ module_param(numifbs, int, 0); MODULE_PARM_DESC(numifbs, "Number of ifb devices"); /* - * dev_ifb->queue_lock is usually taken after dev->ingress_lock, + * dev_ifb->tx_queue.lock is usually taken after dev->ingress_lock, * reversely to e.g. qdisc_lock_tree(). It should be safe until - * ifb doesn't take dev->queue_lock with dev_ifb->ingress_lock. + * ifb doesn't take dev->tx_queue.lock with dev_ifb->ingress_lock. * But lockdep should know that ifb has different locks from dev. */ -static struct lock_class_key ifb_queue_lock_key; +static struct lock_class_key ifb_tx_queue_lock_key; static struct lock_class_key ifb_ingress_lock_key; @@ -258,7 +258,7 @@ static int __init ifb_init_one(int index) if (err < 0) goto err; - lockdep_set_class(&dev_ifb->queue_lock, &ifb_queue_lock_key); + lockdep_set_class(&dev_ifb->tx_queue.lock, &ifb_tx_queue_lock_key); lockdep_set_class(&dev_ifb->ingress_lock, &ifb_ingress_lock_key); return 0; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 515fd25bf0fc..e835acacb479 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -449,6 +449,7 @@ static inline void napi_synchronize(const struct napi_struct *n) #endif struct netdev_queue { + spinlock_t lock; struct net_device *dev; }; @@ -629,7 +630,7 @@ struct net_device unsigned char broadcast[MAX_ADDR_LEN]; /* hw bcast add */ struct netdev_queue rx_queue; - struct netdev_queue tx_queue; + struct netdev_queue tx_queue ____cacheline_aligned_in_smp; /* ingress path synchronizer */ spinlock_t ingress_lock; @@ -639,7 +640,6 @@ struct net_device * Cache line mostly used on queue transmit path (qdisc) */ /* device queue lock */ - spinlock_t queue_lock ____cacheline_aligned_in_smp; struct Qdisc *qdisc; struct Qdisc *qdisc_sleeping; struct list_head qdisc_list; diff --git a/net/core/dev.c b/net/core/dev.c index 9b281c906eb0..05011048b86c 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1667,6 +1667,7 @@ out_kfree_skb: int dev_queue_xmit(struct sk_buff *skb) { struct net_device *dev = skb->dev; + struct netdev_queue *txq; struct Qdisc *q; int rc = -ENOMEM; @@ -1699,14 +1700,15 @@ int dev_queue_xmit(struct sk_buff *skb) } gso: - spin_lock_prefetch(&dev->queue_lock); + txq = &dev->tx_queue; + spin_lock_prefetch(&txq->lock); /* Disable soft irqs for various locks below. Also * stops preemption for RCU. */ rcu_read_lock_bh(); - /* Updates of qdisc are serialized by queue_lock. + /* Updates of qdisc are serialized by queue->lock. * The struct Qdisc which is pointed to by qdisc is now a * rcu structure - it may be accessed without acquiring * a lock (but the structure may be stale.) The freeing of the @@ -1714,7 +1716,7 @@ gso: * more references to it. * * If the qdisc has an enqueue function, we still need to - * hold the queue_lock before calling it, since queue_lock + * hold the queue->lock before calling it, since queue->lock * also serializes access to the device queue. */ @@ -1724,19 +1726,19 @@ gso: #endif if (q->enqueue) { /* Grab device queue */ - spin_lock(&dev->queue_lock); + spin_lock(&txq->lock); q = dev->qdisc; if (q->enqueue) { /* reset queue_mapping to zero */ skb_set_queue_mapping(skb, 0); rc = q->enqueue(skb, q); qdisc_run(dev); - spin_unlock(&dev->queue_lock); + spin_unlock(&txq->lock); rc = rc == NET_XMIT_BYPASS ? NET_XMIT_SUCCESS : rc; goto out; } - spin_unlock(&dev->queue_lock); + spin_unlock(&txq->lock); } /* The device has no queue. Common case for software devices: @@ -1919,14 +1921,17 @@ static void net_tx_action(struct softirq_action *h) while (head) { struct net_device *dev = head; + struct netdev_queue *txq; head = head->next_sched; + txq = &dev->tx_queue; + smp_mb__before_clear_bit(); clear_bit(__LINK_STATE_SCHED, &dev->state); - if (spin_trylock(&dev->queue_lock)) { + if (spin_trylock(&txq->lock)) { qdisc_run(dev); - spin_unlock(&dev->queue_lock); + spin_unlock(&txq->lock); } else { netif_schedule(dev); } @@ -3787,7 +3792,6 @@ int register_netdevice(struct net_device *dev) BUG_ON(!dev_net(dev)); net = dev_net(dev); - spin_lock_init(&dev->queue_lock); spin_lock_init(&dev->_xmit_lock); netdev_set_lockdep_class(&dev->_xmit_lock, dev->type); dev->xmit_lock_owner = -1; @@ -4072,10 +4076,17 @@ static struct net_device_stats *internal_stats(struct net_device *dev) return &dev->stats; } +static void netdev_init_one_queue(struct net_device *dev, + struct netdev_queue *queue) +{ + spin_lock_init(&queue->lock); + queue->dev = dev; +} + static void netdev_init_queues(struct net_device *dev) { - dev->rx_queue.dev = dev; - dev->tx_queue.dev = dev; + netdev_init_one_queue(dev, &dev->rx_queue); + netdev_init_one_queue(dev, &dev->tx_queue); } /** diff --git a/net/mac80211/main.c b/net/mac80211/main.c index cf477ad39dac..12aeaf78ae75 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -636,7 +636,7 @@ int ieee80211_start_tx_ba_session(struct ieee80211_hw *hw, u8 *ra, u16 tid) /* ensure that TX flow won't interrupt us * until the end of the call to requeue function */ - spin_lock_bh(&local->mdev->queue_lock); + spin_lock_bh(&local->mdev->tx_queue.lock); /* create a new queue for this aggregation */ ret = ieee80211_ht_agg_queue_add(local, sta, tid); @@ -675,7 +675,7 @@ int ieee80211_start_tx_ba_session(struct ieee80211_hw *hw, u8 *ra, u16 tid) /* Will put all the packets in the new SW queue */ ieee80211_requeue(local, ieee802_1d_to_ac[tid]); - spin_unlock_bh(&local->mdev->queue_lock); + spin_unlock_bh(&local->mdev->tx_queue.lock); spin_unlock_bh(&sta->lock); /* send an addBA request */ @@ -701,7 +701,7 @@ int ieee80211_start_tx_ba_session(struct ieee80211_hw *hw, u8 *ra, u16 tid) err_unlock_queue: kfree(sta->ampdu_mlme.tid_tx[tid]); sta->ampdu_mlme.tid_tx[tid] = NULL; - spin_unlock_bh(&local->mdev->queue_lock); + spin_unlock_bh(&local->mdev->tx_queue.lock); ret = -EBUSY; err_unlock_sta: spin_unlock_bh(&sta->lock); @@ -875,10 +875,10 @@ void ieee80211_stop_tx_ba_cb(struct ieee80211_hw *hw, u8 *ra, u8 tid) /* avoid ordering issues: we are the only one that can modify * the content of the qdiscs */ - spin_lock_bh(&local->mdev->queue_lock); + spin_lock_bh(&local->mdev->tx_queue.lock); /* remove the queue for this aggregation */ ieee80211_ht_agg_queue_remove(local, sta, tid, 1); - spin_unlock_bh(&local->mdev->queue_lock); + spin_unlock_bh(&local->mdev->tx_queue.lock); /* we just requeued the all the frames that were in the removed * queue, and since we might miss a softirq we do netif_schedule. diff --git a/net/mac80211/wme.c b/net/mac80211/wme.c index 2fbc171130bf..59ed9cae66b9 100644 --- a/net/mac80211/wme.c +++ b/net/mac80211/wme.c @@ -648,7 +648,7 @@ int ieee80211_ht_agg_queue_add(struct ieee80211_local *local, } /** - * the caller needs to hold local->mdev->queue_lock + * the caller needs to hold local->mdev->tx_queue.lock */ void ieee80211_ht_agg_queue_remove(struct ieee80211_local *local, struct sta_info *sta, u16 tid, diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 1f893082a4f6..2a1834f8c7d8 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -606,7 +606,7 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, sch->stats_lock = &dev->ingress_lock; handle = TC_H_MAKE(TC_H_INGRESS, 0); } else { - sch->stats_lock = &dev->queue_lock; + sch->stats_lock = &dev_queue->lock; if (handle == 0) { handle = qdisc_alloc_handle(dev); err = -ENOMEM; diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c index 9f2ace585fd6..99ce3da2b0a4 100644 --- a/net/sched/sch_cbq.c +++ b/net/sched/sch_cbq.c @@ -1746,10 +1746,10 @@ static void cbq_put(struct Qdisc *sch, unsigned long arg) #ifdef CONFIG_NET_CLS_ACT struct cbq_sched_data *q = qdisc_priv(sch); - spin_lock_bh(&qdisc_dev(sch)->queue_lock); + spin_lock_bh(&sch->dev_queue->lock); if (q->rx_class == cl) q->rx_class = NULL; - spin_unlock_bh(&qdisc_dev(sch)->queue_lock); + spin_unlock_bh(&sch->dev_queue->lock); #endif cbq_destroy_class(sch, cl); @@ -1828,7 +1828,7 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t if (tca[TCA_RATE]) gen_replace_estimator(&cl->bstats, &cl->rate_est, - &qdisc_dev(sch)->queue_lock, + &sch->dev_queue->lock, tca[TCA_RATE]); return 0; } @@ -1919,7 +1919,7 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t if (tca[TCA_RATE]) gen_new_estimator(&cl->bstats, &cl->rate_est, - &qdisc_dev(sch)->queue_lock, tca[TCA_RATE]); + &sch->dev_queue->lock, tca[TCA_RATE]); *arg = (unsigned long)cl; return 0; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index b626a4f32b6b..ee8f9f78a095 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -29,31 +29,31 @@ /* Main transmission queue. */ /* Modifications to data participating in scheduling must be protected with - * dev->queue_lock spinlock. + * queue->lock spinlock. * * The idea is the following: * - enqueue, dequeue are serialized via top level device - * spinlock dev->queue_lock. + * spinlock queue->lock. * - ingress filtering is serialized via top level device * spinlock dev->ingress_lock. * - updates to tree and tree walking are only done under the rtnl mutex. */ void qdisc_lock_tree(struct net_device *dev) - __acquires(dev->queue_lock) + __acquires(dev->tx_queue.lock) __acquires(dev->ingress_lock) { - spin_lock_bh(&dev->queue_lock); + spin_lock_bh(&dev->tx_queue.lock); spin_lock(&dev->ingress_lock); } EXPORT_SYMBOL(qdisc_lock_tree); void qdisc_unlock_tree(struct net_device *dev) __releases(dev->ingress_lock) - __releases(dev->queue_lock) + __releases(dev->tx_queue.lock) { spin_unlock(&dev->ingress_lock); - spin_unlock_bh(&dev->queue_lock); + spin_unlock_bh(&dev->tx_queue.lock); } EXPORT_SYMBOL(qdisc_unlock_tree); @@ -118,15 +118,15 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, } /* - * NOTE: Called under dev->queue_lock with locally disabled BH. + * NOTE: Called under queue->lock with locally disabled BH. * * __LINK_STATE_QDISC_RUNNING guarantees only one CPU can process this - * device at a time. dev->queue_lock serializes queue accesses for + * device at a time. queue->lock serializes queue accesses for * this device AND dev->qdisc pointer itself. * * netif_tx_lock serializes accesses to device driver. * - * dev->queue_lock and netif_tx_lock are mutually exclusive, + * queue->lock and netif_tx_lock are mutually exclusive, * if one is grabbed, another must be free. * * Note, that this procedure can be called by a watchdog timer @@ -148,14 +148,14 @@ static inline int qdisc_restart(struct net_device *dev) /* And release queue */ - spin_unlock(&dev->queue_lock); + spin_unlock(&q->dev_queue->lock); HARD_TX_LOCK(dev, smp_processor_id()); if (!netif_subqueue_stopped(dev, skb)) ret = dev_hard_start_xmit(skb, dev); HARD_TX_UNLOCK(dev); - spin_lock(&dev->queue_lock); + spin_lock(&q->dev_queue->lock); q = dev->qdisc; switch (ret) { @@ -482,7 +482,7 @@ struct Qdisc * qdisc_create_dflt(struct net_device *dev, sch = qdisc_alloc(dev_queue, ops); if (IS_ERR(sch)) goto errout; - sch->stats_lock = &dev->queue_lock; + sch->stats_lock = &dev_queue->lock; sch->parent = parentid; if (!ops->init || ops->init(sch, NULL) == 0) @@ -494,7 +494,7 @@ errout: } EXPORT_SYMBOL(qdisc_create_dflt); -/* Under dev->queue_lock and BH! */ +/* Under queue->lock and BH! */ void qdisc_reset(struct Qdisc *qdisc) { @@ -514,7 +514,7 @@ static void __qdisc_destroy(struct rcu_head *head) kfree((char *) qdisc - qdisc->padded); } -/* Under dev->queue_lock and BH! */ +/* Under queue->lock and BH! */ void qdisc_destroy(struct Qdisc *qdisc) { @@ -566,13 +566,13 @@ void dev_activate(struct net_device *dev) /* Delay activation until next carrier-on event */ return; - spin_lock_bh(&dev->queue_lock); + spin_lock_bh(&dev->tx_queue.lock); rcu_assign_pointer(dev->qdisc, dev->qdisc_sleeping); if (dev->qdisc != &noqueue_qdisc) { dev->trans_start = jiffies; dev_watchdog_up(dev); } - spin_unlock_bh(&dev->queue_lock); + spin_unlock_bh(&dev->tx_queue.lock); } void dev_deactivate(struct net_device *dev) @@ -581,7 +581,7 @@ void dev_deactivate(struct net_device *dev) struct sk_buff *skb; int running; - spin_lock_bh(&dev->queue_lock); + spin_lock_bh(&dev->tx_queue.lock); qdisc = dev->qdisc; dev->qdisc = &noop_qdisc; @@ -589,7 +589,7 @@ void dev_deactivate(struct net_device *dev) skb = dev->gso_skb; dev->gso_skb = NULL; - spin_unlock_bh(&dev->queue_lock); + spin_unlock_bh(&dev->tx_queue.lock); kfree_skb(skb); @@ -607,9 +607,9 @@ void dev_deactivate(struct net_device *dev) * Double-check inside queue lock to ensure that all effects * of the queue run are visible when we return. */ - spin_lock_bh(&dev->queue_lock); + spin_lock_bh(&dev->tx_queue.lock); running = test_bit(__LINK_STATE_QDISC_RUNNING, &dev->state); - spin_unlock_bh(&dev->queue_lock); + spin_unlock_bh(&dev->tx_queue.lock); /* * The running flag should never be set at this point because diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 333525422f45..997d520ca580 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -1045,7 +1045,7 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid, if (tca[TCA_RATE]) gen_replace_estimator(&cl->bstats, &cl->rate_est, - &qdisc_dev(sch)->queue_lock, + &sch->dev_queue->lock, tca[TCA_RATE]); return 0; } @@ -1104,7 +1104,7 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid, if (tca[TCA_RATE]) gen_new_estimator(&cl->bstats, &cl->rate_est, - &qdisc_dev(sch)->queue_lock, tca[TCA_RATE]); + &sch->dev_queue->lock, tca[TCA_RATE]); *arg = (unsigned long)cl; return 0; } diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index 31f7d1536e6d..c8ca54cc26b0 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -1043,7 +1043,7 @@ static int htb_dump(struct Qdisc *sch, struct sk_buff *skb) struct nlattr *nest; struct tc_htb_glob gopt; - spin_lock_bh(&qdisc_dev(sch)->queue_lock); + spin_lock_bh(&sch->dev_queue->lock); gopt.direct_pkts = q->direct_pkts; gopt.version = HTB_VER; @@ -1057,11 +1057,11 @@ static int htb_dump(struct Qdisc *sch, struct sk_buff *skb) NLA_PUT(skb, TCA_HTB_INIT, sizeof(gopt), &gopt); nla_nest_end(skb, nest); - spin_unlock_bh(&qdisc_dev(sch)->queue_lock); + spin_unlock_bh(&sch->dev_queue->lock); return skb->len; nla_put_failure: - spin_unlock_bh(&qdisc_dev(sch)->queue_lock); + spin_unlock_bh(&sch->dev_queue->lock); nla_nest_cancel(skb, nest); return -1; } @@ -1073,7 +1073,7 @@ static int htb_dump_class(struct Qdisc *sch, unsigned long arg, struct nlattr *nest; struct tc_htb_opt opt; - spin_lock_bh(&qdisc_dev(sch)->queue_lock); + spin_lock_bh(&sch->dev_queue->lock); tcm->tcm_parent = cl->parent ? cl->parent->common.classid : TC_H_ROOT; tcm->tcm_handle = cl->common.classid; if (!cl->level && cl->un.leaf.q) @@ -1095,11 +1095,11 @@ static int htb_dump_class(struct Qdisc *sch, unsigned long arg, NLA_PUT(skb, TCA_HTB_PARMS, sizeof(opt), &opt); nla_nest_end(skb, nest); - spin_unlock_bh(&qdisc_dev(sch)->queue_lock); + spin_unlock_bh(&sch->dev_queue->lock); return skb->len; nla_put_failure: - spin_unlock_bh(&qdisc_dev(sch)->queue_lock); + spin_unlock_bh(&sch->dev_queue->lock); nla_nest_cancel(skb, nest); return -1; } @@ -1365,7 +1365,7 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, goto failure; gen_new_estimator(&cl->bstats, &cl->rate_est, - &qdisc_dev(sch)->queue_lock, + &sch->dev_queue->lock, tca[TCA_RATE] ? : &est.nla); cl->refcnt = 1; cl->children = 0; @@ -1420,7 +1420,7 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, } else { if (tca[TCA_RATE]) gen_replace_estimator(&cl->bstats, &cl->rate_est, - &qdisc_dev(sch)->queue_lock, + &sch->dev_queue->lock, tca[TCA_RATE]); sch_tree_lock(sch); } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 790582960444..71b73c528f9b 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -333,9 +333,9 @@ static int get_dist_table(struct Qdisc *sch, const struct nlattr *attr) for (i = 0; i < n; i++) d->table[i] = data[i]; - spin_lock_bh(&qdisc_dev(sch)->queue_lock); + spin_lock_bh(&sch->dev_queue->lock); d = xchg(&q->delay_dist, d); - spin_unlock_bh(&qdisc_dev(sch)->queue_lock); + spin_unlock_bh(&sch->dev_queue->lock); kfree(d); return 0; diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index b3fc82623fc6..4f3054e8e1ab 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -154,9 +154,9 @@ teql_destroy(struct Qdisc* sch) master->slaves = NEXT_SLAVE(q); if (q == master->slaves) { master->slaves = NULL; - spin_lock_bh(&master->dev->queue_lock); + spin_lock_bh(&master->dev->tx_queue.lock); qdisc_reset(master->dev->qdisc); - spin_unlock_bh(&master->dev->queue_lock); + spin_unlock_bh(&master->dev->tx_queue.lock); } } skb_queue_purge(&dat->q); -- cgit v1.2.3 From 555353cfa1aee293de445bfa6de43276138ddd82 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 17:33:13 -0700 Subject: netdev: The ingress_lock member is no longer needed. Every qdisc is assosciated with a queue, and in the case of ingress qdiscs that will now be netdev->rx_queue so using that queue's lock is the thing to do. Signed-off-by: David S. Miller --- drivers/net/ifb.c | 8 ++++---- include/linux/netdevice.h | 2 -- net/core/dev.c | 12 +++++++----- net/sched/sch_api.c | 3 +-- net/sched/sch_generic.c | 10 +++++----- 5 files changed, 17 insertions(+), 18 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c index bc3de272a829..ccbd6554f6eb 100644 --- a/drivers/net/ifb.c +++ b/drivers/net/ifb.c @@ -229,13 +229,13 @@ module_param(numifbs, int, 0); MODULE_PARM_DESC(numifbs, "Number of ifb devices"); /* - * dev_ifb->tx_queue.lock is usually taken after dev->ingress_lock, + * dev_ifb->tx_queue.lock is usually taken after dev->rx_queue.lock, * reversely to e.g. qdisc_lock_tree(). It should be safe until - * ifb doesn't take dev->tx_queue.lock with dev_ifb->ingress_lock. + * ifb doesn't take dev->tx_queue.lock with dev_ifb->rx_queue.lock. * But lockdep should know that ifb has different locks from dev. */ static struct lock_class_key ifb_tx_queue_lock_key; -static struct lock_class_key ifb_ingress_lock_key; +static struct lock_class_key ifb_rx_queue_lock_key; static int __init ifb_init_one(int index) @@ -259,7 +259,7 @@ static int __init ifb_init_one(int index) goto err; lockdep_set_class(&dev_ifb->tx_queue.lock, &ifb_tx_queue_lock_key); - lockdep_set_class(&dev_ifb->ingress_lock, &ifb_ingress_lock_key); + lockdep_set_class(&dev_ifb->rx_queue.lock, &ifb_rx_queue_lock_key); return 0; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e835acacb479..633a44c6fa5e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -632,8 +632,6 @@ struct net_device struct netdev_queue rx_queue; struct netdev_queue tx_queue ____cacheline_aligned_in_smp; - /* ingress path synchronizer */ - spinlock_t ingress_lock; struct Qdisc *qdisc_ingress; /* diff --git a/net/core/dev.c b/net/core/dev.c index 05011048b86c..2322fb69fd53 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2014,10 +2014,11 @@ static inline struct sk_buff *handle_macvlan(struct sk_buff *skb, */ static int ing_filter(struct sk_buff *skb) { - struct Qdisc *q; struct net_device *dev = skb->dev; - int result = TC_ACT_OK; u32 ttl = G_TC_RTTL(skb->tc_verd); + struct netdev_queue *rxq; + int result = TC_ACT_OK; + struct Qdisc *q; if (MAX_RED_LOOP < ttl++) { printk(KERN_WARNING @@ -2029,10 +2030,12 @@ static int ing_filter(struct sk_buff *skb) skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl); skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); - spin_lock(&dev->ingress_lock); + rxq = &dev->rx_queue; + + spin_lock(&rxq->lock); if ((q = dev->qdisc_ingress) != NULL) result = q->enqueue(skb, q); - spin_unlock(&dev->ingress_lock); + spin_unlock(&rxq->lock); return result; } @@ -3795,7 +3798,6 @@ int register_netdevice(struct net_device *dev) spin_lock_init(&dev->_xmit_lock); netdev_set_lockdep_class(&dev->_xmit_lock, dev->type); dev->xmit_lock_owner = -1; - spin_lock_init(&dev->ingress_lock); dev->iflink = -1; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 2a1834f8c7d8..570cef2a9c5f 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -601,12 +601,11 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, sch->parent = parent; + sch->stats_lock = &dev_queue->lock; if (handle == TC_H_INGRESS) { sch->flags |= TCQ_F_INGRESS; - sch->stats_lock = &dev->ingress_lock; handle = TC_H_MAKE(TC_H_INGRESS, 0); } else { - sch->stats_lock = &dev_queue->lock; if (handle == 0) { handle = qdisc_alloc_handle(dev); err = -ENOMEM; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index ee8f9f78a095..804d44b00348 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -35,24 +35,24 @@ * - enqueue, dequeue are serialized via top level device * spinlock queue->lock. * - ingress filtering is serialized via top level device - * spinlock dev->ingress_lock. + * spinlock dev->rx_queue.lock. * - updates to tree and tree walking are only done under the rtnl mutex. */ void qdisc_lock_tree(struct net_device *dev) __acquires(dev->tx_queue.lock) - __acquires(dev->ingress_lock) + __acquires(dev->rx_queue.lock) { spin_lock_bh(&dev->tx_queue.lock); - spin_lock(&dev->ingress_lock); + spin_lock(&dev->rx_queue.lock); } EXPORT_SYMBOL(qdisc_lock_tree); void qdisc_unlock_tree(struct net_device *dev) - __releases(dev->ingress_lock) + __releases(dev->rx_queue.lock) __releases(dev->tx_queue.lock) { - spin_unlock(&dev->ingress_lock); + spin_unlock(&dev->rx_queue.lock); spin_unlock_bh(&dev->tx_queue.lock); } EXPORT_SYMBOL(qdisc_unlock_tree); -- cgit v1.2.3 From b0e1e6462df3c5944010b3328a546d8fe5d932cd Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 17:42:10 -0700 Subject: netdev: Move rest of qdisc state into struct netdev_queue Now qdisc, qdisc_sleeping, and qdisc_list also live there. Signed-off-by: David S. Miller --- drivers/isdn/i4l/isdn_net.c | 2 +- include/linux/netdevice.h | 10 ++--- include/net/irda/irda_device.h | 2 +- net/core/dev.c | 4 +- net/core/link_watch.c | 8 +++- net/core/rtnetlink.c | 6 ++- net/ipv6/addrconf.c | 3 +- net/mac80211/wme.c | 20 ++++++---- net/sched/cls_api.c | 7 +++- net/sched/sch_api.c | 34 ++++++++++------ net/sched/sch_generic.c | 90 ++++++++++++++++++++++++++---------------- net/sched/sch_netem.c | 2 +- net/sched/sch_teql.c | 14 ++++--- 13 files changed, 125 insertions(+), 77 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/isdn/i4l/isdn_net.c b/drivers/isdn/i4l/isdn_net.c index ef1a300068dc..457bbd119f9b 100644 --- a/drivers/isdn/i4l/isdn_net.c +++ b/drivers/isdn/i4l/isdn_net.c @@ -287,7 +287,7 @@ isdn_net_unbind_channel(isdn_net_local * lp) BEWARE! This chunk of code cannot be called from hardware interrupt handler. I hope it is true. --ANK */ - qdisc_reset(lp->netdev->dev->qdisc); + qdisc_reset(lp->netdev->dev->tx_queue.qdisc); } lp->dialstate = 0; dev->rx_netdev[isdn_dc2minor(lp->isdn_device, lp->isdn_channel)] = NULL; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 633a44c6fa5e..df702a7b3db5 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -451,6 +451,9 @@ static inline void napi_synchronize(const struct napi_struct *n) struct netdev_queue { spinlock_t lock; struct net_device *dev; + struct Qdisc *qdisc; + struct Qdisc *qdisc_sleeping; + struct list_head qdisc_list; }; /* @@ -634,13 +637,6 @@ struct net_device struct Qdisc *qdisc_ingress; -/* - * Cache line mostly used on queue transmit path (qdisc) - */ - /* device queue lock */ - struct Qdisc *qdisc; - struct Qdisc *qdisc_sleeping; - struct list_head qdisc_list; unsigned long tx_queue_len; /* Max frames per queue allowed */ /* Partially transmitted GSO packet. */ diff --git a/include/net/irda/irda_device.h b/include/net/irda/irda_device.h index f70e9b39ebaf..16fbf672e0b2 100644 --- a/include/net/irda/irda_device.h +++ b/include/net/irda/irda_device.h @@ -223,7 +223,7 @@ int irda_device_is_receiving(struct net_device *dev); /* Interface for internal use */ static inline int irda_device_txqueue_empty(const struct net_device *dev) { - return skb_queue_empty(&dev->qdisc->q); + return skb_queue_empty(&dev->tx_queue.qdisc->q); } int irda_device_set_raw_mode(struct net_device* self, int status); struct net_device *alloc_irdadev(int sizeof_priv); diff --git a/net/core/dev.c b/net/core/dev.c index 2322fb69fd53..ce79c28d739d 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1720,14 +1720,14 @@ gso: * also serializes access to the device queue. */ - q = rcu_dereference(dev->qdisc); + q = rcu_dereference(txq->qdisc); #ifdef CONFIG_NET_CLS_ACT skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS); #endif if (q->enqueue) { /* Grab device queue */ spin_lock(&txq->lock); - q = dev->qdisc; + q = txq->qdisc; if (q->enqueue) { /* reset queue_mapping to zero */ skb_set_queue_mapping(skb, 0); diff --git a/net/core/link_watch.c b/net/core/link_watch.c index a5e372b9ec4d..50218218445b 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -79,8 +79,10 @@ static void rfc2863_policy(struct net_device *dev) static int linkwatch_urgent_event(struct net_device *dev) { + struct netdev_queue *txq = &dev->tx_queue; + return netif_running(dev) && netif_carrier_ok(dev) && - dev->qdisc != dev->qdisc_sleeping; + txq->qdisc != txq->qdisc_sleeping; } @@ -181,7 +183,9 @@ static void __linkwatch_run_queue(int urgent_only) rfc2863_policy(dev); if (dev->flags & IFF_UP) { if (netif_carrier_ok(dev)) { - WARN_ON(dev->qdisc_sleeping == &noop_qdisc); + struct netdev_queue *txq = &dev->tx_queue; + + WARN_ON(txq->qdisc_sleeping == &noop_qdisc); dev_activate(dev); } else dev_deactivate(dev); diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 6c8d7f0ea01a..8ef9f1db610e 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -605,6 +605,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev, int type, u32 pid, u32 seq, u32 change, unsigned int flags) { + struct netdev_queue *txq; struct ifinfomsg *ifm; struct nlmsghdr *nlh; struct net_device_stats *stats; @@ -635,8 +636,9 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev, if (dev->master) NLA_PUT_U32(skb, IFLA_MASTER, dev->master->ifindex); - if (dev->qdisc_sleeping) - NLA_PUT_STRING(skb, IFLA_QDISC, dev->qdisc_sleeping->ops->id); + txq = &dev->tx_queue; + if (txq->qdisc_sleeping) + NLA_PUT_STRING(skb, IFLA_QDISC, txq->qdisc_sleeping->ops->id); if (1) { struct rtnl_link_ifmap map = { diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 8572cb05fc21..5c84c798331d 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -231,7 +231,8 @@ const struct in6_addr in6addr_linklocal_allrouters = IN6ADDR_LINKLOCAL_ALLROUTER /* Check if a valid qdisc is available */ static inline int addrconf_qdisc_ok(struct net_device *dev) { - return (dev->qdisc != &noop_qdisc); + struct netdev_queue *txq = &dev->tx_queue; + return (txq->qdisc != &noop_qdisc); } /* Check if a route is valid prefix route */ diff --git a/net/mac80211/wme.c b/net/mac80211/wme.c index 59ed9cae66b9..6ae43a3c7726 100644 --- a/net/mac80211/wme.c +++ b/net/mac80211/wme.c @@ -574,9 +574,10 @@ static struct Qdisc_ops wme_qdisc_ops __read_mostly = void ieee80211_install_qdisc(struct net_device *dev) { + struct netdev_queue *txq = &dev->tx_queue; struct Qdisc *qdisc; - qdisc = qdisc_create_dflt(dev, &dev->tx_queue, + qdisc = qdisc_create_dflt(dev, txq, &wme_qdisc_ops, TC_H_ROOT); if (!qdisc) { printk(KERN_ERR "%s: qdisc installation failed\n", dev->name); @@ -587,15 +588,17 @@ void ieee80211_install_qdisc(struct net_device *dev) qdisc->handle = 0x80010000; qdisc_lock_tree(dev); - list_add_tail(&qdisc->list, &dev->qdisc_list); - dev->qdisc_sleeping = qdisc; + list_add_tail(&qdisc->list, &txq->qdisc_list); + txq->qdisc_sleeping = qdisc; qdisc_unlock_tree(dev); } int ieee80211_qdisc_installed(struct net_device *dev) { - return dev->qdisc_sleeping->ops == &wme_qdisc_ops; + struct netdev_queue *txq = &dev->tx_queue; + + return txq->qdisc_sleeping->ops == &wme_qdisc_ops; } @@ -614,8 +617,9 @@ int ieee80211_ht_agg_queue_add(struct ieee80211_local *local, struct sta_info *sta, u16 tid) { int i; + struct netdev_queue *txq = &local->mdev->tx_queue; struct ieee80211_sched_data *q = - qdisc_priv(local->mdev->qdisc_sleeping); + qdisc_priv(txq->qdisc_sleeping); DECLARE_MAC_BUF(mac); /* prepare the filter and save it for the SW queue @@ -655,8 +659,9 @@ void ieee80211_ht_agg_queue_remove(struct ieee80211_local *local, u8 requeue) { struct ieee80211_hw *hw = &local->hw; + struct netdev_queue *txq = &local->mdev->tx_queue; struct ieee80211_sched_data *q = - qdisc_priv(local->mdev->qdisc_sleeping); + qdisc_priv(txq->qdisc_sleeping); int agg_queue = sta->tid_to_tx_q[tid]; /* return the qdisc to the pool */ @@ -671,7 +676,8 @@ void ieee80211_ht_agg_queue_remove(struct ieee80211_local *local, void ieee80211_requeue(struct ieee80211_local *local, int queue) { - struct Qdisc *root_qd = local->mdev->qdisc_sleeping; + struct netdev_queue *txq = &local->mdev->tx_queue; + struct Qdisc *root_qd = txq->qdisc_sleeping; struct ieee80211_sched_data *q = qdisc_priv(root_qd); struct Qdisc *qdisc = q->queues[queue]; struct sk_buff *skb = NULL; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index e2389f161e46..b483bbea6118 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -166,7 +166,8 @@ replay: /* Find qdisc */ if (!parent) { - q = dev->qdisc_sleeping; + struct netdev_queue *dev_queue = &dev->tx_queue; + q = dev_queue->qdisc_sleeping; parent = q->handle; } else { q = qdisc_lookup(dev, TC_H_MAJ(t->tcm_parent)); @@ -390,6 +391,7 @@ static int tcf_node_dump(struct tcf_proto *tp, unsigned long n, static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb) { struct net *net = sock_net(skb->sk); + struct netdev_queue *dev_queue; int t; int s_t; struct net_device *dev; @@ -408,8 +410,9 @@ static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb) if ((dev = dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL) return skb->len; + dev_queue = &dev->tx_queue; if (!tcm->tcm_parent) - q = dev->qdisc_sleeping; + q = dev_queue->qdisc_sleeping; else q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); if (!q) diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 570cef2a9c5f..2313fa7c97be 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -185,9 +185,10 @@ EXPORT_SYMBOL(unregister_qdisc); struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) { + struct netdev_queue *dev_queue = &dev->tx_queue; struct Qdisc *q; - list_for_each_entry(q, &dev->qdisc_list, list) { + list_for_each_entry(q, &dev_queue->qdisc_list, list) { if (q->handle == handle) return q; } @@ -441,6 +442,7 @@ static u32 qdisc_alloc_handle(struct net_device *dev) static struct Qdisc * dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc) { + struct netdev_queue *dev_queue; struct Qdisc *oqdisc; if (dev->flags & IFF_UP) @@ -459,8 +461,8 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc) } } else { - - oqdisc = dev->qdisc_sleeping; + dev_queue = &dev->tx_queue; + oqdisc = dev_queue->qdisc_sleeping; /* Prune old scheduler */ if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1) @@ -469,8 +471,8 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc) /* ... and graft new one */ if (qdisc == NULL) qdisc = &noop_qdisc; - dev->qdisc_sleeping = qdisc; - dev->qdisc = &noop_qdisc; + dev_queue->qdisc_sleeping = qdisc; + dev_queue->qdisc = &noop_qdisc; } qdisc_unlock_tree(dev); @@ -633,7 +635,7 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, } } qdisc_lock_tree(dev); - list_add_tail(&sch->list, &dev->qdisc_list); + list_add_tail(&sch->list, &dev_queue->qdisc_list); qdisc_unlock_tree(dev); return sch; @@ -740,7 +742,8 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg) q = dev->qdisc_ingress; } } else { - q = dev->qdisc_sleeping; + struct netdev_queue *dev_queue = &dev->tx_queue; + q = dev_queue->qdisc_sleeping; } if (!q) return -ENOENT; @@ -814,7 +817,8 @@ replay: q = dev->qdisc_ingress; } } else { - q = dev->qdisc_sleeping; + struct netdev_queue *dev_queue = &dev->tx_queue; + q = dev_queue->qdisc_sleeping; } /* It may be default qdisc, ignore it */ @@ -1015,12 +1019,14 @@ static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) read_lock(&dev_base_lock); idx = 0; for_each_netdev(&init_net, dev) { + struct netdev_queue *dev_queue; if (idx < s_idx) goto cont; if (idx > s_idx) s_q_idx = 0; q_idx = 0; - list_for_each_entry(q, &dev->qdisc_list, list) { + dev_queue = &dev->tx_queue; + list_for_each_entry(q, &dev_queue->qdisc_list, list) { if (q_idx < s_q_idx) { q_idx++; continue; @@ -1054,6 +1060,7 @@ done: static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg) { struct net *net = sock_net(skb->sk); + struct netdev_queue *dev_queue; struct tcmsg *tcm = NLMSG_DATA(n); struct nlattr *tca[TCA_MAX + 1]; struct net_device *dev; @@ -1091,6 +1098,7 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg) /* Step 1. Determine qdisc handle X:0 */ + dev_queue = &dev->tx_queue; if (pid != TC_H_ROOT) { u32 qid1 = TC_H_MAJ(pid); @@ -1101,7 +1109,7 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg) } else if (qid1) { qid = qid1; } else if (qid == 0) - qid = dev->qdisc_sleeping->handle; + qid = dev_queue->qdisc_sleeping->handle; /* Now qid is genuine qdisc handle consistent both with parent and child. @@ -1112,7 +1120,7 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg) pid = TC_H_MAKE(qid, pid); } else { if (qid == 0) - qid = dev->qdisc_sleeping->handle; + qid = dev_queue->qdisc_sleeping->handle; } /* OK. Locate qdisc */ @@ -1248,6 +1256,7 @@ static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walk static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) { struct net *net = sock_net(skb->sk); + struct netdev_queue *dev_queue; int t; int s_t; struct net_device *dev; @@ -1266,7 +1275,8 @@ static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) s_t = cb->args[0]; t = 0; - list_for_each_entry(q, &dev->qdisc_list, list) { + dev_queue = &dev->tx_queue; + list_for_each_entry(q, &dev_queue->qdisc_list, list) { if (t < s_t || !q->ops->cl_ops || (tcm->tcm_parent && TC_H_MAJ(tcm->tcm_parent) != q->handle)) { diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 804d44b00348..3223e5ba76aa 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -122,7 +122,7 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, * * __LINK_STATE_QDISC_RUNNING guarantees only one CPU can process this * device at a time. queue->lock serializes queue accesses for - * this device AND dev->qdisc pointer itself. + * this device AND txq->qdisc pointer itself. * * netif_tx_lock serializes accesses to device driver. * @@ -138,7 +138,8 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, */ static inline int qdisc_restart(struct net_device *dev) { - struct Qdisc *q = dev->qdisc; + struct netdev_queue *txq = &dev->tx_queue; + struct Qdisc *q = txq->qdisc; struct sk_buff *skb; int ret = NETDEV_TX_BUSY; @@ -148,15 +149,15 @@ static inline int qdisc_restart(struct net_device *dev) /* And release queue */ - spin_unlock(&q->dev_queue->lock); + spin_unlock(&txq->lock); HARD_TX_LOCK(dev, smp_processor_id()); if (!netif_subqueue_stopped(dev, skb)) ret = dev_hard_start_xmit(skb, dev); HARD_TX_UNLOCK(dev); - spin_lock(&q->dev_queue->lock); - q = dev->qdisc; + spin_lock(&txq->lock); + q = txq->qdisc; switch (ret) { case NETDEV_TX_OK: @@ -207,9 +208,10 @@ void __qdisc_run(struct net_device *dev) static void dev_watchdog(unsigned long arg) { struct net_device *dev = (struct net_device *)arg; + struct netdev_queue *txq = &dev->tx_queue; netif_tx_lock(dev); - if (dev->qdisc != &noop_qdisc) { + if (txq->qdisc != &noop_qdisc) { if (netif_device_present(dev) && netif_running(dev) && netif_carrier_ok(dev)) { @@ -539,53 +541,63 @@ EXPORT_SYMBOL(qdisc_destroy); void dev_activate(struct net_device *dev) { + struct netdev_queue *txq = &dev->tx_queue; + /* No queueing discipline is attached to device; create default one i.e. pfifo_fast for devices, which need queueing and noqueue_qdisc for virtual interfaces */ - if (dev->qdisc_sleeping == &noop_qdisc) { + if (txq->qdisc_sleeping == &noop_qdisc) { struct Qdisc *qdisc; if (dev->tx_queue_len) { - qdisc = qdisc_create_dflt(dev, &dev->tx_queue, + qdisc = qdisc_create_dflt(dev, txq, &pfifo_fast_ops, TC_H_ROOT); if (qdisc == NULL) { printk(KERN_INFO "%s: activation failed\n", dev->name); return; } - list_add_tail(&qdisc->list, &dev->qdisc_list); + list_add_tail(&qdisc->list, &txq->qdisc_list); } else { qdisc = &noqueue_qdisc; } - dev->qdisc_sleeping = qdisc; + txq->qdisc_sleeping = qdisc; } if (!netif_carrier_ok(dev)) /* Delay activation until next carrier-on event */ return; - spin_lock_bh(&dev->tx_queue.lock); - rcu_assign_pointer(dev->qdisc, dev->qdisc_sleeping); - if (dev->qdisc != &noqueue_qdisc) { + spin_lock_bh(&txq->lock); + rcu_assign_pointer(txq->qdisc, txq->qdisc_sleeping); + if (txq->qdisc != &noqueue_qdisc) { dev->trans_start = jiffies; dev_watchdog_up(dev); } - spin_unlock_bh(&dev->tx_queue.lock); + spin_unlock_bh(&txq->lock); +} + +static void dev_deactivate_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + struct Qdisc *qdisc_default) +{ + struct Qdisc *qdisc = dev_queue->qdisc; + + if (qdisc) { + dev_queue->qdisc = qdisc_default; + qdisc_reset(qdisc); + } } void dev_deactivate(struct net_device *dev) { - struct Qdisc *qdisc; struct sk_buff *skb; int running; spin_lock_bh(&dev->tx_queue.lock); - qdisc = dev->qdisc; - dev->qdisc = &noop_qdisc; - - qdisc_reset(qdisc); + dev_deactivate_queue(dev, &dev->tx_queue, &noop_qdisc); skb = dev->gso_skb; dev->gso_skb = NULL; @@ -622,32 +634,44 @@ void dev_deactivate(struct net_device *dev) } while (WARN_ON_ONCE(running)); } +static void dev_init_scheduler_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + struct Qdisc *qdisc) +{ + dev_queue->qdisc = qdisc; + dev_queue->qdisc_sleeping = qdisc; + INIT_LIST_HEAD(&dev_queue->qdisc_list); +} + void dev_init_scheduler(struct net_device *dev) { qdisc_lock_tree(dev); - dev->qdisc = &noop_qdisc; - dev->qdisc_sleeping = &noop_qdisc; - INIT_LIST_HEAD(&dev->qdisc_list); + dev_init_scheduler_queue(dev, &dev->tx_queue, &noop_qdisc); + dev_init_scheduler_queue(dev, &dev->rx_queue, NULL); qdisc_unlock_tree(dev); setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev); } -void dev_shutdown(struct net_device *dev) +static void dev_shutdown_scheduler_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + struct Qdisc *qdisc_default) { - struct Qdisc *qdisc; + struct Qdisc *qdisc = dev_queue->qdisc_sleeping; + + if (qdisc) { + dev_queue->qdisc = qdisc_default; + dev_queue->qdisc_sleeping = qdisc_default; - qdisc_lock_tree(dev); - qdisc = dev->qdisc_sleeping; - dev->qdisc = &noop_qdisc; - dev->qdisc_sleeping = &noop_qdisc; - qdisc_destroy(qdisc); -#if defined(CONFIG_NET_SCH_INGRESS) || defined(CONFIG_NET_SCH_INGRESS_MODULE) - if ((qdisc = dev->qdisc_ingress) != NULL) { - dev->qdisc_ingress = NULL; qdisc_destroy(qdisc); } -#endif +} + +void dev_shutdown(struct net_device *dev) +{ + qdisc_lock_tree(dev); + dev_shutdown_scheduler_queue(dev, &dev->tx_queue, &noop_qdisc); + dev_shutdown_scheduler_queue(dev, &dev->rx_queue, NULL); BUG_TRAP(!timer_pending(&dev->watchdog_timer)); qdisc_unlock_tree(dev); } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 71b73c528f9b..4093f1eaaf60 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -180,7 +180,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch) * skb will be queued. */ if (count > 1 && (skb2 = skb_clone(skb, GFP_ATOMIC)) != NULL) { - struct Qdisc *rootq = qdisc_dev(sch)->qdisc; + struct Qdisc *rootq = qdisc_dev(sch)->tx_queue.qdisc; u32 dupsave = q->duplicate; /* prevent duplicating a dup... */ q->duplicate = 0; diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 4f3054e8e1ab..8ac05981be20 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -107,17 +107,19 @@ static struct sk_buff * teql_dequeue(struct Qdisc* sch) { struct teql_sched_data *dat = qdisc_priv(sch); + struct netdev_queue *dat_queue; struct sk_buff *skb; skb = __skb_dequeue(&dat->q); + dat_queue = &dat->m->dev->tx_queue; if (skb == NULL) { - struct net_device *m = qdisc_dev(dat->m->dev->qdisc); + struct net_device *m = qdisc_dev(dat_queue->qdisc); if (m) { dat->m->slaves = sch; netif_wake_queue(m); } } - sch->q.qlen = dat->q.qlen + dat->m->dev->qdisc->q.qlen; + sch->q.qlen = dat->q.qlen + dat_queue->qdisc->q.qlen; return skb; } @@ -155,7 +157,7 @@ teql_destroy(struct Qdisc* sch) if (q == master->slaves) { master->slaves = NULL; spin_lock_bh(&master->dev->tx_queue.lock); - qdisc_reset(master->dev->qdisc); + qdisc_reset(master->dev->tx_queue.qdisc); spin_unlock_bh(&master->dev->tx_queue.lock); } } @@ -216,7 +218,7 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt) static int __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, struct net_device *dev) { - struct teql_sched_data *q = qdisc_priv(dev->qdisc); + struct teql_sched_data *q = qdisc_priv(dev->tx_queue.qdisc); struct neighbour *mn = skb->dst->neighbour; struct neighbour *n = q->ncache; @@ -252,7 +254,7 @@ __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, struct net_device * static inline int teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, struct net_device *dev) { - if (dev->qdisc == &noop_qdisc) + if (dev->tx_queue.qdisc == &noop_qdisc) return -ENODEV; if (dev->header_ops == NULL || @@ -284,7 +286,7 @@ restart: do { struct net_device *slave = qdisc_dev(q); - if (slave->qdisc_sleeping != q) + if (slave->tx_queue.qdisc_sleeping != q) continue; if (netif_queue_stopped(slave) || __netif_subqueue_stopped(slave, subq) || -- cgit v1.2.3 From 816f3258e70db38d6d92c8d871377179fd69160f Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 22:49:00 -0700 Subject: netdev: Kill qdisc_ingress, use netdev->rx_queue.qdisc instead. Now that our qdisc management is bi-directional, per-queue, and fully orthogonal, there is no reason to have a special ingress qdisc pointer in struct net_device. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 3 --- net/core/dev.c | 4 ++-- net/sched/sch_api.c | 11 ++++++----- 3 files changed, 8 insertions(+), 10 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index df702a7b3db5..e7c49246fd88 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -634,9 +634,6 @@ struct net_device struct netdev_queue rx_queue; struct netdev_queue tx_queue ____cacheline_aligned_in_smp; - - struct Qdisc *qdisc_ingress; - unsigned long tx_queue_len; /* Max frames per queue allowed */ /* Partially transmitted GSO packet. */ diff --git a/net/core/dev.c b/net/core/dev.c index ce79c28d739d..ab760a954d99 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2033,7 +2033,7 @@ static int ing_filter(struct sk_buff *skb) rxq = &dev->rx_queue; spin_lock(&rxq->lock); - if ((q = dev->qdisc_ingress) != NULL) + if ((q = rxq->qdisc) != NULL) result = q->enqueue(skb, q); spin_unlock(&rxq->lock); @@ -2044,7 +2044,7 @@ static inline struct sk_buff *handle_ing(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, struct net_device *orig_dev) { - if (!skb->dev->qdisc_ingress) + if (!skb->dev->rx_queue.qdisc) goto out; if (*pt_prev) { diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 2313fa7c97be..4003c280b69f 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -450,14 +450,15 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc) qdisc_lock_tree(dev); if (qdisc && qdisc->flags&TCQ_F_INGRESS) { - oqdisc = dev->qdisc_ingress; + dev_queue = &dev->rx_queue; + oqdisc = dev_queue->qdisc; /* Prune old scheduler */ if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1) { /* delete */ qdisc_reset(oqdisc); - dev->qdisc_ingress = NULL; + dev_queue->qdisc = NULL; } else { /* new */ - dev->qdisc_ingress = qdisc; + dev_queue->qdisc = qdisc; } } else { @@ -739,7 +740,7 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg) return -ENOENT; q = qdisc_leaf(p, clid); } else { /* ingress */ - q = dev->qdisc_ingress; + q = dev->rx_queue.qdisc; } } else { struct netdev_queue *dev_queue = &dev->tx_queue; @@ -814,7 +815,7 @@ replay: return -ENOENT; q = qdisc_leaf(p, clid); } else { /*ingress */ - q = dev->qdisc_ingress; + q = dev->rx_queue.qdisc; } } else { struct netdev_queue *dev_queue = &dev->tx_queue; -- cgit v1.2.3 From ee609cb36220d18c0cf476b066a5ab7e6f6d3a69 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 22:58:37 -0700 Subject: netdev: Move next_sched into struct netdev_queue. We schedule queues, not the device, for output queue processing in BH. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 5 ++--- net/core/dev.c | 15 +++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e7c49246fd88..1379c822e51d 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -454,6 +454,7 @@ struct netdev_queue { struct Qdisc *qdisc; struct Qdisc *qdisc_sleeping; struct list_head qdisc_list; + struct netdev_queue *next_sched; }; /* @@ -545,8 +546,6 @@ struct net_device #define NETIF_F_V6_CSUM (NETIF_F_GEN_CSUM | NETIF_F_IPV6_CSUM) #define NETIF_F_ALL_CSUM (NETIF_F_V4_CSUM | NETIF_F_V6_CSUM) - struct net_device *next_sched; - /* Interface index. Unique device identifier */ int ifindex; int iflink; @@ -940,7 +939,7 @@ static inline int unregister_gifconf(unsigned int family) */ struct softnet_data { - struct net_device *output_queue; + struct netdev_queue *output_queue; struct sk_buff_head input_pkt_queue; struct list_head poll_list; struct sk_buff *completion_queue; diff --git a/net/core/dev.c b/net/core/dev.c index ab760a954d99..d6b8d3c3e6ec 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1323,13 +1323,14 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) void __netif_schedule(struct net_device *dev) { if (!test_and_set_bit(__LINK_STATE_SCHED, &dev->state)) { + struct netdev_queue *txq = &dev->tx_queue; unsigned long flags; struct softnet_data *sd; local_irq_save(flags); sd = &__get_cpu_var(softnet_data); - dev->next_sched = sd->output_queue; - sd->output_queue = dev; + txq->next_sched = sd->output_queue; + sd->output_queue = txq; raise_softirq_irqoff(NET_TX_SOFTIRQ); local_irq_restore(flags); } @@ -1912,7 +1913,7 @@ static void net_tx_action(struct softirq_action *h) } if (sd->output_queue) { - struct net_device *head; + struct netdev_queue *head; local_irq_disable(); head = sd->output_queue; @@ -1920,12 +1921,10 @@ static void net_tx_action(struct softirq_action *h) local_irq_enable(); while (head) { - struct net_device *dev = head; - struct netdev_queue *txq; + struct netdev_queue *txq = head; + struct net_device *dev = txq->dev; head = head->next_sched; - txq = &dev->tx_queue; - smp_mb__before_clear_bit(); clear_bit(__LINK_STATE_SCHED, &dev->state); @@ -4346,7 +4345,7 @@ static int dev_cpu_callback(struct notifier_block *nfb, void *ocpu) { struct sk_buff **list_skb; - struct net_device **list_net; + struct netdev_queue **list_net; struct sk_buff *skb; unsigned int cpu, oldcpu = (unsigned long)ocpu; struct softnet_data *sd, *oldsd; -- cgit v1.2.3 From 86d804e10a37cd86f16bf72386c37e843a98a74b Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 23:11:25 -0700 Subject: netdev: Make netif_schedule() routines work with netdev_queue objects. Only plain netif_schedule() remains taking a net_device, mostly as a compatability item while we transition the rest of these interfaces. Everything else calls netif_schedule_queue() or __netif_schedule(), both of which take a netdev_queue pointer. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 17 ++++++++++++----- net/core/dev.c | 9 +++++---- net/mac80211/main.c | 4 ++-- net/sched/sch_api.c | 4 ++-- net/sched/sch_cbq.c | 2 +- net/sched/sch_generic.c | 10 +++++----- 6 files changed, 27 insertions(+), 19 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index aae6c6d153f2..28aa8e77cee9 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -952,12 +952,19 @@ DECLARE_PER_CPU(struct softnet_data,softnet_data); #define HAVE_NETIF_QUEUE -extern void __netif_schedule(struct net_device *dev); +extern void __netif_schedule(struct netdev_queue *txq); -static inline void netif_schedule(struct net_device *dev) +static inline void netif_schedule_queue(struct netdev_queue *txq) { + struct net_device *dev = txq->dev; + if (!test_bit(__LINK_STATE_XOFF, &dev->state)) - __netif_schedule(dev); + __netif_schedule(txq); +} + +static inline void netif_schedule(struct net_device *dev) +{ + netif_schedule_queue(&dev->tx_queue); } /** @@ -987,7 +994,7 @@ static inline void netif_wake_queue(struct net_device *dev) } #endif if (test_and_clear_bit(__LINK_STATE_XOFF, &dev->state)) - __netif_schedule(dev); + __netif_schedule(&dev->tx_queue); } /** @@ -1103,7 +1110,7 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index) #endif if (test_and_clear_bit(__LINK_STATE_XOFF, &dev->egress_subqueue[queue_index].state)) - __netif_schedule(dev); + __netif_schedule(&dev->tx_queue); #endif } diff --git a/net/core/dev.c b/net/core/dev.c index d6b8d3c3e6ec..0dc888ad4217 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1320,12 +1320,13 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) } -void __netif_schedule(struct net_device *dev) +void __netif_schedule(struct netdev_queue *txq) { + struct net_device *dev = txq->dev; + if (!test_and_set_bit(__LINK_STATE_SCHED, &dev->state)) { - struct netdev_queue *txq = &dev->tx_queue; - unsigned long flags; struct softnet_data *sd; + unsigned long flags; local_irq_save(flags); sd = &__get_cpu_var(softnet_data); @@ -1932,7 +1933,7 @@ static void net_tx_action(struct softirq_action *h) qdisc_run(dev); spin_unlock(&txq->lock); } else { - netif_schedule(dev); + netif_schedule_queue(txq); } } } diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 2968baa66b91..1c4d3ba6b878 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -885,10 +885,10 @@ void ieee80211_stop_tx_ba_cb(struct ieee80211_hw *hw, u8 *ra, u8 tid) spin_unlock_bh(&txq->lock); /* we just requeued the all the frames that were in the removed - * queue, and since we might miss a softirq we do netif_schedule. + * queue, and since we might miss a softirq we do netif_schedule_queue. * ieee80211_wake_queue is not used here as this queue is not * necessarily stopped */ - netif_schedule(local->mdev); + netif_schedule_queue(txq); spin_lock_bh(&sta->lock); *state = HT_AGG_STATE_IDLE; sta->ampdu_mlme.addba_req_num[tid] = 0; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index e73bd68aa7ae..95873f8dd37c 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -282,11 +282,11 @@ static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) { struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog, timer); - struct net_device *dev = qdisc_dev(wd->qdisc); + struct netdev_queue *txq = wd->qdisc->dev_queue; wd->qdisc->flags &= ~TCQ_F_THROTTLED; smp_wmb(); - netif_schedule(dev); + netif_schedule_queue(txq); return HRTIMER_NORESTART; } diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c index 99ce3da2b0a4..4efc836cbf38 100644 --- a/net/sched/sch_cbq.c +++ b/net/sched/sch_cbq.c @@ -650,7 +650,7 @@ static enum hrtimer_restart cbq_undelay(struct hrtimer *timer) } sch->flags &= ~TCQ_F_THROTTLED; - netif_schedule(qdisc_dev(sch)); + netif_schedule_queue(sch->dev_queue); return HRTIMER_NORESTART; } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 8247a406a401..407dfdb142a4 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -62,7 +62,7 @@ static inline int qdisc_qlen(struct Qdisc *q) return q->q.qlen; } -static inline int dev_requeue_skb(struct sk_buff *skb, struct net_device *dev, +static inline int dev_requeue_skb(struct sk_buff *skb, struct netdev_queue *dev_queue, struct Qdisc *q) { @@ -71,7 +71,7 @@ static inline int dev_requeue_skb(struct sk_buff *skb, struct net_device *dev, else q->ops->requeue(skb, q); - netif_schedule(dev); + netif_schedule_queue(dev_queue); return 0; } @@ -114,7 +114,7 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, * some time. */ __get_cpu_var(netdev_rx_stat).cpu_collision++; - ret = dev_requeue_skb(skb, dev, dev_queue, q); + ret = dev_requeue_skb(skb, dev_queue, q); } return ret; @@ -179,7 +179,7 @@ static inline int qdisc_restart(struct net_device *dev) printk(KERN_WARNING "BUG %s code %d qlen %d\n", dev->name, ret, q->q.qlen); - ret = dev_requeue_skb(skb, dev, txq, q); + ret = dev_requeue_skb(skb, txq, q); break; } @@ -200,7 +200,7 @@ void __qdisc_run(struct net_device *dev) * 2. we've been doing it for too long. */ if (need_resched() || jiffies != start_time) { - netif_schedule(dev); + netif_schedule_queue(&dev->tx_queue); break; } } -- cgit v1.2.3 From eb6aafe3f843cb0e939546c03540a3b4911b6964 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 23:12:38 -0700 Subject: pkt_sched: Make qdisc_run take a netdev_queue. This allows us to use this calling convention all the way down into qdisc_restart(). Signed-off-by: David S. Miller --- include/net/pkt_sched.h | 8 +++++--- net/core/dev.c | 4 ++-- net/sched/sch_generic.c | 26 ++++++++++++++------------ 3 files changed, 21 insertions(+), 17 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 8e3a0c4e9d97..2311d242bb35 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -84,13 +84,15 @@ extern struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab); extern void qdisc_put_rtab(struct qdisc_rate_table *tab); -extern void __qdisc_run(struct net_device *dev); +extern void __qdisc_run(struct netdev_queue *txq); -static inline void qdisc_run(struct net_device *dev) +static inline void qdisc_run(struct netdev_queue *txq) { + struct net_device *dev = txq->dev; + if (!netif_queue_stopped(dev) && !test_and_set_bit(__LINK_STATE_QDISC_RUNNING, &dev->state)) - __qdisc_run(dev); + __qdisc_run(txq); } extern int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp, diff --git a/net/core/dev.c b/net/core/dev.c index 0dc888ad4217..0218b0b9be80 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1734,7 +1734,7 @@ gso: /* reset queue_mapping to zero */ skb_set_queue_mapping(skb, 0); rc = q->enqueue(skb, q); - qdisc_run(dev); + qdisc_run(txq); spin_unlock(&txq->lock); rc = rc == NET_XMIT_BYPASS ? NET_XMIT_SUCCESS : rc; @@ -1930,7 +1930,7 @@ static void net_tx_action(struct softirq_action *h) clear_bit(__LINK_STATE_SCHED, &dev->state); if (spin_trylock(&txq->lock)) { - qdisc_run(dev); + qdisc_run(txq); spin_unlock(&txq->lock); } else { netif_schedule_queue(txq); diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 407dfdb142a4..fcc7533f0bcc 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -75,9 +75,8 @@ static inline int dev_requeue_skb(struct sk_buff *skb, return 0; } -static inline struct sk_buff *dev_dequeue_skb(struct net_device *dev, - struct netdev_queue *dev_queue, - struct Qdisc *q) +static inline struct sk_buff *dequeue_skb(struct netdev_queue *dev_queue, + struct Qdisc *q) { struct sk_buff *skb; @@ -90,10 +89,10 @@ static inline struct sk_buff *dev_dequeue_skb(struct net_device *dev, } static inline int handle_dev_cpu_collision(struct sk_buff *skb, - struct net_device *dev, struct netdev_queue *dev_queue, struct Qdisc *q) { + struct net_device *dev = dev_queue->dev; int ret; if (unlikely(dev->xmit_lock_owner == smp_processor_id())) { @@ -139,21 +138,23 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, * >0 - queue is not empty. * */ -static inline int qdisc_restart(struct net_device *dev) +static inline int qdisc_restart(struct netdev_queue *txq) { - struct netdev_queue *txq = &dev->tx_queue; struct Qdisc *q = txq->qdisc; - struct sk_buff *skb; int ret = NETDEV_TX_BUSY; + struct net_device *dev; + struct sk_buff *skb; /* Dequeue packet */ - if (unlikely((skb = dev_dequeue_skb(dev, txq, q)) == NULL)) + if (unlikely((skb = dequeue_skb(txq, q)) == NULL)) return 0; /* And release queue */ spin_unlock(&txq->lock); + dev = txq->dev; + HARD_TX_LOCK(dev, smp_processor_id()); if (!netif_subqueue_stopped(dev, skb)) ret = dev_hard_start_xmit(skb, dev); @@ -170,7 +171,7 @@ static inline int qdisc_restart(struct net_device *dev) case NETDEV_TX_LOCKED: /* Driver try lock failed */ - ret = handle_dev_cpu_collision(skb, dev, txq, q); + ret = handle_dev_cpu_collision(skb, txq, q); break; default: @@ -186,11 +187,12 @@ static inline int qdisc_restart(struct net_device *dev) return ret; } -void __qdisc_run(struct net_device *dev) +void __qdisc_run(struct netdev_queue *txq) { + struct net_device *dev = txq->dev; unsigned long start_time = jiffies; - while (qdisc_restart(dev)) { + while (qdisc_restart(txq)) { if (netif_queue_stopped(dev)) break; @@ -200,7 +202,7 @@ void __qdisc_run(struct net_device *dev) * 2. we've been doing it for too long. */ if (need_resched() || jiffies != start_time) { - netif_schedule_queue(&dev->tx_queue); + netif_schedule_queue(txq); break; } } -- cgit v1.2.3 From c773e847ea8f6812804e40f52399c6921a00eab1 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 8 Jul 2008 23:13:53 -0700 Subject: netdev: Move _xmit_lock and xmit_lock_owner into netdev_queue. Accesses are mostly structured such that when there are multiple TX queues the code transformations will be a little bit simpler. Signed-off-by: David S. Miller --- drivers/net/bonding/bond_main.c | 13 ++++++- drivers/net/hamradio/bpqether.c | 12 ++++++- drivers/net/macvlan.c | 14 +++++++- drivers/net/wireless/hostap/hostap_hw.c | 12 ++++++- include/linux/netdevice.h | 62 +++++++++++++++++++++------------ net/8021q/vlan_dev.c | 15 ++++++-- net/core/dev.c | 28 ++++++++++----- net/netrom/af_netrom.c | 12 ++++++- net/rose/af_rose.c | 12 ++++++- net/sched/sch_generic.c | 9 +++-- 10 files changed, 145 insertions(+), 44 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index d57b65dc2c72..dc733d75a5e9 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -5019,6 +5019,17 @@ static int bond_check_params(struct bond_params *params) static struct lock_class_key bonding_netdev_xmit_lock_key; +static void bond_set_lockdep_class_one(struct netdev_queue *txq) +{ + lockdep_set_class(&txq->_xmit_lock, + &bonding_netdev_xmit_lock_key); +} + +static void bond_set_lockdep_class(struct net_device *dev) +{ + bond_set_lockdep_class_one(&dev->tx_queue); +} + /* Create a new bond based on the specified name and bonding parameters. * If name is NULL, obtain a suitable "bond%d" name for us. * Caller must NOT hold rtnl_lock; we need to release it here before we @@ -5076,7 +5087,7 @@ int bond_create(char *name, struct bond_params *params) goto out_bond; } - lockdep_set_class(&bond_dev->_xmit_lock, &bonding_netdev_xmit_lock_key); + bond_set_lockdep_class(bond_dev); netif_carrier_off(bond_dev); diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c index 5f4b4c6c9f76..fb186b8c3d4d 100644 --- a/drivers/net/hamradio/bpqether.c +++ b/drivers/net/hamradio/bpqether.c @@ -124,6 +124,16 @@ static LIST_HEAD(bpq_devices); */ static struct lock_class_key bpq_netdev_xmit_lock_key; +static void bpq_set_lockdep_class_one(struct netdev_queue *txq) +{ + lockdep_set_class(&txq->_xmit_lock, &bpq_netdev_xmit_lock_key); +} + +static void bpq_set_lockdep_class(struct net_device *dev) +{ + bpq_set_lockdep_class_one(&dev->tx_queue); +} + /* ------------------------------------------------------------------------ */ @@ -523,7 +533,7 @@ static int bpq_new_device(struct net_device *edev) err = register_netdevice(ndev); if (err) goto error; - lockdep_set_class(&ndev->_xmit_lock, &bpq_netdev_xmit_lock_key); + bpq_set_lockdep_class(ndev); /* List protected by RTNL */ list_add_rcu(&bpq->bpq_list, &bpq_devices); diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index c36a03ae9bfb..c02ceaa4a216 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -277,6 +277,17 @@ static struct lock_class_key macvlan_netdev_xmit_lock_key; #define MACVLAN_STATE_MASK \ ((1<<__LINK_STATE_NOCARRIER) | (1<<__LINK_STATE_DORMANT)) +static void macvlan_set_lockdep_class_one(struct netdev_queue *txq) +{ + lockdep_set_class(&txq->_xmit_lock, + &macvlan_netdev_xmit_lock_key); +} + +static void macvlan_set_lockdep_class(struct net_device *dev) +{ + macvlan_set_lockdep_class_one(&dev->tx_queue); +} + static int macvlan_init(struct net_device *dev) { struct macvlan_dev *vlan = netdev_priv(dev); @@ -287,7 +298,8 @@ static int macvlan_init(struct net_device *dev) dev->features = lowerdev->features & MACVLAN_FEATURES; dev->iflink = lowerdev->ifindex; - lockdep_set_class(&dev->_xmit_lock, &macvlan_netdev_xmit_lock_key); + macvlan_set_lockdep_class(dev); + return 0; } diff --git a/drivers/net/wireless/hostap/hostap_hw.c b/drivers/net/wireless/hostap/hostap_hw.c index 09004a632ae7..c1f4bb005d92 100644 --- a/drivers/net/wireless/hostap/hostap_hw.c +++ b/drivers/net/wireless/hostap/hostap_hw.c @@ -3102,6 +3102,16 @@ static void prism2_clear_set_tim_queue(local_info_t *local) */ static struct lock_class_key hostap_netdev_xmit_lock_key; +static void prism2_set_lockdep_class_one(struct netdev_queue *txq) +{ + lockdep_set_class(&txq->_xmit_lock, + &hostap_netdev_xmit_lock_key); +} + +static void prism2_set_lockdep_class(struct net_device *dev) +{ + prism2_set_lockdep_class_one(&dev->tx_queue); +} static struct net_device * prism2_init_local_data(struct prism2_helper_functions *funcs, int card_idx, @@ -3268,7 +3278,7 @@ while (0) if (ret >= 0) ret = register_netdevice(dev); - lockdep_set_class(&dev->_xmit_lock, &hostap_netdev_xmit_lock_key); + prism2_set_lockdep_class(dev); rtnl_unlock(); if (ret < 0) { printk(KERN_WARNING "%s: register netdevice failed!\n", diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 28aa8e77cee9..c8d5f128858d 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -453,6 +453,8 @@ struct netdev_queue { struct net_device *dev; struct Qdisc *qdisc; struct sk_buff *gso_skb; + spinlock_t _xmit_lock; + int xmit_lock_owner; struct Qdisc *qdisc_sleeping; struct list_head qdisc_list; struct netdev_queue *next_sched; @@ -639,12 +641,6 @@ struct net_device /* * One part is mostly used on xmit path (device) */ - /* hard_start_xmit synchronizer */ - spinlock_t _xmit_lock ____cacheline_aligned_in_smp; - /* cpu id of processor entered to hard_start_xmit or -1, - if nobody entered there. - */ - int xmit_lock_owner; void *priv; /* pointer to private data */ int (*hard_start_xmit) (struct sk_buff *skb, struct net_device *dev); @@ -1402,52 +1398,72 @@ static inline void netif_rx_complete(struct net_device *dev, * * Get network device transmit lock */ -static inline void __netif_tx_lock(struct net_device *dev, int cpu) +static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu) { - spin_lock(&dev->_xmit_lock); - dev->xmit_lock_owner = cpu; + spin_lock(&txq->_xmit_lock); + txq->xmit_lock_owner = cpu; } static inline void netif_tx_lock(struct net_device *dev) { - __netif_tx_lock(dev, smp_processor_id()); + __netif_tx_lock(&dev->tx_queue, smp_processor_id()); +} + +static inline void __netif_tx_lock_bh(struct netdev_queue *txq) +{ + spin_lock_bh(&txq->_xmit_lock); + txq->xmit_lock_owner = smp_processor_id(); } static inline void netif_tx_lock_bh(struct net_device *dev) { - spin_lock_bh(&dev->_xmit_lock); - dev->xmit_lock_owner = smp_processor_id(); + __netif_tx_lock_bh(&dev->tx_queue); } -static inline int netif_tx_trylock(struct net_device *dev) +static inline int __netif_tx_trylock(struct netdev_queue *txq) { - int ok = spin_trylock(&dev->_xmit_lock); + int ok = spin_trylock(&txq->_xmit_lock); if (likely(ok)) - dev->xmit_lock_owner = smp_processor_id(); + txq->xmit_lock_owner = smp_processor_id(); return ok; } +static inline int netif_tx_trylock(struct net_device *dev) +{ + return __netif_tx_trylock(&dev->tx_queue); +} + +static inline void __netif_tx_unlock(struct netdev_queue *txq) +{ + txq->xmit_lock_owner = -1; + spin_unlock(&txq->_xmit_lock); +} + static inline void netif_tx_unlock(struct net_device *dev) { - dev->xmit_lock_owner = -1; - spin_unlock(&dev->_xmit_lock); + __netif_tx_unlock(&dev->tx_queue); +} + +static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) +{ + txq->xmit_lock_owner = -1; + spin_unlock_bh(&txq->_xmit_lock); } static inline void netif_tx_unlock_bh(struct net_device *dev) { - dev->xmit_lock_owner = -1; - spin_unlock_bh(&dev->_xmit_lock); + __netif_tx_unlock_bh(&dev->tx_queue); } -#define HARD_TX_LOCK(dev, cpu) { \ +#define HARD_TX_LOCK(dev, txq, cpu) { \ if ((dev->features & NETIF_F_LLTX) == 0) { \ - __netif_tx_lock(dev, cpu); \ + __netif_tx_lock(txq, cpu); \ } \ } -#define HARD_TX_UNLOCK(dev) { \ +#define HARD_TX_UNLOCK(dev, txq) { \ if ((dev->features & NETIF_F_LLTX) == 0) { \ - netif_tx_unlock(dev); \ + __netif_tx_unlock(txq); \ } \ } diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index b6e52c025fd8..8efa399823e3 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -627,6 +627,18 @@ static void vlan_dev_set_rx_mode(struct net_device *vlan_dev) */ static struct lock_class_key vlan_netdev_xmit_lock_key; +static void vlan_dev_set_lockdep_one(struct netdev_queue *txq, + int subclass) +{ + lockdep_set_class_and_subclass(&txq->_xmit_lock, + &vlan_netdev_xmit_lock_key, subclass); +} + +static void vlan_dev_set_lockdep_class(struct net_device *dev, int subclass) +{ + vlan_dev_set_lockdep_one(&dev->tx_queue, subclass); +} + static const struct header_ops vlan_header_ops = { .create = vlan_dev_hard_header, .rebuild = vlan_dev_rebuild_header, @@ -668,8 +680,7 @@ static int vlan_dev_init(struct net_device *dev) if (is_vlan_dev(real_dev)) subclass = 1; - lockdep_set_class_and_subclass(&dev->_xmit_lock, - &vlan_netdev_xmit_lock_key, subclass); + vlan_dev_set_lockdep_class(dev, subclass); return 0; } diff --git a/net/core/dev.c b/net/core/dev.c index 0218b0b9be80..a29a359b15d1 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -258,7 +258,7 @@ DEFINE_PER_CPU(struct softnet_data, softnet_data); #ifdef CONFIG_DEBUG_LOCK_ALLOC /* - * register_netdevice() inits dev->_xmit_lock and sets lockdep class + * register_netdevice() inits txq->_xmit_lock and sets lockdep class * according to dev->type */ static const unsigned short netdev_lock_type[] = @@ -1758,19 +1758,19 @@ gso: if (dev->flags & IFF_UP) { int cpu = smp_processor_id(); /* ok because BHs are off */ - if (dev->xmit_lock_owner != cpu) { + if (txq->xmit_lock_owner != cpu) { - HARD_TX_LOCK(dev, cpu); + HARD_TX_LOCK(dev, txq, cpu); if (!netif_queue_stopped(dev) && !netif_subqueue_stopped(dev, skb)) { rc = 0; if (!dev_hard_start_xmit(skb, dev)) { - HARD_TX_UNLOCK(dev); + HARD_TX_UNLOCK(dev, txq); goto out; } } - HARD_TX_UNLOCK(dev); + HARD_TX_UNLOCK(dev, txq); if (net_ratelimit()) printk(KERN_CRIT "Virtual device %s asks to " "queue packet!\n", dev->name); @@ -3761,6 +3761,20 @@ static void rollback_registered(struct net_device *dev) dev_put(dev); } +static void __netdev_init_queue_locks_one(struct netdev_queue *dev_queue, + struct net_device *dev) +{ + spin_lock_init(&dev_queue->_xmit_lock); + netdev_set_lockdep_class(&dev_queue->_xmit_lock, dev->type); + dev_queue->xmit_lock_owner = -1; +} + +static void netdev_init_queue_locks(struct net_device *dev) +{ + __netdev_init_queue_locks_one(&dev->tx_queue, dev); + __netdev_init_queue_locks_one(&dev->rx_queue, dev); +} + /** * register_netdevice - register a network device * @dev: device to register @@ -3795,9 +3809,7 @@ int register_netdevice(struct net_device *dev) BUG_ON(!dev_net(dev)); net = dev_net(dev); - spin_lock_init(&dev->_xmit_lock); - netdev_set_lockdep_class(&dev->_xmit_lock, dev->type); - dev->xmit_lock_owner = -1; + netdev_init_queue_locks(dev); dev->iflink = -1; diff --git a/net/netrom/af_netrom.c b/net/netrom/af_netrom.c index 74884f4a6255..819afc449e1e 100644 --- a/net/netrom/af_netrom.c +++ b/net/netrom/af_netrom.c @@ -74,6 +74,16 @@ static const struct proto_ops nr_proto_ops; */ static struct lock_class_key nr_netdev_xmit_lock_key; +static void nr_set_lockdep_one(struct netdev_queue *txq) +{ + lockdep_set_class(&txq->_xmit_lock, &nr_netdev_xmit_lock_key); +} + +static void nr_set_lockdep_key(struct net_device *dev) +{ + nr_set_lockdep_one(&dev->tx_queue); +} + /* * Socket removal during an interrupt is now safe. */ @@ -1430,7 +1440,7 @@ static int __init nr_proto_init(void) free_netdev(dev); goto fail; } - lockdep_set_class(&dev->_xmit_lock, &nr_netdev_xmit_lock_key); + nr_set_lockdep_key(dev); dev_nr[i] = dev; } diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index 46461a69cd0f..7dbbc0891623 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -75,6 +75,16 @@ ax25_address rose_callsign; */ static struct lock_class_key rose_netdev_xmit_lock_key; +static void rose_set_lockdep_one(struct netdev_queue *txq) +{ + lockdep_set_class(&txq->_xmit_lock, &rose_netdev_xmit_lock_key); +} + +static void rose_set_lockdep_key(struct net_device *dev) +{ + rose_set_lockdep_one(&dev->tx_queue); +} + /* * Convert a ROSE address into text. */ @@ -1576,7 +1586,7 @@ static int __init rose_proto_init(void) free_netdev(dev); goto fail; } - lockdep_set_class(&dev->_xmit_lock, &rose_netdev_xmit_lock_key); + rose_set_lockdep_key(dev); dev_rose[i] = dev; } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index fcc7533f0bcc..b6a36d394663 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -92,10 +92,9 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, struct netdev_queue *dev_queue, struct Qdisc *q) { - struct net_device *dev = dev_queue->dev; int ret; - if (unlikely(dev->xmit_lock_owner == smp_processor_id())) { + if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) { /* * Same CPU holding the lock. It may be a transient * configuration error, when hard_start_xmit() recurses. We @@ -105,7 +104,7 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, kfree_skb(skb); if (net_ratelimit()) printk(KERN_WARNING "Dead loop on netdevice %s, " - "fix it urgently!\n", dev->name); + "fix it urgently!\n", dev_queue->dev->name); ret = qdisc_qlen(q); } else { /* @@ -155,10 +154,10 @@ static inline int qdisc_restart(struct netdev_queue *txq) dev = txq->dev; - HARD_TX_LOCK(dev, smp_processor_id()); + HARD_TX_LOCK(dev, txq, smp_processor_id()); if (!netif_subqueue_stopped(dev, skb)) ret = dev_hard_start_xmit(skb, dev); - HARD_TX_UNLOCK(dev); + HARD_TX_UNLOCK(dev, txq); spin_lock(&txq->lock); q = txq->qdisc; -- cgit v1.2.3 From bc1d0411b804ad190cdadabac48a10067f17b9e6 Mon Sep 17 00:00:00 2001 From: Patrick McHardy Date: Mon, 14 Jul 2008 22:49:30 -0700 Subject: vlan: deliver packets received with VLAN acceleration to network taps When VLAN header stripping is used, packets currently bypass packet sockets (and other network taps) completely. For locally existing VLANs, they appear directly on the VLAN device, for unknown VLANs they are silently dropped. Add a new function netif_nit_deliver() to deliver incoming packets to all network interface taps and use it in __vlan_hwaccel_rx() to make VLAN packets visible on the underlying device. Signed-off-by: Patrick McHardy Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 + net/8021q/vlan_core.c | 4 ++++ net/core/dev.c | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index b54ec16dfbda..ba5c4639ea91 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1165,6 +1165,7 @@ extern int netif_rx(struct sk_buff *skb); extern int netif_rx_ni(struct sk_buff *skb); #define HAVE_NETIF_RECEIVE_SKB 1 extern int netif_receive_skb(struct sk_buff *skb); +extern void netif_nit_deliver(struct sk_buff *skb); extern int dev_valid_name(const char *name); extern int dev_ioctl(struct net *net, unsigned int cmd, void __user *); extern int dev_ethtool(struct net *net, struct ifreq *); diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c index 68df12d3664b..916061f681b6 100644 --- a/net/8021q/vlan_core.c +++ b/net/8021q/vlan_core.c @@ -14,6 +14,9 @@ int __vlan_hwaccel_rx(struct sk_buff *skb, struct vlan_group *grp, return NET_RX_DROP; } + skb->vlan_tci = vlan_tci; + netif_nit_deliver(skb); + skb->dev = vlan_group_get_device(grp, vlan_tci & VLAN_VID_MASK); if (skb->dev == NULL) { dev_kfree_skb_any(skb); @@ -22,6 +25,7 @@ int __vlan_hwaccel_rx(struct sk_buff *skb, struct vlan_group *grp, return NET_RX_SUCCESS; } skb->dev->last_rx = jiffies; + skb->vlan_tci = 0; stats = &skb->dev->stats; stats->rx_packets++; diff --git a/net/core/dev.c b/net/core/dev.c index a29a359b15d1..feaab4898a5b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2068,6 +2068,33 @@ out: } #endif +/* + * netif_nit_deliver - deliver received packets to network taps + * @skb: buffer + * + * This function is used to deliver incoming packets to network + * taps. It should be used when the normal netif_receive_skb path + * is bypassed, for example because of VLAN acceleration. + */ +void netif_nit_deliver(struct sk_buff *skb) +{ + struct packet_type *ptype; + + if (list_empty(&ptype_all)) + return; + + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + skb->mac_len = skb->network_header - skb->mac_header; + + rcu_read_lock(); + list_for_each_entry_rcu(ptype, &ptype_all, list) { + if (!ptype->dev || ptype->dev == skb->dev) + deliver_skb(skb, ptype, skb->dev); + } + rcu_read_unlock(); +} + /** * netif_receive_skb - process receive buffer from network * @skb: buffer to process -- cgit v1.2.3 From f1f28aa3510ddb84c966bac65611bb866c77a092 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 15 Jul 2008 00:08:33 -0700 Subject: netdev: Add addr_list_lock to struct net_device. This will be used to protect the per-device unicast and multicast address lists, as well as the callbacks into the drivers which configure such state such as ->set_rx_mode() and ->set_multicast_list(). Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 + net/core/dev.c | 1 + 2 files changed, 2 insertions(+) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index ba5c4639ea91..fd0365219181 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -609,6 +609,7 @@ struct net_device unsigned char addr_len; /* hardware address length */ unsigned short dev_id; /* for shared network cards */ + spinlock_t addr_list_lock; struct dev_addr_list *uc_list; /* Secondary unicast mac addresses */ int uc_count; /* Number of installed ucasts */ int uc_promisc; diff --git a/net/core/dev.c b/net/core/dev.c index feaab4898a5b..d933d1bfa6fa 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3836,6 +3836,7 @@ int register_netdevice(struct net_device *dev) BUG_ON(!dev_net(dev)); net = dev_net(dev); + spin_lock_init(&dev->addr_list_lock); netdev_init_queue_locks(dev); dev->iflink = -1; -- cgit v1.2.3 From e308a5d806c852f56590ffdd3834d0df0cbed8d7 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 15 Jul 2008 00:13:44 -0700 Subject: netdev: Add netdev->addr_list_lock protection. Add netif_addr_{lock,unlock}{,_bh}() helpers. Use them to protect operations that operate on or read the network device unicast and multicast address lists. Also use them in cases where the code simply wants to block calls into the driver's ->set_rx_mode() and ->set_multicast_list() methods. Signed-off-by: David S. Miller --- drivers/infiniband/ulp/ipoib/ipoib_multicast.c | 2 ++ drivers/media/dvb/dvb-core/dvb_net.c | 2 ++ drivers/net/bonding/bond_main.c | 8 ++++++++ drivers/net/forcedeth.c | 16 ++++++++++++++++ drivers/net/hamradio/6pack.c | 2 ++ drivers/net/hamradio/mkiss.c | 2 ++ drivers/net/ibm_newemac/core.c | 4 ++++ drivers/net/sfc/efx.c | 2 ++ drivers/net/wireless/libertas/main.c | 2 ++ include/linux/netdevice.h | 20 ++++++++++++++++++++ net/core/dev.c | 14 ++++++++++++++ net/core/dev_mcast.c | 12 ++++++++++++ net/mac80211/main.c | 4 ++++ net/mac80211/mlme.c | 4 ++++ 14 files changed, 94 insertions(+) (limited to 'net/core/dev.c') diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index 3f663fb852c1..261ab7150431 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -775,6 +775,7 @@ void ipoib_mcast_restart_task(struct work_struct *work) local_irq_save(flags); netif_tx_lock(dev); + netif_addr_lock(dev); spin_lock(&priv->lock); /* @@ -851,6 +852,7 @@ void ipoib_mcast_restart_task(struct work_struct *work) } spin_unlock(&priv->lock); + netif_addr_unlock(dev); netif_tx_unlock(dev); local_irq_restore(flags); diff --git a/drivers/media/dvb/dvb-core/dvb_net.c b/drivers/media/dvb/dvb-core/dvb_net.c index c2334aef4143..809d18c663bc 100644 --- a/drivers/media/dvb/dvb-core/dvb_net.c +++ b/drivers/media/dvb/dvb-core/dvb_net.c @@ -1134,6 +1134,7 @@ static void wq_set_multicast_list (struct work_struct *work) dvb_net_feed_stop(dev); priv->rx_mode = RX_MODE_UNI; netif_tx_lock_bh(dev); + netif_addr_lock(dev); if (dev->flags & IFF_PROMISC) { dprintk("%s: promiscuous mode\n", dev->name); @@ -1158,6 +1159,7 @@ static void wq_set_multicast_list (struct work_struct *work) } } + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); dvb_net_feed_start(dev); } diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 8ae7ff313218..ea71abd6f728 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -1568,10 +1568,12 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev) } netif_tx_lock_bh(bond_dev); + netif_addr_lock(bond_dev); /* upload master's mc_list to new slave */ for (dmi = bond_dev->mc_list; dmi; dmi = dmi->next) { dev_mc_add (slave_dev, dmi->dmi_addr, dmi->dmi_addrlen, 0); } + netif_addr_unlock(bond_dev); netif_tx_unlock_bh(bond_dev); } @@ -1937,7 +1939,9 @@ int bond_release(struct net_device *bond_dev, struct net_device *slave_dev) /* flush master's mc_list from slave */ netif_tx_lock_bh(bond_dev); + netif_addr_lock(bond_dev); bond_mc_list_flush(bond_dev, slave_dev); + netif_addr_unlock(bond_dev); netif_tx_unlock_bh(bond_dev); } @@ -2060,7 +2064,9 @@ static int bond_release_all(struct net_device *bond_dev) /* flush master's mc_list from slave */ netif_tx_lock_bh(bond_dev); + netif_addr_lock(bond_dev); bond_mc_list_flush(bond_dev, slave_dev); + netif_addr_unlock(bond_dev); netif_tx_unlock_bh(bond_dev); } @@ -4674,7 +4680,9 @@ static void bond_free_all(void) bond_work_cancel_all(bond); netif_tx_lock_bh(bond_dev); + netif_addr_lock(bond_dev); bond_mc_list_destroy(bond); + netif_addr_unlock(bond_dev); netif_tx_unlock_bh(bond_dev); /* Release the bonded slaves */ bond_release_all(bond_dev); diff --git a/drivers/net/forcedeth.c b/drivers/net/forcedeth.c index 786d668c612e..4ed89fa9ae46 100644 --- a/drivers/net/forcedeth.c +++ b/drivers/net/forcedeth.c @@ -2831,6 +2831,7 @@ static int nv_change_mtu(struct net_device *dev, int new_mtu) */ nv_disable_irq(dev); netif_tx_lock_bh(dev); + netif_addr_lock(dev); spin_lock(&np->lock); /* stop engines */ nv_stop_rxtx(dev); @@ -2855,6 +2856,7 @@ static int nv_change_mtu(struct net_device *dev, int new_mtu) /* restart rx engine */ nv_start_rxtx(dev); spin_unlock(&np->lock); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); nv_enable_irq(dev); } @@ -2891,6 +2893,7 @@ static int nv_set_mac_address(struct net_device *dev, void *addr) if (netif_running(dev)) { netif_tx_lock_bh(dev); + netif_addr_lock(dev); spin_lock_irq(&np->lock); /* stop rx engine */ @@ -2902,6 +2905,7 @@ static int nv_set_mac_address(struct net_device *dev, void *addr) /* restart rx engine */ nv_start_rx(dev); spin_unlock_irq(&np->lock); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); } else { nv_copy_mac_to_hw(dev); @@ -3971,6 +3975,7 @@ static void nv_do_nic_poll(unsigned long data) printk(KERN_INFO "forcedeth: MAC in recoverable error state\n"); if (netif_running(dev)) { netif_tx_lock_bh(dev); + netif_addr_lock(dev); spin_lock(&np->lock); /* stop engines */ nv_stop_rxtx(dev); @@ -3995,6 +4000,7 @@ static void nv_do_nic_poll(unsigned long data) /* restart rx engine */ nv_start_rxtx(dev); spin_unlock(&np->lock); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); } } @@ -4202,6 +4208,7 @@ static int nv_set_settings(struct net_device *dev, struct ethtool_cmd *ecmd) nv_disable_irq(dev); netif_tx_lock_bh(dev); + netif_addr_lock(dev); /* with plain spinlock lockdep complains */ spin_lock_irqsave(&np->lock, flags); /* stop engines */ @@ -4215,6 +4222,7 @@ static int nv_set_settings(struct net_device *dev, struct ethtool_cmd *ecmd) */ nv_stop_rxtx(dev); spin_unlock_irqrestore(&np->lock, flags); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); } @@ -4360,10 +4368,12 @@ static int nv_nway_reset(struct net_device *dev) if (netif_running(dev)) { nv_disable_irq(dev); netif_tx_lock_bh(dev); + netif_addr_lock(dev); spin_lock(&np->lock); /* stop engines */ nv_stop_rxtx(dev); spin_unlock(&np->lock); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); printk(KERN_INFO "%s: link down.\n", dev->name); } @@ -4471,6 +4481,7 @@ static int nv_set_ringparam(struct net_device *dev, struct ethtool_ringparam* ri if (netif_running(dev)) { nv_disable_irq(dev); netif_tx_lock_bh(dev); + netif_addr_lock(dev); spin_lock(&np->lock); /* stop engines */ nv_stop_rxtx(dev); @@ -4519,6 +4530,7 @@ static int nv_set_ringparam(struct net_device *dev, struct ethtool_ringparam* ri /* restart engines */ nv_start_rxtx(dev); spin_unlock(&np->lock); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); nv_enable_irq(dev); } @@ -4556,10 +4568,12 @@ static int nv_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam* if (netif_running(dev)) { nv_disable_irq(dev); netif_tx_lock_bh(dev); + netif_addr_lock(dev); spin_lock(&np->lock); /* stop engines */ nv_stop_rxtx(dev); spin_unlock(&np->lock); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); } @@ -4946,6 +4960,7 @@ static void nv_self_test(struct net_device *dev, struct ethtool_test *test, u64 napi_disable(&np->napi); #endif netif_tx_lock_bh(dev); + netif_addr_lock(dev); spin_lock_irq(&np->lock); nv_disable_hw_interrupts(dev, np->irqmask); if (!(np->msi_flags & NV_MSI_X_ENABLED)) { @@ -4959,6 +4974,7 @@ static void nv_self_test(struct net_device *dev, struct ethtool_test *test, u64 /* drain rx queue */ nv_drain_rxtx(dev); spin_unlock_irq(&np->lock); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); } diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c index 06ad9f302b5a..ffc937f5d15d 100644 --- a/drivers/net/hamradio/6pack.c +++ b/drivers/net/hamradio/6pack.c @@ -300,7 +300,9 @@ static int sp_set_mac_address(struct net_device *dev, void *addr) struct sockaddr_ax25 *sa = addr; netif_tx_lock_bh(dev); + netif_addr_lock(dev); memcpy(dev->dev_addr, &sa->sax25_call, AX25_ADDR_LEN); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return 0; diff --git a/drivers/net/hamradio/mkiss.c b/drivers/net/hamradio/mkiss.c index 65166035aca0..b8740e6a5cec 100644 --- a/drivers/net/hamradio/mkiss.c +++ b/drivers/net/hamradio/mkiss.c @@ -356,7 +356,9 @@ static int ax_set_mac_address(struct net_device *dev, void *addr) struct sockaddr_ax25 *sa = addr; netif_tx_lock_bh(dev); + netif_addr_lock(dev); memcpy(dev->dev_addr, &sa->sax25_call, AX25_ADDR_LEN); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return 0; diff --git a/drivers/net/ibm_newemac/core.c b/drivers/net/ibm_newemac/core.c index babc79ad490b..9ca57d365599 100644 --- a/drivers/net/ibm_newemac/core.c +++ b/drivers/net/ibm_newemac/core.c @@ -295,7 +295,9 @@ static void emac_rx_disable(struct emac_instance *dev) static inline void emac_netif_stop(struct emac_instance *dev) { netif_tx_lock_bh(dev->ndev); + netif_addr_lock(dev->ndev); dev->no_mcast = 1; + netif_addr_unlock(dev->ndev); netif_tx_unlock_bh(dev->ndev); dev->ndev->trans_start = jiffies; /* prevent tx timeout */ mal_poll_disable(dev->mal, &dev->commac); @@ -305,9 +307,11 @@ static inline void emac_netif_stop(struct emac_instance *dev) static inline void emac_netif_start(struct emac_instance *dev) { netif_tx_lock_bh(dev->ndev); + netif_addr_lock(dev->ndev); dev->no_mcast = 0; if (dev->mcast_pending && netif_running(dev->ndev)) __emac_set_multicast_list(dev); + netif_addr_unlock(dev->ndev); netif_tx_unlock_bh(dev->ndev); netif_wake_queue(dev->ndev); diff --git a/drivers/net/sfc/efx.c b/drivers/net/sfc/efx.c index 74265d8553b8..e1257e556e48 100644 --- a/drivers/net/sfc/efx.c +++ b/drivers/net/sfc/efx.c @@ -697,6 +697,8 @@ static void efx_stop_port(struct efx_nic *efx) /* Serialise against efx_set_multicast_list() */ if (efx_dev_registered(efx)) { netif_tx_lock_bh(efx->net_dev); + netif_addr_lock(efx->net_dev); + netif_addr_unlock(efx->net_dev); netif_tx_unlock_bh(efx->net_dev); } } diff --git a/drivers/net/wireless/libertas/main.c b/drivers/net/wireless/libertas/main.c index abd6d9ed8f4b..42e9b2771eab 100644 --- a/drivers/net/wireless/libertas/main.c +++ b/drivers/net/wireless/libertas/main.c @@ -594,6 +594,7 @@ static int lbs_add_mcast_addrs(struct cmd_ds_mac_multicast_adr *cmd, return nr_addrs; netif_tx_lock_bh(dev); + netif_addr_lock(dev); for (mc_list = dev->mc_list; mc_list; mc_list = mc_list->next) { if (mac_in_list(cmd->maclist, nr_addrs, mc_list->dmi_addr)) { lbs_deb_net("mcast address %s:%s skipped\n", dev->name, @@ -608,6 +609,7 @@ static int lbs_add_mcast_addrs(struct cmd_ds_mac_multicast_adr *cmd, print_mac(mac, mc_list->dmi_addr)); i++; } + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); if (mc_list) return -EOVERFLOW; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index fd0365219181..570cf7affa72 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1498,6 +1498,26 @@ static inline void netif_tx_disable(struct net_device *dev) netif_tx_unlock_bh(dev); } +static inline void netif_addr_lock(struct net_device *dev) +{ + spin_lock(&dev->addr_list_lock); +} + +static inline void netif_addr_lock_bh(struct net_device *dev) +{ + spin_lock_bh(&dev->addr_list_lock); +} + +static inline void netif_addr_unlock(struct net_device *dev) +{ + spin_unlock(&dev->addr_list_lock); +} + +static inline void netif_addr_unlock_bh(struct net_device *dev) +{ + spin_unlock_bh(&dev->addr_list_lock); +} + /* These functions live elsewhere (drivers/net/net_init.c, but related) */ extern void ether_setup(struct net_device *dev); diff --git a/net/core/dev.c b/net/core/dev.c index d933d1bfa6fa..ef1502d71f25 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2982,7 +2982,9 @@ void __dev_set_rx_mode(struct net_device *dev) void dev_set_rx_mode(struct net_device *dev) { netif_tx_lock_bh(dev); + netif_addr_lock(dev); __dev_set_rx_mode(dev); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); } @@ -3062,9 +3064,11 @@ int dev_unicast_delete(struct net_device *dev, void *addr, int alen) ASSERT_RTNL(); netif_tx_lock_bh(dev); + netif_addr_lock(dev); err = __dev_addr_delete(&dev->uc_list, &dev->uc_count, addr, alen, 0); if (!err) __dev_set_rx_mode(dev); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return err; } @@ -3088,9 +3092,11 @@ int dev_unicast_add(struct net_device *dev, void *addr, int alen) ASSERT_RTNL(); netif_tx_lock_bh(dev); + netif_addr_lock(dev); err = __dev_addr_add(&dev->uc_list, &dev->uc_count, addr, alen, 0); if (!err) __dev_set_rx_mode(dev); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return err; } @@ -3159,10 +3165,12 @@ int dev_unicast_sync(struct net_device *to, struct net_device *from) int err = 0; netif_tx_lock_bh(to); + netif_addr_lock(to); err = __dev_addr_sync(&to->uc_list, &to->uc_count, &from->uc_list, &from->uc_count); if (!err) __dev_set_rx_mode(to); + netif_addr_unlock(to); netif_tx_unlock_bh(to); return err; } @@ -3180,13 +3188,17 @@ EXPORT_SYMBOL(dev_unicast_sync); void dev_unicast_unsync(struct net_device *to, struct net_device *from) { netif_tx_lock_bh(from); + netif_addr_lock(from); netif_tx_lock_bh(to); + netif_addr_lock(to); __dev_addr_unsync(&to->uc_list, &to->uc_count, &from->uc_list, &from->uc_count); __dev_set_rx_mode(to); + netif_addr_unlock(to); netif_tx_unlock_bh(to); + netif_addr_unlock(from); netif_tx_unlock_bh(from); } EXPORT_SYMBOL(dev_unicast_unsync); @@ -3208,6 +3220,7 @@ static void __dev_addr_discard(struct dev_addr_list **list) static void dev_addr_discard(struct net_device *dev) { netif_tx_lock_bh(dev); + netif_addr_lock(dev); __dev_addr_discard(&dev->uc_list); dev->uc_count = 0; @@ -3215,6 +3228,7 @@ static void dev_addr_discard(struct net_device *dev) __dev_addr_discard(&dev->mc_list); dev->mc_count = 0; + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); } diff --git a/net/core/dev_mcast.c b/net/core/dev_mcast.c index f8a3455f4493..b6b2a129971a 100644 --- a/net/core/dev_mcast.c +++ b/net/core/dev_mcast.c @@ -73,6 +73,7 @@ int dev_mc_delete(struct net_device *dev, void *addr, int alen, int glbl) int err; netif_tx_lock_bh(dev); + netif_addr_lock(dev); err = __dev_addr_delete(&dev->mc_list, &dev->mc_count, addr, alen, glbl); if (!err) { @@ -83,6 +84,7 @@ int dev_mc_delete(struct net_device *dev, void *addr, int alen, int glbl) __dev_set_rx_mode(dev); } + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return err; } @@ -96,9 +98,11 @@ int dev_mc_add(struct net_device *dev, void *addr, int alen, int glbl) int err; netif_tx_lock_bh(dev); + netif_addr_lock(dev); err = __dev_addr_add(&dev->mc_list, &dev->mc_count, addr, alen, glbl); if (!err) __dev_set_rx_mode(dev); + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return err; } @@ -120,10 +124,12 @@ int dev_mc_sync(struct net_device *to, struct net_device *from) int err = 0; netif_tx_lock_bh(to); + netif_addr_lock(to); err = __dev_addr_sync(&to->mc_list, &to->mc_count, &from->mc_list, &from->mc_count); if (!err) __dev_set_rx_mode(to); + netif_addr_unlock(to); netif_tx_unlock_bh(to); return err; @@ -144,13 +150,17 @@ EXPORT_SYMBOL(dev_mc_sync); void dev_mc_unsync(struct net_device *to, struct net_device *from) { netif_tx_lock_bh(from); + netif_addr_lock(from); netif_tx_lock_bh(to); + netif_addr_lock(to); __dev_addr_unsync(&to->mc_list, &to->mc_count, &from->mc_list, &from->mc_count); __dev_set_rx_mode(to); + netif_addr_unlock(to); netif_tx_unlock_bh(to); + netif_addr_unlock(from); netif_tx_unlock_bh(from); } EXPORT_SYMBOL(dev_mc_unsync); @@ -165,6 +175,7 @@ static int dev_mc_seq_show(struct seq_file *seq, void *v) return 0; netif_tx_lock_bh(dev); + netif_addr_lock(dev); for (m = dev->mc_list; m; m = m->next) { int i; @@ -176,6 +187,7 @@ static int dev_mc_seq_show(struct seq_file *seq, void *v) seq_putc(seq, '\n'); } + netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return 0; } diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 36859e794928..095b7d928d64 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -292,7 +292,9 @@ static int ieee80211_open(struct net_device *dev) local->fif_other_bss++; netif_tx_lock_bh(local->mdev); + netif_addr_lock(local->mdev); ieee80211_configure_filter(local); + netif_addr_unlock(local->mdev); netif_tx_unlock_bh(local->mdev); break; case IEEE80211_IF_TYPE_STA: @@ -491,7 +493,9 @@ static int ieee80211_stop(struct net_device *dev) local->fif_other_bss--; netif_tx_lock_bh(local->mdev); + netif_addr_lock(local->mdev); ieee80211_configure_filter(local); + netif_addr_unlock(local->mdev); netif_tx_unlock_bh(local->mdev); break; case IEEE80211_IF_TYPE_MESH_POINT: diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 8f51375317dd..1232ba25e1e9 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -3869,6 +3869,7 @@ void ieee80211_scan_completed(struct ieee80211_hw *hw) netif_tx_lock_bh(local->mdev); + netif_addr_lock(local->mdev); local->filter_flags &= ~FIF_BCN_PRBRESP_PROMISC; local->ops->configure_filter(local_to_hw(local), FIF_BCN_PRBRESP_PROMISC, @@ -3876,6 +3877,7 @@ void ieee80211_scan_completed(struct ieee80211_hw *hw) local->mdev->mc_count, local->mdev->mc_list); + netif_addr_unlock(local->mdev); netif_tx_unlock_bh(local->mdev); rcu_read_lock(); @@ -4063,12 +4065,14 @@ static int ieee80211_sta_start_scan(struct net_device *dev, local->scan_dev = dev; netif_tx_lock_bh(local->mdev); + netif_addr_lock(local->mdev); local->filter_flags |= FIF_BCN_PRBRESP_PROMISC; local->ops->configure_filter(local_to_hw(local), FIF_BCN_PRBRESP_PROMISC, &local->filter_flags, local->mdev->mc_count, local->mdev->mc_list); + netif_addr_unlock(local->mdev); netif_tx_unlock_bh(local->mdev); /* TODO: start scan as soon as all nullfunc frames are ACKed */ -- cgit v1.2.3 From b9e40857682ecfc5bcd0356a23ff409883ffb982 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 15 Jul 2008 00:15:08 -0700 Subject: netdev: Do not use TX lock to protect address lists. Now that we have a specific lock to protect the network device unicast and multicast lists, remove extraneous grabs of the TX lock in cases where the code only needs address list protection. Signed-off-by: David S. Miller --- drivers/infiniband/ulp/ipoib/ipoib_multicast.c | 2 -- drivers/media/dvb/dvb-core/dvb_net.c | 6 ++-- drivers/net/bonding/bond_main.c | 24 ++++++---------- drivers/net/sfc/efx.c | 6 ++-- drivers/net/wireless/libertas/main.c | 6 ++-- net/core/dev.c | 38 ++++++++------------------ net/core/dev_mcast.c | 32 +++++++--------------- net/mac80211/main.c | 12 +++----- net/mac80211/mlme.c | 6 ++-- 9 files changed, 42 insertions(+), 90 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index 261ab7150431..cd2fb955040f 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -774,7 +774,6 @@ void ipoib_mcast_restart_task(struct work_struct *work) ipoib_mcast_stop_thread(dev, 0); local_irq_save(flags); - netif_tx_lock(dev); netif_addr_lock(dev); spin_lock(&priv->lock); @@ -853,7 +852,6 @@ void ipoib_mcast_restart_task(struct work_struct *work) spin_unlock(&priv->lock); netif_addr_unlock(dev); - netif_tx_unlock(dev); local_irq_restore(flags); /* We have to cancel outside of the spinlock */ diff --git a/drivers/media/dvb/dvb-core/dvb_net.c b/drivers/media/dvb/dvb-core/dvb_net.c index 809d18c663bc..c2c033722a93 100644 --- a/drivers/media/dvb/dvb-core/dvb_net.c +++ b/drivers/media/dvb/dvb-core/dvb_net.c @@ -1133,8 +1133,7 @@ static void wq_set_multicast_list (struct work_struct *work) dvb_net_feed_stop(dev); priv->rx_mode = RX_MODE_UNI; - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); if (dev->flags & IFF_PROMISC) { dprintk("%s: promiscuous mode\n", dev->name); @@ -1159,8 +1158,7 @@ static void wq_set_multicast_list (struct work_struct *work) } } - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); dvb_net_feed_start(dev); } diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index ea71abd6f728..fd87dbe7999a 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -1567,14 +1567,12 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev) goto err_close; } - netif_tx_lock_bh(bond_dev); - netif_addr_lock(bond_dev); + netif_addr_lock_bh(bond_dev); /* upload master's mc_list to new slave */ for (dmi = bond_dev->mc_list; dmi; dmi = dmi->next) { dev_mc_add (slave_dev, dmi->dmi_addr, dmi->dmi_addrlen, 0); } - netif_addr_unlock(bond_dev); - netif_tx_unlock_bh(bond_dev); + netif_addr_unlock_bh(bond_dev); } if (bond->params.mode == BOND_MODE_8023AD) { @@ -1938,11 +1936,9 @@ int bond_release(struct net_device *bond_dev, struct net_device *slave_dev) } /* flush master's mc_list from slave */ - netif_tx_lock_bh(bond_dev); - netif_addr_lock(bond_dev); + netif_addr_lock_bh(bond_dev); bond_mc_list_flush(bond_dev, slave_dev); - netif_addr_unlock(bond_dev); - netif_tx_unlock_bh(bond_dev); + netif_addr_unlock_bh(bond_dev); } netdev_set_master(slave_dev, NULL); @@ -2063,11 +2059,9 @@ static int bond_release_all(struct net_device *bond_dev) } /* flush master's mc_list from slave */ - netif_tx_lock_bh(bond_dev); - netif_addr_lock(bond_dev); + netif_addr_lock_bh(bond_dev); bond_mc_list_flush(bond_dev, slave_dev); - netif_addr_unlock(bond_dev); - netif_tx_unlock_bh(bond_dev); + netif_addr_unlock_bh(bond_dev); } netdev_set_master(slave_dev, NULL); @@ -4679,11 +4673,9 @@ static void bond_free_all(void) struct net_device *bond_dev = bond->dev; bond_work_cancel_all(bond); - netif_tx_lock_bh(bond_dev); - netif_addr_lock(bond_dev); + netif_addr_lock_bh(bond_dev); bond_mc_list_destroy(bond); - netif_addr_unlock(bond_dev); - netif_tx_unlock_bh(bond_dev); + netif_addr_unlock_bh(bond_dev); /* Release the bonded slaves */ bond_release_all(bond_dev); bond_destroy(bond); diff --git a/drivers/net/sfc/efx.c b/drivers/net/sfc/efx.c index e1257e556e48..7b2015f9e469 100644 --- a/drivers/net/sfc/efx.c +++ b/drivers/net/sfc/efx.c @@ -696,10 +696,8 @@ static void efx_stop_port(struct efx_nic *efx) /* Serialise against efx_set_multicast_list() */ if (efx_dev_registered(efx)) { - netif_tx_lock_bh(efx->net_dev); - netif_addr_lock(efx->net_dev); - netif_addr_unlock(efx->net_dev); - netif_tx_unlock_bh(efx->net_dev); + netif_addr_lock_bh(efx->net_dev); + netif_addr_unlock_bh(efx->net_dev); } } diff --git a/drivers/net/wireless/libertas/main.c b/drivers/net/wireless/libertas/main.c index 42e9b2771eab..14d5d61cec4c 100644 --- a/drivers/net/wireless/libertas/main.c +++ b/drivers/net/wireless/libertas/main.c @@ -593,8 +593,7 @@ static int lbs_add_mcast_addrs(struct cmd_ds_mac_multicast_adr *cmd, if ((dev->flags & (IFF_UP|IFF_MULTICAST)) != (IFF_UP|IFF_MULTICAST)) return nr_addrs; - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); for (mc_list = dev->mc_list; mc_list; mc_list = mc_list->next) { if (mac_in_list(cmd->maclist, nr_addrs, mc_list->dmi_addr)) { lbs_deb_net("mcast address %s:%s skipped\n", dev->name, @@ -609,8 +608,7 @@ static int lbs_add_mcast_addrs(struct cmd_ds_mac_multicast_adr *cmd, print_mac(mac, mc_list->dmi_addr)); i++; } - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); if (mc_list) return -EOVERFLOW; diff --git a/net/core/dev.c b/net/core/dev.c index ef1502d71f25..9b49f74a9820 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2981,11 +2981,9 @@ void __dev_set_rx_mode(struct net_device *dev) void dev_set_rx_mode(struct net_device *dev) { - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); __dev_set_rx_mode(dev); - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); } int __dev_addr_delete(struct dev_addr_list **list, int *count, @@ -3063,13 +3061,11 @@ int dev_unicast_delete(struct net_device *dev, void *addr, int alen) ASSERT_RTNL(); - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); err = __dev_addr_delete(&dev->uc_list, &dev->uc_count, addr, alen, 0); if (!err) __dev_set_rx_mode(dev); - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); return err; } EXPORT_SYMBOL(dev_unicast_delete); @@ -3091,13 +3087,11 @@ int dev_unicast_add(struct net_device *dev, void *addr, int alen) ASSERT_RTNL(); - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); err = __dev_addr_add(&dev->uc_list, &dev->uc_count, addr, alen, 0); if (!err) __dev_set_rx_mode(dev); - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); return err; } EXPORT_SYMBOL(dev_unicast_add); @@ -3164,14 +3158,12 @@ int dev_unicast_sync(struct net_device *to, struct net_device *from) { int err = 0; - netif_tx_lock_bh(to); - netif_addr_lock(to); + netif_addr_lock_bh(to); err = __dev_addr_sync(&to->uc_list, &to->uc_count, &from->uc_list, &from->uc_count); if (!err) __dev_set_rx_mode(to); - netif_addr_unlock(to); - netif_tx_unlock_bh(to); + netif_addr_unlock_bh(to); return err; } EXPORT_SYMBOL(dev_unicast_sync); @@ -3187,9 +3179,7 @@ EXPORT_SYMBOL(dev_unicast_sync); */ void dev_unicast_unsync(struct net_device *to, struct net_device *from) { - netif_tx_lock_bh(from); - netif_addr_lock(from); - netif_tx_lock_bh(to); + netif_addr_lock_bh(from); netif_addr_lock(to); __dev_addr_unsync(&to->uc_list, &to->uc_count, @@ -3197,9 +3187,7 @@ void dev_unicast_unsync(struct net_device *to, struct net_device *from) __dev_set_rx_mode(to); netif_addr_unlock(to); - netif_tx_unlock_bh(to); - netif_addr_unlock(from); - netif_tx_unlock_bh(from); + netif_addr_unlock_bh(from); } EXPORT_SYMBOL(dev_unicast_unsync); @@ -3219,8 +3207,7 @@ static void __dev_addr_discard(struct dev_addr_list **list) static void dev_addr_discard(struct net_device *dev) { - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); __dev_addr_discard(&dev->uc_list); dev->uc_count = 0; @@ -3228,8 +3215,7 @@ static void dev_addr_discard(struct net_device *dev) __dev_addr_discard(&dev->mc_list); dev->mc_count = 0; - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); } unsigned dev_get_flags(const struct net_device *dev) diff --git a/net/core/dev_mcast.c b/net/core/dev_mcast.c index b6b2a129971a..5402b3b38e0d 100644 --- a/net/core/dev_mcast.c +++ b/net/core/dev_mcast.c @@ -72,8 +72,7 @@ int dev_mc_delete(struct net_device *dev, void *addr, int alen, int glbl) { int err; - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); err = __dev_addr_delete(&dev->mc_list, &dev->mc_count, addr, alen, glbl); if (!err) { @@ -84,8 +83,7 @@ int dev_mc_delete(struct net_device *dev, void *addr, int alen, int glbl) __dev_set_rx_mode(dev); } - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); return err; } @@ -97,13 +95,11 @@ int dev_mc_add(struct net_device *dev, void *addr, int alen, int glbl) { int err; - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); err = __dev_addr_add(&dev->mc_list, &dev->mc_count, addr, alen, glbl); if (!err) __dev_set_rx_mode(dev); - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); return err; } @@ -123,14 +119,12 @@ int dev_mc_sync(struct net_device *to, struct net_device *from) { int err = 0; - netif_tx_lock_bh(to); - netif_addr_lock(to); + netif_addr_lock_bh(to); err = __dev_addr_sync(&to->mc_list, &to->mc_count, &from->mc_list, &from->mc_count); if (!err) __dev_set_rx_mode(to); - netif_addr_unlock(to); - netif_tx_unlock_bh(to); + netif_addr_unlock_bh(to); return err; } @@ -149,9 +143,7 @@ EXPORT_SYMBOL(dev_mc_sync); */ void dev_mc_unsync(struct net_device *to, struct net_device *from) { - netif_tx_lock_bh(from); - netif_addr_lock(from); - netif_tx_lock_bh(to); + netif_addr_lock_bh(from); netif_addr_lock(to); __dev_addr_unsync(&to->mc_list, &to->mc_count, @@ -159,9 +151,7 @@ void dev_mc_unsync(struct net_device *to, struct net_device *from) __dev_set_rx_mode(to); netif_addr_unlock(to); - netif_tx_unlock_bh(to); - netif_addr_unlock(from); - netif_tx_unlock_bh(from); + netif_addr_unlock_bh(from); } EXPORT_SYMBOL(dev_mc_unsync); @@ -174,8 +164,7 @@ static int dev_mc_seq_show(struct seq_file *seq, void *v) if (v == SEQ_START_TOKEN) return 0; - netif_tx_lock_bh(dev); - netif_addr_lock(dev); + netif_addr_lock_bh(dev); for (m = dev->mc_list; m; m = m->next) { int i; @@ -187,8 +176,7 @@ static int dev_mc_seq_show(struct seq_file *seq, void *v) seq_putc(seq, '\n'); } - netif_addr_unlock(dev); - netif_tx_unlock_bh(dev); + netif_addr_unlock_bh(dev); return 0; } diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 095b7d928d64..af0056e7e5b3 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -291,11 +291,9 @@ static int ieee80211_open(struct net_device *dev) if (sdata->u.mntr_flags & MONITOR_FLAG_OTHER_BSS) local->fif_other_bss++; - netif_tx_lock_bh(local->mdev); - netif_addr_lock(local->mdev); + netif_addr_lock_bh(local->mdev); ieee80211_configure_filter(local); - netif_addr_unlock(local->mdev); - netif_tx_unlock_bh(local->mdev); + netif_addr_unlock_bh(local->mdev); break; case IEEE80211_IF_TYPE_STA: case IEEE80211_IF_TYPE_IBSS: @@ -492,11 +490,9 @@ static int ieee80211_stop(struct net_device *dev) if (sdata->u.mntr_flags & MONITOR_FLAG_OTHER_BSS) local->fif_other_bss--; - netif_tx_lock_bh(local->mdev); - netif_addr_lock(local->mdev); + netif_addr_lock_bh(local->mdev); ieee80211_configure_filter(local); - netif_addr_unlock(local->mdev); - netif_tx_unlock_bh(local->mdev); + netif_addr_unlock_bh(local->mdev); break; case IEEE80211_IF_TYPE_MESH_POINT: case IEEE80211_IF_TYPE_STA: diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 1232ba25e1e9..d7c371e36bf0 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -4064,16 +4064,14 @@ static int ieee80211_sta_start_scan(struct net_device *dev, local->scan_band = IEEE80211_BAND_2GHZ; local->scan_dev = dev; - netif_tx_lock_bh(local->mdev); - netif_addr_lock(local->mdev); + netif_addr_lock_bh(local->mdev); local->filter_flags |= FIF_BCN_PRBRESP_PROMISC; local->ops->configure_filter(local_to_hw(local), FIF_BCN_PRBRESP_PROMISC, &local->filter_flags, local->mdev->mc_count, local->mdev->mc_list); - netif_addr_unlock(local->mdev); - netif_tx_unlock_bh(local->mdev); + netif_addr_unlock_bh(local->mdev); /* TODO: start scan as soon as all nullfunc frames are ACKed */ queue_delayed_work(local->hw.workqueue, &local->scan_work, -- cgit v1.2.3 From e8a0464cc950972824e2e128028ae3db666ec1ed Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Thu, 17 Jul 2008 00:34:19 -0700 Subject: netdev: Allocate multiple queues for TX. alloc_netdev_mq() now allocates an array of netdev_queue structures for TX, based upon the queue_count argument. Furthermore, all accesses to the TX queues are now vectored through the netdev_get_tx_queue() and netdev_for_each_tx_queue() interfaces. This makes it easy to grep the tree for all things that want to get to a TX queue of a net device. Problem spots which are not really multiqueue aware yet, and only work with one queue, can easily be spotted by grepping for all netdev_get_tx_queue() calls that pass in a zero index. Signed-off-by: David S. Miller --- drivers/net/bonding/bond_main.c | 6 +- drivers/net/hamradio/bpqether.c | 6 +- drivers/net/ifb.c | 12 ++- drivers/net/macvlan.c | 6 +- drivers/net/wireless/hostap/hostap_hw.c | 6 +- include/linux/netdevice.h | 69 ++++++++----- include/net/sch_generic.h | 37 +++++-- net/8021q/vlan_dev.c | 10 +- net/core/dev.c | 40 +++++-- net/core/rtnetlink.c | 2 +- net/mac80211/main.c | 4 +- net/mac80211/wme.c | 12 +-- net/netrom/af_netrom.c | 6 +- net/rose/af_rose.c | 6 +- net/sched/cls_api.c | 4 +- net/sched/sch_api.c | 32 ++++-- net/sched/sch_generic.c | 178 +++++++++++++++++++++++--------- net/sched/sch_teql.c | 21 ++-- 18 files changed, 320 insertions(+), 137 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index fd87dbe7999a..9737c06045d6 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -5042,7 +5042,9 @@ static int bond_check_params(struct bond_params *params) static struct lock_class_key bonding_netdev_xmit_lock_key; -static void bond_set_lockdep_class_one(struct netdev_queue *txq) +static void bond_set_lockdep_class_one(struct net_device *dev, + struct netdev_queue *txq, + void *_unused) { lockdep_set_class(&txq->_xmit_lock, &bonding_netdev_xmit_lock_key); @@ -5050,7 +5052,7 @@ static void bond_set_lockdep_class_one(struct netdev_queue *txq) static void bond_set_lockdep_class(struct net_device *dev) { - bond_set_lockdep_class_one(&dev->tx_queue); + netdev_for_each_tx_queue(dev, bond_set_lockdep_class_one, NULL); } /* Create a new bond based on the specified name and bonding parameters. diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c index fb186b8c3d4d..b6500b2aacf2 100644 --- a/drivers/net/hamradio/bpqether.c +++ b/drivers/net/hamradio/bpqether.c @@ -124,14 +124,16 @@ static LIST_HEAD(bpq_devices); */ static struct lock_class_key bpq_netdev_xmit_lock_key; -static void bpq_set_lockdep_class_one(struct netdev_queue *txq) +static void bpq_set_lockdep_class_one(struct net_device *dev, + struct netdev_queue *txq, + void *_unused) { lockdep_set_class(&txq->_xmit_lock, &bpq_netdev_xmit_lock_key); } static void bpq_set_lockdep_class(struct net_device *dev) { - bpq_set_lockdep_class_one(&dev->tx_queue); + netdev_for_each_tx_queue(dev, bpq_set_lockdep_class_one, NULL); } /* ------------------------------------------------------------------------ */ diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c index ccbd6554f6eb..897b05e79ed0 100644 --- a/drivers/net/ifb.c +++ b/drivers/net/ifb.c @@ -229,14 +229,20 @@ module_param(numifbs, int, 0); MODULE_PARM_DESC(numifbs, "Number of ifb devices"); /* - * dev_ifb->tx_queue.lock is usually taken after dev->rx_queue.lock, + * dev_ifb's TX queue lock is usually taken after dev->rx_queue.lock, * reversely to e.g. qdisc_lock_tree(). It should be safe until - * ifb doesn't take dev->tx_queue.lock with dev_ifb->rx_queue.lock. + * ifb doesn't take dev's TX queue lock with dev_ifb->rx_queue.lock. * But lockdep should know that ifb has different locks from dev. */ static struct lock_class_key ifb_tx_queue_lock_key; static struct lock_class_key ifb_rx_queue_lock_key; +static void set_tx_lockdep_key(struct net_device *dev, + struct netdev_queue *txq, + void *_unused) +{ + lockdep_set_class(&txq->lock, &ifb_tx_queue_lock_key); +} static int __init ifb_init_one(int index) { @@ -258,7 +264,7 @@ static int __init ifb_init_one(int index) if (err < 0) goto err; - lockdep_set_class(&dev_ifb->tx_queue.lock, &ifb_tx_queue_lock_key); + netdev_for_each_tx_queue(dev_ifb, set_tx_lockdep_key, NULL); lockdep_set_class(&dev_ifb->rx_queue.lock, &ifb_rx_queue_lock_key); return 0; diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 980001c2cf96..72745ce588c6 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -285,7 +285,9 @@ static struct lock_class_key macvlan_netdev_xmit_lock_key; #define MACVLAN_STATE_MASK \ ((1<<__LINK_STATE_NOCARRIER) | (1<<__LINK_STATE_DORMANT)) -static void macvlan_set_lockdep_class_one(struct netdev_queue *txq) +static void macvlan_set_lockdep_class_one(struct net_device *dev, + struct netdev_queue *txq, + void *_unused) { lockdep_set_class(&txq->_xmit_lock, &macvlan_netdev_xmit_lock_key); @@ -293,7 +295,7 @@ static void macvlan_set_lockdep_class_one(struct netdev_queue *txq) static void macvlan_set_lockdep_class(struct net_device *dev) { - macvlan_set_lockdep_class_one(&dev->tx_queue); + netdev_for_each_tx_queue(dev, macvlan_set_lockdep_class_one, NULL); } static int macvlan_init(struct net_device *dev) diff --git a/drivers/net/wireless/hostap/hostap_hw.c b/drivers/net/wireless/hostap/hostap_hw.c index c1f4bb005d92..13d5882f1f21 100644 --- a/drivers/net/wireless/hostap/hostap_hw.c +++ b/drivers/net/wireless/hostap/hostap_hw.c @@ -3102,7 +3102,9 @@ static void prism2_clear_set_tim_queue(local_info_t *local) */ static struct lock_class_key hostap_netdev_xmit_lock_key; -static void prism2_set_lockdep_class_one(struct netdev_queue *txq) +static void prism2_set_lockdep_class_one(struct net_device *dev, + struct netdev_queue *txq, + void *_unused) { lockdep_set_class(&txq->_xmit_lock, &hostap_netdev_xmit_lock_key); @@ -3110,7 +3112,7 @@ static void prism2_set_lockdep_class_one(struct netdev_queue *txq) static void prism2_set_lockdep_class(struct net_device *dev) { - prism2_set_lockdep_class_one(&dev->tx_queue); + netdev_for_each_tx_queue(dev, prism2_set_lockdep_class_one, NULL); } static struct net_device * diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 570cf7affa72..f25d4f5a31b0 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -463,7 +463,7 @@ struct netdev_queue { struct Qdisc *qdisc_sleeping; struct list_head qdisc_list; struct netdev_queue *next_sched; -}; +} ____cacheline_aligned_in_smp; /* * The DEVICE structure. @@ -641,7 +641,9 @@ struct net_device unsigned char broadcast[MAX_ADDR_LEN]; /* hw bcast add */ struct netdev_queue rx_queue; - struct netdev_queue tx_queue ____cacheline_aligned_in_smp; + + struct netdev_queue *_tx ____cacheline_aligned_in_smp; + unsigned int num_tx_queues; unsigned long tx_queue_len; /* Max frames per queue allowed */ /* @@ -764,6 +766,25 @@ struct net_device #define NETDEV_ALIGN 32 #define NETDEV_ALIGN_CONST (NETDEV_ALIGN - 1) +static inline +struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev, + unsigned int index) +{ + return &dev->_tx[index]; +} + +static inline void netdev_for_each_tx_queue(struct net_device *dev, + void (*f)(struct net_device *, + struct netdev_queue *, + void *), + void *arg) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) + f(dev, &dev->_tx[i], arg); +} + /* * Net namespace inlines */ @@ -977,7 +998,7 @@ static inline void netif_schedule_queue(struct netdev_queue *txq) static inline void netif_schedule(struct net_device *dev) { - netif_schedule_queue(&dev->tx_queue); + netif_schedule_queue(netdev_get_tx_queue(dev, 0)); } /** @@ -993,7 +1014,7 @@ static inline void netif_tx_start_queue(struct netdev_queue *dev_queue) static inline void netif_start_queue(struct net_device *dev) { - netif_tx_start_queue(&dev->tx_queue); + netif_tx_start_queue(netdev_get_tx_queue(dev, 0)); } /** @@ -1017,7 +1038,7 @@ static inline void netif_tx_wake_queue(struct netdev_queue *dev_queue) static inline void netif_wake_queue(struct net_device *dev) { - netif_tx_wake_queue(&dev->tx_queue); + netif_tx_wake_queue(netdev_get_tx_queue(dev, 0)); } /** @@ -1034,7 +1055,7 @@ static inline void netif_tx_stop_queue(struct netdev_queue *dev_queue) static inline void netif_stop_queue(struct net_device *dev) { - netif_tx_stop_queue(&dev->tx_queue); + netif_tx_stop_queue(netdev_get_tx_queue(dev, 0)); } /** @@ -1050,7 +1071,7 @@ static inline int netif_tx_queue_stopped(const struct netdev_queue *dev_queue) static inline int netif_queue_stopped(const struct net_device *dev) { - return netif_tx_queue_stopped(&dev->tx_queue); + return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0)); } /** @@ -1134,7 +1155,7 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index) #endif if (test_and_clear_bit(__QUEUE_STATE_XOFF, &dev->egress_subqueue[queue_index].state)) - __netif_schedule(&dev->tx_queue); + __netif_schedule(netdev_get_tx_queue(dev, 0)); } /** @@ -1430,18 +1451,19 @@ static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu) static inline void netif_tx_lock(struct net_device *dev) { - __netif_tx_lock(&dev->tx_queue, smp_processor_id()); -} + int cpu = smp_processor_id(); + unsigned int i; -static inline void __netif_tx_lock_bh(struct netdev_queue *txq) -{ - spin_lock_bh(&txq->_xmit_lock); - txq->xmit_lock_owner = smp_processor_id(); + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + __netif_tx_lock(txq, cpu); + } } static inline void netif_tx_lock_bh(struct net_device *dev) { - __netif_tx_lock_bh(&dev->tx_queue); + local_bh_disable(); + netif_tx_lock(dev); } static inline int __netif_tx_trylock(struct netdev_queue *txq) @@ -1454,7 +1476,7 @@ static inline int __netif_tx_trylock(struct netdev_queue *txq) static inline int netif_tx_trylock(struct net_device *dev) { - return __netif_tx_trylock(&dev->tx_queue); + return __netif_tx_trylock(netdev_get_tx_queue(dev, 0)); } static inline void __netif_tx_unlock(struct netdev_queue *txq) @@ -1465,18 +1487,19 @@ static inline void __netif_tx_unlock(struct netdev_queue *txq) static inline void netif_tx_unlock(struct net_device *dev) { - __netif_tx_unlock(&dev->tx_queue); -} + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + __netif_tx_unlock(txq); + } -static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) -{ - txq->xmit_lock_owner = -1; - spin_unlock_bh(&txq->_xmit_lock); } static inline void netif_tx_unlock_bh(struct net_device *dev) { - __netif_tx_unlock_bh(&dev->tx_queue); + netif_tx_unlock(dev); + local_bh_enable(); } #define HARD_TX_LOCK(dev, txq, cpu) { \ diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 5ba66b555578..b47f556c66f8 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -230,32 +230,47 @@ extern void tcf_destroy_chain(struct tcf_proto **fl); /* Reset all TX qdiscs of a device. */ static inline void qdisc_reset_all_tx(struct net_device *dev) { - qdisc_reset(dev->tx_queue.qdisc); + unsigned int i; + for (i = 0; i < dev->num_tx_queues; i++) + qdisc_reset(netdev_get_tx_queue(dev, i)->qdisc); } /* Are all TX queues of the device empty? */ static inline bool qdisc_all_tx_empty(const struct net_device *dev) { - const struct netdev_queue *txq = &dev->tx_queue; - const struct Qdisc *q = txq->qdisc; + unsigned int i; + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + const struct Qdisc *q = txq->qdisc; - return (q->q.qlen == 0); + if (q->q.qlen) + return false; + } + return true; } /* Are any of the TX qdiscs changing? */ static inline bool qdisc_tx_changing(struct net_device *dev) { - struct netdev_queue *txq = &dev->tx_queue; - - return (txq->qdisc != txq->qdisc_sleeping); + unsigned int i; + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + if (txq->qdisc != txq->qdisc_sleeping) + return true; + } + return false; } -/* Is the device using the noop qdisc? */ +/* Is the device using the noop qdisc on all queues? */ static inline bool qdisc_tx_is_noop(const struct net_device *dev) { - const struct netdev_queue *txq = &dev->tx_queue; - - return (txq->qdisc == &noop_qdisc); + unsigned int i; + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + if (txq->qdisc != &noop_qdisc) + return false; + } + return true; } static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 6b985f23fd9f..f42bc2b26b85 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -570,16 +570,18 @@ static void vlan_dev_set_rx_mode(struct net_device *vlan_dev) */ static struct lock_class_key vlan_netdev_xmit_lock_key; -static void vlan_dev_set_lockdep_one(struct netdev_queue *txq, - int subclass) +static void vlan_dev_set_lockdep_one(struct net_device *dev, + struct netdev_queue *txq, + void *_subclass) { lockdep_set_class_and_subclass(&txq->_xmit_lock, - &vlan_netdev_xmit_lock_key, subclass); + &vlan_netdev_xmit_lock_key, + *(int *)_subclass); } static void vlan_dev_set_lockdep_class(struct net_device *dev, int subclass) { - vlan_dev_set_lockdep_one(&dev->tx_queue, subclass); + netdev_for_each_tx_queue(dev, vlan_dev_set_lockdep_one, &subclass); } static const struct header_ops vlan_header_ops = { diff --git a/net/core/dev.c b/net/core/dev.c index 9b49f74a9820..69378f250695 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1666,6 +1666,12 @@ out_kfree_skb: * --BLG */ +static struct netdev_queue *dev_pick_tx(struct net_device *dev, + struct sk_buff *skb) +{ + return netdev_get_tx_queue(dev, 0); +} + int dev_queue_xmit(struct sk_buff *skb) { struct net_device *dev = skb->dev; @@ -1702,7 +1708,7 @@ int dev_queue_xmit(struct sk_buff *skb) } gso: - txq = &dev->tx_queue; + txq = dev_pick_tx(dev, skb); spin_lock_prefetch(&txq->lock); /* Disable soft irqs for various locks below. Also @@ -3788,8 +3794,9 @@ static void rollback_registered(struct net_device *dev) dev_put(dev); } -static void __netdev_init_queue_locks_one(struct netdev_queue *dev_queue, - struct net_device *dev) +static void __netdev_init_queue_locks_one(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_unused) { spin_lock_init(&dev_queue->_xmit_lock); netdev_set_lockdep_class(&dev_queue->_xmit_lock, dev->type); @@ -3798,8 +3805,8 @@ static void __netdev_init_queue_locks_one(struct netdev_queue *dev_queue, static void netdev_init_queue_locks(struct net_device *dev) { - __netdev_init_queue_locks_one(&dev->tx_queue, dev); - __netdev_init_queue_locks_one(&dev->rx_queue, dev); + netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL); + __netdev_init_queue_locks_one(dev, &dev->rx_queue, NULL); } /** @@ -4119,7 +4126,8 @@ static struct net_device_stats *internal_stats(struct net_device *dev) } static void netdev_init_one_queue(struct net_device *dev, - struct netdev_queue *queue) + struct netdev_queue *queue, + void *_unused) { spin_lock_init(&queue->lock); queue->dev = dev; @@ -4127,8 +4135,8 @@ static void netdev_init_one_queue(struct net_device *dev, static void netdev_init_queues(struct net_device *dev) { - netdev_init_one_queue(dev, &dev->rx_queue); - netdev_init_one_queue(dev, &dev->tx_queue); + netdev_init_one_queue(dev, &dev->rx_queue, NULL); + netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); } /** @@ -4145,9 +4153,10 @@ static void netdev_init_queues(struct net_device *dev) struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name, void (*setup)(struct net_device *), unsigned int queue_count) { - void *p; + struct netdev_queue *tx; struct net_device *dev; int alloc_size; + void *p; BUG_ON(strlen(name) >= sizeof(dev->name)); @@ -4167,11 +4176,22 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name, return NULL; } + tx = kzalloc(sizeof(struct netdev_queue) * queue_count, GFP_KERNEL); + if (!tx) { + printk(KERN_ERR "alloc_netdev: Unable to allocate " + "tx qdiscs.\n"); + kfree(p); + return NULL; + } + dev = (struct net_device *) (((long)p + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST); dev->padded = (char *)dev - (char *)p; dev_net_set(dev, &init_net); + dev->_tx = tx; + dev->num_tx_queues = queue_count; + if (sizeof_priv) { dev->priv = ((char *)dev + ((sizeof(struct net_device) + @@ -4205,6 +4225,8 @@ void free_netdev(struct net_device *dev) { release_net(dev_net(dev)); + kfree(dev->_tx); + /* Compatibility with error handling in drivers */ if (dev->reg_state == NETREG_UNINITIALIZED) { kfree((char *)dev - dev->padded); diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 8ef9f1db610e..71edb8b36341 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -636,7 +636,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev, if (dev->master) NLA_PUT_U32(skb, IFLA_MASTER, dev->master->ifindex); - txq = &dev->tx_queue; + txq = netdev_get_tx_queue(dev, 0); if (txq->qdisc_sleeping) NLA_PUT_STRING(skb, IFLA_QDISC, txq->qdisc_sleeping->ops->id); diff --git a/net/mac80211/main.c b/net/mac80211/main.c index af0056e7e5b3..b486e634f4fe 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -621,7 +621,7 @@ int ieee80211_start_tx_ba_session(struct ieee80211_hw *hw, u8 *ra, u16 tid) /* ensure that TX flow won't interrupt us * until the end of the call to requeue function */ - txq = &local->mdev->tx_queue; + txq = netdev_get_tx_queue(local->mdev, 0); spin_lock_bh(&txq->lock); /* create a new queue for this aggregation */ @@ -862,7 +862,7 @@ void ieee80211_stop_tx_ba_cb(struct ieee80211_hw *hw, u8 *ra, u8 tid) /* avoid ordering issues: we are the only one that can modify * the content of the qdiscs */ - txq = &local->mdev->tx_queue; + txq = netdev_get_tx_queue(local->mdev, 0); spin_lock_bh(&txq->lock); /* remove the queue for this aggregation */ ieee80211_ht_agg_queue_remove(local, sta, tid, 1); diff --git a/net/mac80211/wme.c b/net/mac80211/wme.c index 6ae43a3c7726..f014cd38c2d0 100644 --- a/net/mac80211/wme.c +++ b/net/mac80211/wme.c @@ -574,7 +574,7 @@ static struct Qdisc_ops wme_qdisc_ops __read_mostly = void ieee80211_install_qdisc(struct net_device *dev) { - struct netdev_queue *txq = &dev->tx_queue; + struct netdev_queue *txq = netdev_get_tx_queue(dev, 0); struct Qdisc *qdisc; qdisc = qdisc_create_dflt(dev, txq, @@ -596,7 +596,7 @@ void ieee80211_install_qdisc(struct net_device *dev) int ieee80211_qdisc_installed(struct net_device *dev) { - struct netdev_queue *txq = &dev->tx_queue; + struct netdev_queue *txq = netdev_get_tx_queue(dev, 0); return txq->qdisc_sleeping->ops == &wme_qdisc_ops; } @@ -617,7 +617,7 @@ int ieee80211_ht_agg_queue_add(struct ieee80211_local *local, struct sta_info *sta, u16 tid) { int i; - struct netdev_queue *txq = &local->mdev->tx_queue; + struct netdev_queue *txq = netdev_get_tx_queue(local->mdev, 0); struct ieee80211_sched_data *q = qdisc_priv(txq->qdisc_sleeping); DECLARE_MAC_BUF(mac); @@ -652,14 +652,14 @@ int ieee80211_ht_agg_queue_add(struct ieee80211_local *local, } /** - * the caller needs to hold local->mdev->tx_queue.lock + * the caller needs to hold netdev_get_tx_queue(local->mdev, X)->lock */ void ieee80211_ht_agg_queue_remove(struct ieee80211_local *local, struct sta_info *sta, u16 tid, u8 requeue) { struct ieee80211_hw *hw = &local->hw; - struct netdev_queue *txq = &local->mdev->tx_queue; + struct netdev_queue *txq = netdev_get_tx_queue(local->mdev, 0); struct ieee80211_sched_data *q = qdisc_priv(txq->qdisc_sleeping); int agg_queue = sta->tid_to_tx_q[tid]; @@ -676,7 +676,7 @@ void ieee80211_ht_agg_queue_remove(struct ieee80211_local *local, void ieee80211_requeue(struct ieee80211_local *local, int queue) { - struct netdev_queue *txq = &local->mdev->tx_queue; + struct netdev_queue *txq = netdev_get_tx_queue(local->mdev, 0); struct Qdisc *root_qd = txq->qdisc_sleeping; struct ieee80211_sched_data *q = qdisc_priv(root_qd); struct Qdisc *qdisc = q->queues[queue]; diff --git a/net/netrom/af_netrom.c b/net/netrom/af_netrom.c index 819afc449e1e..d41be0d66eb0 100644 --- a/net/netrom/af_netrom.c +++ b/net/netrom/af_netrom.c @@ -74,14 +74,16 @@ static const struct proto_ops nr_proto_ops; */ static struct lock_class_key nr_netdev_xmit_lock_key; -static void nr_set_lockdep_one(struct netdev_queue *txq) +static void nr_set_lockdep_one(struct net_device *dev, + struct netdev_queue *txq, + void *_unused) { lockdep_set_class(&txq->_xmit_lock, &nr_netdev_xmit_lock_key); } static void nr_set_lockdep_key(struct net_device *dev) { - nr_set_lockdep_one(&dev->tx_queue); + netdev_for_each_tx_queue(dev, nr_set_lockdep_one, NULL); } /* diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index 7dbbc0891623..f3a691f34909 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -75,14 +75,16 @@ ax25_address rose_callsign; */ static struct lock_class_key rose_netdev_xmit_lock_key; -static void rose_set_lockdep_one(struct netdev_queue *txq) +static void rose_set_lockdep_one(struct net_device *dev, + struct netdev_queue *txq, + void *_unused) { lockdep_set_class(&txq->_xmit_lock, &rose_netdev_xmit_lock_key); } static void rose_set_lockdep_key(struct net_device *dev) { - rose_set_lockdep_one(&dev->tx_queue); + netdev_for_each_tx_queue(dev, rose_set_lockdep_one, NULL); } /* diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index b483bbea6118..d0b0a9b14394 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -166,7 +166,7 @@ replay: /* Find qdisc */ if (!parent) { - struct netdev_queue *dev_queue = &dev->tx_queue; + struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, 0); q = dev_queue->qdisc_sleeping; parent = q->handle; } else { @@ -410,7 +410,7 @@ static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb) if ((dev = dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL) return skb->len; - dev_queue = &dev->tx_queue; + dev_queue = netdev_get_tx_queue(dev, 0); if (!tcm->tcm_parent) q = dev_queue->qdisc_sleeping; else diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 95873f8dd37c..830ccc544a15 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -183,9 +183,8 @@ EXPORT_SYMBOL(unregister_qdisc); (root qdisc, all its children, children of children etc.) */ -struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) +static struct Qdisc *__qdisc_lookup(struct netdev_queue *dev_queue, u32 handle) { - struct netdev_queue *dev_queue = &dev->tx_queue; struct Qdisc *q; list_for_each_entry(q, &dev_queue->qdisc_list, list) { @@ -195,6 +194,19 @@ struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) return NULL; } +struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + struct Qdisc *q = __qdisc_lookup(txq, handle); + if (q) + return q; + } + return NULL; +} + static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid) { unsigned long cl; @@ -462,7 +474,7 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc) } } else { - dev_queue = &dev->tx_queue; + dev_queue = netdev_get_tx_queue(dev, 0); oqdisc = dev_queue->qdisc_sleeping; /* Prune old scheduler */ @@ -742,7 +754,8 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg) q = dev->rx_queue.qdisc; } } else { - struct netdev_queue *dev_queue = &dev->tx_queue; + struct netdev_queue *dev_queue; + dev_queue = netdev_get_tx_queue(dev, 0); q = dev_queue->qdisc_sleeping; } if (!q) @@ -817,7 +830,8 @@ replay: q = dev->rx_queue.qdisc; } } else { - struct netdev_queue *dev_queue = &dev->tx_queue; + struct netdev_queue *dev_queue; + dev_queue = netdev_get_tx_queue(dev, 0); q = dev_queue->qdisc_sleeping; } @@ -899,7 +913,7 @@ create_n_graft: tcm->tcm_parent, tcm->tcm_parent, tca, &err); else - q = qdisc_create(dev, &dev->tx_queue, + q = qdisc_create(dev, netdev_get_tx_queue(dev, 0), tcm->tcm_parent, tcm->tcm_handle, tca, &err); if (q == NULL) { @@ -1025,7 +1039,7 @@ static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) if (idx > s_idx) s_q_idx = 0; q_idx = 0; - dev_queue = &dev->tx_queue; + dev_queue = netdev_get_tx_queue(dev, 0); list_for_each_entry(q, &dev_queue->qdisc_list, list) { if (q_idx < s_q_idx) { q_idx++; @@ -1098,7 +1112,7 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg) /* Step 1. Determine qdisc handle X:0 */ - dev_queue = &dev->tx_queue; + dev_queue = netdev_get_tx_queue(dev, 0); if (pid != TC_H_ROOT) { u32 qid1 = TC_H_MAJ(pid); @@ -1275,7 +1289,7 @@ static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) s_t = cb->args[0]; t = 0; - dev_queue = &dev->tx_queue; + dev_queue = netdev_get_tx_queue(dev, 0); list_for_each_entry(q, &dev_queue->qdisc_list, list) { if (t < s_t || !q->ops->cl_ops || (tcm->tcm_parent && diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 243de935b182..4e2b865cbba0 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -40,20 +40,30 @@ */ void qdisc_lock_tree(struct net_device *dev) - __acquires(dev->tx_queue.lock) __acquires(dev->rx_queue.lock) { - spin_lock_bh(&dev->tx_queue.lock); + unsigned int i; + + local_bh_disable(); + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + spin_lock(&txq->lock); + } spin_lock(&dev->rx_queue.lock); } EXPORT_SYMBOL(qdisc_lock_tree); void qdisc_unlock_tree(struct net_device *dev) __releases(dev->rx_queue.lock) - __releases(dev->tx_queue.lock) { + unsigned int i; + spin_unlock(&dev->rx_queue.lock); - spin_unlock_bh(&dev->tx_queue.lock); + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + spin_unlock(&txq->lock); + } + local_bh_enable(); } EXPORT_SYMBOL(qdisc_unlock_tree); @@ -212,22 +222,37 @@ void __qdisc_run(struct netdev_queue *txq) static void dev_watchdog(unsigned long arg) { struct net_device *dev = (struct net_device *)arg; - struct netdev_queue *txq = &dev->tx_queue; netif_tx_lock(dev); - if (txq->qdisc != &noop_qdisc) { + if (!qdisc_tx_is_noop(dev)) { if (netif_device_present(dev) && netif_running(dev) && netif_carrier_ok(dev)) { - if (netif_queue_stopped(dev) && - time_after(jiffies, dev->trans_start + dev->watchdog_timeo)) { + int some_queue_stopped = 0; + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq; + + txq = netdev_get_tx_queue(dev, i); + if (netif_tx_queue_stopped(txq)) { + some_queue_stopped = 1; + break; + } + } - printk(KERN_INFO "NETDEV WATCHDOG: %s: transmit timed out\n", + if (some_queue_stopped && + time_after(jiffies, (dev->trans_start + + dev->watchdog_timeo))) { + printk(KERN_INFO "NETDEV WATCHDOG: %s: " + "transmit timed out\n", dev->name); dev->tx_timeout(dev); WARN_ON_ONCE(1); } - if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo))) + if (!mod_timer(&dev->watchdog_timer, + round_jiffies(jiffies + + dev->watchdog_timeo))) dev_hold(dev); } } @@ -542,9 +567,55 @@ void qdisc_destroy(struct Qdisc *qdisc) } EXPORT_SYMBOL(qdisc_destroy); +static bool dev_all_qdisc_sleeping_noop(struct net_device *dev) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + + if (txq->qdisc_sleeping != &noop_qdisc) + return false; + } + return true; +} + +static void attach_one_default_qdisc(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_unused) +{ + struct Qdisc *qdisc; + + if (dev->tx_queue_len) { + qdisc = qdisc_create_dflt(dev, dev_queue, + &pfifo_fast_ops, TC_H_ROOT); + if (!qdisc) { + printk(KERN_INFO "%s: activation failed\n", dev->name); + return; + } + list_add_tail(&qdisc->list, &dev_queue->qdisc_list); + } else { + qdisc = &noqueue_qdisc; + } + dev_queue->qdisc_sleeping = qdisc; +} + +static void transition_one_qdisc(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_need_watchdog) +{ + int *need_watchdog_p = _need_watchdog; + + spin_lock_bh(&dev_queue->lock); + rcu_assign_pointer(dev_queue->qdisc, dev_queue->qdisc_sleeping); + if (dev_queue->qdisc != &noqueue_qdisc) + *need_watchdog_p = 1; + spin_unlock_bh(&dev_queue->lock); +} + void dev_activate(struct net_device *dev) { - struct netdev_queue *txq = &dev->tx_queue; + int need_watchdog; /* No queueing discipline is attached to device; create default one i.e. pfifo_fast for devices, @@ -552,39 +623,27 @@ void dev_activate(struct net_device *dev) virtual interfaces */ - if (txq->qdisc_sleeping == &noop_qdisc) { - struct Qdisc *qdisc; - if (dev->tx_queue_len) { - qdisc = qdisc_create_dflt(dev, txq, - &pfifo_fast_ops, - TC_H_ROOT); - if (qdisc == NULL) { - printk(KERN_INFO "%s: activation failed\n", dev->name); - return; - } - list_add_tail(&qdisc->list, &txq->qdisc_list); - } else { - qdisc = &noqueue_qdisc; - } - txq->qdisc_sleeping = qdisc; - } + if (dev_all_qdisc_sleeping_noop(dev)) + netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL); if (!netif_carrier_ok(dev)) /* Delay activation until next carrier-on event */ return; - spin_lock_bh(&txq->lock); - rcu_assign_pointer(txq->qdisc, txq->qdisc_sleeping); - if (txq->qdisc != &noqueue_qdisc) { + need_watchdog = 0; + netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog); + + if (need_watchdog) { dev->trans_start = jiffies; dev_watchdog_up(dev); } - spin_unlock_bh(&txq->lock); } -static void dev_deactivate_queue(struct netdev_queue *dev_queue, - struct Qdisc *qdisc_default) +static void dev_deactivate_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_qdisc_default) { + struct Qdisc *qdisc_default = _qdisc_default; struct Qdisc *qdisc; struct sk_buff *skb; @@ -603,12 +662,35 @@ static void dev_deactivate_queue(struct netdev_queue *dev_queue, kfree_skb(skb); } +static bool some_qdisc_is_running(struct net_device *dev, int lock) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *dev_queue; + int val; + + dev_queue = netdev_get_tx_queue(dev, i); + + if (lock) + spin_lock_bh(&dev_queue->lock); + + val = test_bit(__QUEUE_STATE_QDISC_RUNNING, &dev_queue->state); + + if (lock) + spin_unlock_bh(&dev_queue->lock); + + if (val) + return true; + } + return false; +} + void dev_deactivate(struct net_device *dev) { - struct netdev_queue *dev_queue = &dev->tx_queue; - int running; + bool running; - dev_deactivate_queue(dev_queue, &noop_qdisc); + netdev_for_each_tx_queue(dev, dev_deactivate_queue, &noop_qdisc); dev_watchdog_down(dev); @@ -617,17 +699,14 @@ void dev_deactivate(struct net_device *dev) /* Wait for outstanding qdisc_run calls. */ do { - while (test_bit(__QUEUE_STATE_QDISC_RUNNING, &dev_queue->state)) + while (some_qdisc_is_running(dev, 0)) yield(); /* * Double-check inside queue lock to ensure that all effects * of the queue run are visible when we return. */ - spin_lock_bh(&dev_queue->lock); - running = test_bit(__QUEUE_STATE_QDISC_RUNNING, - &dev_queue->state); - spin_unlock_bh(&dev_queue->lock); + running = some_qdisc_is_running(dev, 1); /* * The running flag should never be set at this point because @@ -642,8 +721,10 @@ void dev_deactivate(struct net_device *dev) static void dev_init_scheduler_queue(struct net_device *dev, struct netdev_queue *dev_queue, - struct Qdisc *qdisc) + void *_qdisc) { + struct Qdisc *qdisc = _qdisc; + dev_queue->qdisc = qdisc; dev_queue->qdisc_sleeping = qdisc; INIT_LIST_HEAD(&dev_queue->qdisc_list); @@ -652,18 +733,19 @@ static void dev_init_scheduler_queue(struct net_device *dev, void dev_init_scheduler(struct net_device *dev) { qdisc_lock_tree(dev); - dev_init_scheduler_queue(dev, &dev->tx_queue, &noop_qdisc); + netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc); dev_init_scheduler_queue(dev, &dev->rx_queue, NULL); qdisc_unlock_tree(dev); setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev); } -static void dev_shutdown_scheduler_queue(struct net_device *dev, - struct netdev_queue *dev_queue, - struct Qdisc *qdisc_default) +static void shutdown_scheduler_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_qdisc_default) { struct Qdisc *qdisc = dev_queue->qdisc_sleeping; + struct Qdisc *qdisc_default = _qdisc_default; if (qdisc) { dev_queue->qdisc = qdisc_default; @@ -676,8 +758,8 @@ static void dev_shutdown_scheduler_queue(struct net_device *dev, void dev_shutdown(struct net_device *dev) { qdisc_lock_tree(dev); - dev_shutdown_scheduler_queue(dev, &dev->tx_queue, &noop_qdisc); - dev_shutdown_scheduler_queue(dev, &dev->rx_queue, NULL); + netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc); + shutdown_scheduler_queue(dev, &dev->rx_queue, NULL); BUG_TRAP(!timer_pending(&dev->watchdog_timer)); qdisc_unlock_tree(dev); } diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 8ac05981be20..44a2c3451f4d 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -111,7 +111,7 @@ teql_dequeue(struct Qdisc* sch) struct sk_buff *skb; skb = __skb_dequeue(&dat->q); - dat_queue = &dat->m->dev->tx_queue; + dat_queue = netdev_get_tx_queue(dat->m->dev, 0); if (skb == NULL) { struct net_device *m = qdisc_dev(dat_queue->qdisc); if (m) { @@ -155,10 +155,13 @@ teql_destroy(struct Qdisc* sch) if (q == master->slaves) { master->slaves = NEXT_SLAVE(q); if (q == master->slaves) { + struct netdev_queue *txq; + + txq = netdev_get_tx_queue(master->dev, 0); master->slaves = NULL; - spin_lock_bh(&master->dev->tx_queue.lock); - qdisc_reset(master->dev->tx_queue.qdisc); - spin_unlock_bh(&master->dev->tx_queue.lock); + spin_lock_bh(&txq->lock); + qdisc_reset(txq->qdisc); + spin_unlock_bh(&txq->lock); } } skb_queue_purge(&dat->q); @@ -218,7 +221,8 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt) static int __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, struct net_device *dev) { - struct teql_sched_data *q = qdisc_priv(dev->tx_queue.qdisc); + struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, 0); + struct teql_sched_data *q = qdisc_priv(dev_queue->qdisc); struct neighbour *mn = skb->dst->neighbour; struct neighbour *n = q->ncache; @@ -254,7 +258,8 @@ __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, struct net_device * static inline int teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, struct net_device *dev) { - if (dev->tx_queue.qdisc == &noop_qdisc) + struct netdev_queue *txq = netdev_get_tx_queue(dev, 0); + if (txq->qdisc == &noop_qdisc) return -ENODEV; if (dev->header_ops == NULL || @@ -285,8 +290,10 @@ restart: do { struct net_device *slave = qdisc_dev(q); + struct netdev_queue *slave_txq; - if (slave->tx_queue.qdisc_sleeping != q) + slave_txq = netdev_get_tx_queue(slave, 0); + if (slave_txq->qdisc_sleeping != q) continue; if (netif_queue_stopped(slave) || __netif_subqueue_stopped(slave, subq) || -- cgit v1.2.3 From fd2ea0a79faad824258af5dcec1927aa24d81c16 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Thu, 17 Jul 2008 01:56:23 -0700 Subject: net: Use queue aware tests throughout. This effectively "flips the switch" by making the core networking and multiqueue-aware drivers use the new TX multiqueue structures. Non-multiqueue drivers need no changes. The interfaces they use such as netif_stop_queue() degenerate into an operation on TX queue zero. So everything "just works" for them. Code that really wants to do "X" to all TX queues now invokes a routine that does so, such as netif_tx_wake_all_queues(), netif_tx_stop_all_queues(), etc. pktgen and netpoll required a little bit more surgery than the others. In particular the pktgen changes, whilst functional, could be largely improved. The initial check in pktgen_xmit() will sometimes check the wrong queue, which is mostly harmless. The thing to do is probably to invoke fill_packet() earlier. The bulk of the netpoll changes is to make the code operate solely on the TX queue indicated by by the SKB queue mapping. Setting of the SKB queue mapping is entirely confined inside of net/core/dev.c:dev_pick_tx(). If we end up needing any kind of special semantics (drops, for example) it will be implemented here. Finally, we now have a "real_num_tx_queues" which is where the driver indicates how many TX queues are actually active. With IGB changes from Jeff Kirsher. Signed-off-by: David S. Miller --- drivers/net/cpmac.c | 20 ++++------ drivers/net/igb/igb_main.c | 19 +++------ drivers/net/ixgbe/ixgbe_ethtool.c | 10 +---- drivers/net/ixgbe/ixgbe_main.c | 15 +++---- drivers/net/s2io.c | 48 +++++++++-------------- include/linux/netdevice.h | 82 ++++++++++++++++++++++++++++++++++----- include/net/pkt_sched.h | 4 +- net/core/dev.c | 28 ++++++------- net/core/netpoll.c | 24 +++++++----- net/core/pktgen.c | 69 ++++++++++++++++++++------------ net/sched/sch_generic.c | 5 +-- net/sched/sch_teql.c | 6 +-- 12 files changed, 187 insertions(+), 143 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/net/cpmac.c b/drivers/net/cpmac.c index 7c7b54e4828e..fbd4280c102c 100644 --- a/drivers/net/cpmac.c +++ b/drivers/net/cpmac.c @@ -544,7 +544,7 @@ fatal_error: spin_unlock(&priv->rx_lock); netif_rx_complete(priv->dev, napi); - netif_stop_queue(priv->dev); + netif_tx_stop_all_queues(priv->dev); napi_disable(&priv->napi); atomic_inc(&priv->reset_pending); @@ -750,9 +750,7 @@ static void cpmac_hw_error(struct work_struct *work) barrier(); atomic_dec(&priv->reset_pending); - for (i = 0; i < CPMAC_QUEUES; i++) - netif_wake_subqueue(priv->dev, i); - netif_wake_queue(priv->dev); + netif_tx_wake_all_queues(priv->dev); cpmac_write(priv->regs, CPMAC_MAC_INT_ENABLE, 3); } @@ -781,7 +779,7 @@ static void cpmac_check_status(struct net_device *dev) dev->name, tx_code, tx_channel, macstatus); } - netif_stop_queue(dev); + netif_tx_stop_all_queues(dev); cpmac_hw_stop(dev); if (schedule_work(&priv->reset_work)) atomic_inc(&priv->reset_pending); @@ -842,9 +840,7 @@ static void cpmac_tx_timeout(struct net_device *dev) barrier(); atomic_dec(&priv->reset_pending); - netif_wake_queue(priv->dev); - for (i = 0; i < CPMAC_QUEUES; i++) - netif_wake_subqueue(dev, i); + netif_tx_wake_all_queues(priv->dev); } static int cpmac_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) @@ -935,7 +931,7 @@ static void cpmac_adjust_link(struct net_device *dev) spin_lock(&priv->lock); if (priv->phy->link) { - netif_start_queue(dev); + netif_tx_start_all_queues(dev); if (priv->phy->duplex != priv->oldduplex) { new_state = 1; priv->oldduplex = priv->phy->duplex; @@ -949,10 +945,10 @@ static void cpmac_adjust_link(struct net_device *dev) if (!priv->oldlink) { new_state = 1; priv->oldlink = 1; - netif_schedule(dev); + netif_tx_schedule_all(dev); } } else if (priv->oldlink) { - netif_stop_queue(dev); + netif_tx_stop_all_queues(dev); new_state = 1; priv->oldlink = 0; priv->oldspeed = 0; @@ -1072,7 +1068,7 @@ static int cpmac_stop(struct net_device *dev) struct cpmac_priv *priv = netdev_priv(dev); struct resource *mem; - netif_stop_queue(dev); + netif_tx_stop_all_queues(dev); cancel_work_sync(&priv->reset_work); napi_disable(&priv->napi); diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c index 471c194cd54e..81bba6983dde 100644 --- a/drivers/net/igb/igb_main.c +++ b/drivers/net/igb/igb_main.c @@ -533,7 +533,7 @@ msi_only: adapter->flags |= IGB_FLAG_HAS_MSI; /* Notify the stack of the (possibly) reduced Tx Queue count. */ - adapter->netdev->egress_subqueue_count = adapter->num_tx_queues; + adapter->netdev->real_num_tx_queues = adapter->num_tx_queues; return; } @@ -821,9 +821,7 @@ void igb_down(struct igb_adapter *adapter) wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN); /* flush and sleep below */ - netif_stop_queue(netdev); - for (i = 0; i < adapter->num_tx_queues; i++) - netif_stop_subqueue(netdev, i); + netif_tx_stop_all_queues(netdev); /* disable transmits in the hardware */ tctl = rd32(E1000_TCTL); @@ -1266,9 +1264,7 @@ static int __devinit igb_probe(struct pci_dev *pdev, /* tell the stack to leave us alone until igb_open() is called */ netif_carrier_off(netdev); - netif_stop_queue(netdev); - for (i = 0; i < adapter->num_tx_queues; i++) - netif_stop_subqueue(netdev, i); + netif_tx_stop_all_queues(netdev); strcpy(netdev->name, "eth%d"); err = register_netdev(netdev); @@ -2315,7 +2311,6 @@ static void igb_watchdog_task(struct work_struct *work) struct e1000_mac_info *mac = &adapter->hw.mac; u32 link; s32 ret_val; - int i; if ((netif_carrier_ok(netdev)) && (rd32(E1000_STATUS) & E1000_STATUS_LU)) @@ -2371,9 +2366,7 @@ static void igb_watchdog_task(struct work_struct *work) } netif_carrier_on(netdev); - netif_wake_queue(netdev); - for (i = 0; i < adapter->num_tx_queues; i++) - netif_wake_subqueue(netdev, i); + netif_tx_wake_all_queues(netdev); if (!test_bit(__IGB_DOWN, &adapter->state)) mod_timer(&adapter->phy_info_timer, @@ -2385,9 +2378,7 @@ static void igb_watchdog_task(struct work_struct *work) adapter->link_duplex = 0; dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); netif_carrier_off(netdev); - netif_stop_queue(netdev); - for (i = 0; i < adapter->num_tx_queues; i++) - netif_stop_subqueue(netdev, i); + netif_tx_stop_all_queues(netdev); if (!test_bit(__IGB_DOWN, &adapter->state)) mod_timer(&adapter->phy_info_timer, round_jiffies(jiffies + 2 * HZ)); diff --git a/drivers/net/ixgbe/ixgbe_ethtool.c b/drivers/net/ixgbe/ixgbe_ethtool.c index 81b769093d22..3efe5dda10af 100644 --- a/drivers/net/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ixgbe/ixgbe_ethtool.c @@ -252,16 +252,10 @@ static int ixgbe_set_tso(struct net_device *netdev, u32 data) netdev->features |= NETIF_F_TSO; netdev->features |= NETIF_F_TSO6; } else { - struct ixgbe_adapter *adapter = netdev_priv(netdev); - int i; - netif_stop_queue(netdev); - for (i = 0; i < adapter->num_tx_queues; i++) - netif_stop_subqueue(netdev, i); + netif_tx_stop_all_queues(netdev); netdev->features &= ~NETIF_F_TSO; netdev->features &= ~NETIF_F_TSO6; - for (i = 0; i < adapter->num_tx_queues; i++) - netif_start_subqueue(netdev, i); - netif_start_queue(netdev); + netif_tx_start_all_queues(netdev); } return 0; } diff --git a/drivers/net/ixgbe/ixgbe_main.c b/drivers/net/ixgbe/ixgbe_main.c index e6df9233f5ef..6af8fb5c4b5f 100644 --- a/drivers/net/ixgbe/ixgbe_main.c +++ b/drivers/net/ixgbe/ixgbe_main.c @@ -2013,7 +2013,7 @@ void ixgbe_down(struct ixgbe_adapter *adapter) del_timer_sync(&adapter->watchdog_timer); netif_carrier_off(netdev); - netif_stop_queue(netdev); + netif_tx_stop_all_queues(netdev); if (!pci_channel_offline(adapter->pdev)) ixgbe_reset(adapter); @@ -2359,7 +2359,7 @@ try_msi: out: /* Notify the stack of the (possibly) reduced Tx Queue count. */ - adapter->netdev->egress_subqueue_count = adapter->num_tx_queues; + adapter->netdev->real_num_tx_queues = adapter->num_tx_queues; return err; } @@ -2896,7 +2896,6 @@ static void ixgbe_watchdog(unsigned long data) struct net_device *netdev = adapter->netdev; bool link_up; u32 link_speed = 0; - int i; adapter->hw.mac.ops.check_link(&adapter->hw, &(link_speed), &link_up); @@ -2917,9 +2916,7 @@ static void ixgbe_watchdog(unsigned long data) (FLOW_TX ? "TX" : "None")))); netif_carrier_on(netdev); - netif_wake_queue(netdev); - for (i = 0; i < adapter->num_tx_queues; i++) - netif_wake_subqueue(netdev, i); + netif_tx_wake_all_queues(netdev); } else { /* Force detection of hung controller */ adapter->detect_tx_hung = true; @@ -2928,7 +2925,7 @@ static void ixgbe_watchdog(unsigned long data) if (netif_carrier_ok(netdev)) { DPRINTK(LINK, INFO, "NIC Link is Down\n"); netif_carrier_off(netdev); - netif_stop_queue(netdev); + netif_tx_stop_all_queues(netdev); } } @@ -3631,9 +3628,7 @@ static int __devinit ixgbe_probe(struct pci_dev *pdev, ixgbe_start_hw(hw); netif_carrier_off(netdev); - netif_stop_queue(netdev); - for (i = 0; i < adapter->num_tx_queues; i++) - netif_stop_subqueue(netdev, i); + netif_tx_stop_all_queues(netdev); ixgbe_napi_add_all(adapter); diff --git a/drivers/net/s2io.c b/drivers/net/s2io.c index 5f0fcb04afff..9dae40ccf048 100644 --- a/drivers/net/s2io.c +++ b/drivers/net/s2io.c @@ -545,63 +545,53 @@ static struct pci_driver s2io_driver = { /* netqueue manipulation helper functions */ static inline void s2io_stop_all_tx_queue(struct s2io_nic *sp) { - int i; - if (sp->config.multiq) { - for (i = 0; i < sp->config.tx_fifo_num; i++) - netif_stop_subqueue(sp->dev, i); - } else { + if (!sp->config.multiq) { + int i; + for (i = 0; i < sp->config.tx_fifo_num; i++) sp->mac_control.fifos[i].queue_state = FIFO_QUEUE_STOP; - netif_stop_queue(sp->dev); } + netif_tx_stop_all_queues(sp->dev); } static inline void s2io_stop_tx_queue(struct s2io_nic *sp, int fifo_no) { - if (sp->config.multiq) - netif_stop_subqueue(sp->dev, fifo_no); - else { + if (!sp->config.multiq) sp->mac_control.fifos[fifo_no].queue_state = FIFO_QUEUE_STOP; - netif_stop_queue(sp->dev); - } + + netif_tx_stop_all_queues(sp->dev); } static inline void s2io_start_all_tx_queue(struct s2io_nic *sp) { - int i; - if (sp->config.multiq) { - for (i = 0; i < sp->config.tx_fifo_num; i++) - netif_start_subqueue(sp->dev, i); - } else { + if (!sp->config.multiq) { + int i; + for (i = 0; i < sp->config.tx_fifo_num; i++) sp->mac_control.fifos[i].queue_state = FIFO_QUEUE_START; - netif_start_queue(sp->dev); } + netif_tx_start_all_queues(sp->dev); } static inline void s2io_start_tx_queue(struct s2io_nic *sp, int fifo_no) { - if (sp->config.multiq) - netif_start_subqueue(sp->dev, fifo_no); - else { + if (!sp->config.multiq) sp->mac_control.fifos[fifo_no].queue_state = FIFO_QUEUE_START; - netif_start_queue(sp->dev); - } + + netif_tx_start_all_queues(sp->dev); } static inline void s2io_wake_all_tx_queue(struct s2io_nic *sp) { - int i; - if (sp->config.multiq) { - for (i = 0; i < sp->config.tx_fifo_num; i++) - netif_wake_subqueue(sp->dev, i); - } else { + if (!sp->config.multiq) { + int i; + for (i = 0; i < sp->config.tx_fifo_num; i++) sp->mac_control.fifos[i].queue_state = FIFO_QUEUE_START; - netif_wake_queue(sp->dev); } + netif_tx_wake_all_queues(sp->dev); } static inline void s2io_wake_tx_queue( @@ -8691,5 +8681,5 @@ static void s2io_io_resume(struct pci_dev *pdev) } netif_device_attach(netdev); - netif_wake_queue(netdev); + netif_tx_wake_all_queues(netdev); } diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index c02227b9dd7b..b5c1e7df64fc 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -642,7 +642,13 @@ struct net_device struct netdev_queue rx_queue; struct netdev_queue *_tx ____cacheline_aligned_in_smp; + + /* Number of TX queues allocated at alloc_netdev_mq() time */ unsigned int num_tx_queues; + + /* Number of TX queues currently active in device */ + unsigned int real_num_tx_queues; + unsigned long tx_queue_len; /* Max frames per queue allowed */ /* @@ -1000,6 +1006,14 @@ static inline void netif_schedule(struct net_device *dev) netif_schedule_queue(netdev_get_tx_queue(dev, 0)); } +static inline void netif_tx_schedule_all(struct net_device *dev) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) + netif_schedule_queue(netdev_get_tx_queue(dev, i)); +} + /** * netif_start_queue - allow transmit * @dev: network device @@ -1016,6 +1030,16 @@ static inline void netif_start_queue(struct net_device *dev) netif_tx_start_queue(netdev_get_tx_queue(dev, 0)); } +static inline void netif_tx_start_all_queues(struct net_device *dev) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + netif_tx_start_queue(txq); + } +} + /** * netif_wake_queue - restart transmit * @dev: network device @@ -1040,6 +1064,16 @@ static inline void netif_wake_queue(struct net_device *dev) netif_tx_wake_queue(netdev_get_tx_queue(dev, 0)); } +static inline void netif_tx_wake_all_queues(struct net_device *dev) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + netif_tx_wake_queue(txq); + } +} + /** * netif_stop_queue - stop transmitted packets * @dev: network device @@ -1057,6 +1091,16 @@ static inline void netif_stop_queue(struct net_device *dev) netif_tx_stop_queue(netdev_get_tx_queue(dev, 0)); } +static inline void netif_tx_stop_all_queues(struct net_device *dev) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + netif_tx_stop_queue(txq); + } +} + /** * netif_queue_stopped - test if transmit queue is flowblocked * @dev: network device @@ -1100,7 +1144,8 @@ static inline int netif_running(const struct net_device *dev) */ static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index) { - clear_bit(__QUEUE_STATE_XOFF, &dev->egress_subqueue[queue_index].state); + struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); + clear_bit(__QUEUE_STATE_XOFF, &txq->state); } /** @@ -1112,11 +1157,12 @@ static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index) */ static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); #ifdef CONFIG_NETPOLL_TRAP if (netpoll_trap()) return; #endif - set_bit(__QUEUE_STATE_XOFF, &dev->egress_subqueue[queue_index].state); + set_bit(__QUEUE_STATE_XOFF, &txq->state); } /** @@ -1129,8 +1175,8 @@ static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index) static inline int __netif_subqueue_stopped(const struct net_device *dev, u16 queue_index) { - return test_bit(__QUEUE_STATE_XOFF, - &dev->egress_subqueue[queue_index].state); + struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); + return test_bit(__QUEUE_STATE_XOFF, &txq->state); } static inline int netif_subqueue_stopped(const struct net_device *dev, @@ -1148,13 +1194,13 @@ static inline int netif_subqueue_stopped(const struct net_device *dev, */ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); #ifdef CONFIG_NETPOLL_TRAP if (netpoll_trap()) return; #endif - if (test_and_clear_bit(__QUEUE_STATE_XOFF, - &dev->egress_subqueue[queue_index].state)) - __netif_schedule(netdev_get_tx_queue(dev, 0)); + if (test_and_clear_bit(__QUEUE_STATE_XOFF, &txq->state)) + __netif_schedule(txq); } /** @@ -1198,7 +1244,8 @@ extern int dev_set_mtu(struct net_device *, int); extern int dev_set_mac_address(struct net_device *, struct sockaddr *); extern int dev_hard_start_xmit(struct sk_buff *skb, - struct net_device *dev); + struct net_device *dev, + struct netdev_queue *txq); extern int netdev_budget; @@ -1447,6 +1494,12 @@ static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu) txq->xmit_lock_owner = cpu; } +static inline void __netif_tx_lock_bh(struct netdev_queue *txq) +{ + spin_lock_bh(&txq->_xmit_lock); + txq->xmit_lock_owner = smp_processor_id(); +} + static inline void netif_tx_lock(struct net_device *dev) { int cpu = smp_processor_id(); @@ -1483,6 +1536,12 @@ static inline void __netif_tx_unlock(struct netdev_queue *txq) spin_unlock(&txq->_xmit_lock); } +static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) +{ + txq->xmit_lock_owner = -1; + spin_unlock_bh(&txq->_xmit_lock); +} + static inline void netif_tx_unlock(struct net_device *dev) { unsigned int i; @@ -1514,8 +1573,13 @@ static inline void netif_tx_unlock_bh(struct net_device *dev) static inline void netif_tx_disable(struct net_device *dev) { + unsigned int i; + netif_tx_lock_bh(dev); - netif_stop_queue(dev); + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + netif_tx_stop_queue(txq); + } netif_tx_unlock_bh(dev); } diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index d58c1a5eb845..cb9527815606 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -88,9 +88,7 @@ extern void __qdisc_run(struct netdev_queue *txq); static inline void qdisc_run(struct netdev_queue *txq) { - struct net_device *dev = txq->dev; - - if (!netif_queue_stopped(dev) && + if (!netif_tx_queue_stopped(txq) && !test_and_set_bit(__QUEUE_STATE_QDISC_RUNNING, &txq->state)) __qdisc_run(txq); } diff --git a/net/core/dev.c b/net/core/dev.c index 69378f250695..f027a1ac4fbb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1598,7 +1598,8 @@ static int dev_gso_segment(struct sk_buff *skb) return 0; } -int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev) +int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, + struct netdev_queue *txq) { if (likely(!skb->next)) { if (!list_empty(&ptype_all)) @@ -1627,9 +1628,7 @@ gso: skb->next = nskb; return rc; } - if (unlikely((netif_queue_stopped(dev) || - netif_subqueue_stopped(dev, skb)) && - skb->next)) + if (unlikely(netif_tx_queue_stopped(txq) && skb->next)) return NETDEV_TX_BUSY; } while (skb->next); @@ -1669,7 +1668,10 @@ out_kfree_skb: static struct netdev_queue *dev_pick_tx(struct net_device *dev, struct sk_buff *skb) { - return netdev_get_tx_queue(dev, 0); + u16 queue_index = 0; + + skb_set_queue_mapping(skb, queue_index); + return netdev_get_tx_queue(dev, queue_index); } int dev_queue_xmit(struct sk_buff *skb) @@ -1737,8 +1739,6 @@ gso: spin_lock(&txq->lock); q = txq->qdisc; if (q->enqueue) { - /* reset queue_mapping to zero */ - skb_set_queue_mapping(skb, 0); rc = q->enqueue(skb, q); qdisc_run(txq); spin_unlock(&txq->lock); @@ -1768,10 +1768,9 @@ gso: HARD_TX_LOCK(dev, txq, cpu); - if (!netif_queue_stopped(dev) && - !netif_subqueue_stopped(dev, skb)) { + if (!netif_tx_queue_stopped(txq)) { rc = 0; - if (!dev_hard_start_xmit(skb, dev)) { + if (!dev_hard_start_xmit(skb, dev, txq)) { HARD_TX_UNLOCK(dev, txq); goto out; } @@ -4160,8 +4159,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name, BUG_ON(strlen(name) >= sizeof(dev->name)); - alloc_size = sizeof(struct net_device) + - sizeof(struct net_device_subqueue) * (queue_count - 1); + alloc_size = sizeof(struct net_device); if (sizeof_priv) { /* ensure 32-byte alignment of private area */ alloc_size = (alloc_size + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST; @@ -4191,16 +4189,14 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name, dev->_tx = tx; dev->num_tx_queues = queue_count; + dev->real_num_tx_queues = queue_count; if (sizeof_priv) { dev->priv = ((char *)dev + - ((sizeof(struct net_device) + - (sizeof(struct net_device_subqueue) * - (queue_count - 1)) + NETDEV_ALIGN_CONST) + ((sizeof(struct net_device) + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST)); } - dev->egress_subqueue_count = queue_count; dev->gso_max_size = GSO_MAX_SIZE; netdev_init_queues(dev); diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 8fb134da0346..c12720895ecf 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -58,25 +58,27 @@ static void queue_process(struct work_struct *work) while ((skb = skb_dequeue(&npinfo->txq))) { struct net_device *dev = skb->dev; + struct netdev_queue *txq; if (!netif_device_present(dev) || !netif_running(dev)) { __kfree_skb(skb); continue; } + txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); + local_irq_save(flags); - netif_tx_lock(dev); - if ((netif_queue_stopped(dev) || - netif_subqueue_stopped(dev, skb)) || - dev->hard_start_xmit(skb, dev) != NETDEV_TX_OK) { + __netif_tx_lock(txq, smp_processor_id()); + if (netif_tx_queue_stopped(txq) || + dev->hard_start_xmit(skb, dev) != NETDEV_TX_OK) { skb_queue_head(&npinfo->txq, skb); - netif_tx_unlock(dev); + __netif_tx_unlock(txq); local_irq_restore(flags); schedule_delayed_work(&npinfo->tx_work, HZ/10); return; } - netif_tx_unlock(dev); + __netif_tx_unlock(txq); local_irq_restore(flags); } } @@ -278,17 +280,19 @@ static void netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) /* don't get messages out of order, and no recursion */ if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) { + struct netdev_queue *txq; unsigned long flags; + txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); + local_irq_save(flags); /* try until next clock tick */ for (tries = jiffies_to_usecs(1)/USEC_PER_POLL; tries > 0; --tries) { - if (netif_tx_trylock(dev)) { - if (!netif_queue_stopped(dev) && - !netif_subqueue_stopped(dev, skb)) + if (__netif_tx_trylock(txq)) { + if (!netif_tx_queue_stopped(txq)) status = dev->hard_start_xmit(skb, dev); - netif_tx_unlock(dev); + __netif_tx_unlock(txq); if (status == NETDEV_TX_OK) break; diff --git a/net/core/pktgen.c b/net/core/pktgen.c index fdf537707e51..906802db4ed4 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -2123,6 +2123,24 @@ static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow) } } #endif +static void set_cur_queue_map(struct pktgen_dev *pkt_dev) +{ + if (pkt_dev->queue_map_min < pkt_dev->queue_map_max) { + __u16 t; + if (pkt_dev->flags & F_QUEUE_MAP_RND) { + t = random32() % + (pkt_dev->queue_map_max - + pkt_dev->queue_map_min + 1) + + pkt_dev->queue_map_min; + } else { + t = pkt_dev->cur_queue_map + 1; + if (t > pkt_dev->queue_map_max) + t = pkt_dev->queue_map_min; + } + pkt_dev->cur_queue_map = t; + } +} + /* Increment/randomize headers according to flags and current values * for IP src/dest, UDP src/dst port, MAC-Addr src/dst */ @@ -2325,19 +2343,7 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) pkt_dev->cur_pkt_size = t; } - if (pkt_dev->queue_map_min < pkt_dev->queue_map_max) { - __u16 t; - if (pkt_dev->flags & F_QUEUE_MAP_RND) { - t = random32() % - (pkt_dev->queue_map_max - pkt_dev->queue_map_min + 1) - + pkt_dev->queue_map_min; - } else { - t = pkt_dev->cur_queue_map + 1; - if (t > pkt_dev->queue_map_max) - t = pkt_dev->queue_map_min; - } - pkt_dev->cur_queue_map = t; - } + set_cur_queue_map(pkt_dev); pkt_dev->flows[flow].count++; } @@ -2458,7 +2464,7 @@ static struct sk_buff *fill_packet_ipv4(struct net_device *odev, __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ - + u16 queue_map; if (pkt_dev->nr_labels) protocol = htons(ETH_P_MPLS_UC); @@ -2469,6 +2475,7 @@ static struct sk_buff *fill_packet_ipv4(struct net_device *odev, /* Update any of the values, used when we're incrementing various * fields. */ + queue_map = pkt_dev->cur_queue_map; mod_cur_headers(pkt_dev); datalen = (odev->hard_header_len + 16) & ~0xf; @@ -2507,7 +2514,7 @@ static struct sk_buff *fill_packet_ipv4(struct net_device *odev, skb->network_header = skb->tail; skb->transport_header = skb->network_header + sizeof(struct iphdr); skb_put(skb, sizeof(struct iphdr) + sizeof(struct udphdr)); - skb_set_queue_mapping(skb, pkt_dev->cur_queue_map); + skb_set_queue_mapping(skb, queue_map); iph = ip_hdr(skb); udph = udp_hdr(skb); @@ -2797,6 +2804,7 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev, __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ + u16 queue_map; if (pkt_dev->nr_labels) protocol = htons(ETH_P_MPLS_UC); @@ -2807,6 +2815,7 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev, /* Update any of the values, used when we're incrementing various * fields. */ + queue_map = pkt_dev->cur_queue_map; mod_cur_headers(pkt_dev); skb = alloc_skb(pkt_dev->cur_pkt_size + 64 + 16 + @@ -2844,7 +2853,7 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev, skb->network_header = skb->tail; skb->transport_header = skb->network_header + sizeof(struct ipv6hdr); skb_put(skb, sizeof(struct ipv6hdr) + sizeof(struct udphdr)); - skb_set_queue_mapping(skb, pkt_dev->cur_queue_map); + skb_set_queue_mapping(skb, queue_map); iph = ipv6_hdr(skb); udph = udp_hdr(skb); @@ -3263,7 +3272,9 @@ static void pktgen_rem_thread(struct pktgen_thread *t) static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev) { struct net_device *odev = NULL; + struct netdev_queue *txq; __u64 idle_start = 0; + u16 queue_map; int ret; odev = pkt_dev->odev; @@ -3285,9 +3296,15 @@ static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev) } } - if ((netif_queue_stopped(odev) || - (pkt_dev->skb && - netif_subqueue_stopped(odev, pkt_dev->skb))) || + if (!pkt_dev->skb) { + set_cur_queue_map(pkt_dev); + queue_map = pkt_dev->cur_queue_map; + } else { + queue_map = skb_get_queue_mapping(pkt_dev->skb); + } + + txq = netdev_get_tx_queue(odev, queue_map); + if (netif_tx_queue_stopped(txq) || need_resched()) { idle_start = getCurUs(); @@ -3303,8 +3320,7 @@ static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev) pkt_dev->idle_acc += getCurUs() - idle_start; - if (netif_queue_stopped(odev) || - netif_subqueue_stopped(odev, pkt_dev->skb)) { + if (netif_tx_queue_stopped(txq)) { pkt_dev->next_tx_us = getCurUs(); /* TODO */ pkt_dev->next_tx_ns = 0; goto out; /* Try the next interface */ @@ -3331,9 +3347,12 @@ static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev) } } - netif_tx_lock_bh(odev); - if (!netif_queue_stopped(odev) && - !netif_subqueue_stopped(odev, pkt_dev->skb)) { + /* fill_packet() might have changed the queue */ + queue_map = skb_get_queue_mapping(pkt_dev->skb); + txq = netdev_get_tx_queue(odev, queue_map); + + __netif_tx_lock_bh(txq); + if (!netif_tx_queue_stopped(txq)) { atomic_inc(&(pkt_dev->skb->users)); retry_now: @@ -3377,7 +3396,7 @@ static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev) pkt_dev->next_tx_ns = 0; } - netif_tx_unlock_bh(odev); + __netif_tx_unlock_bh(txq); /* If pkt_dev->count is zero, then run forever */ if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) { diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 4e2b865cbba0..2f575b9017d1 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -166,7 +166,7 @@ static inline int qdisc_restart(struct netdev_queue *txq) HARD_TX_LOCK(dev, txq, smp_processor_id()); if (!netif_subqueue_stopped(dev, skb)) - ret = dev_hard_start_xmit(skb, dev); + ret = dev_hard_start_xmit(skb, dev, txq); HARD_TX_UNLOCK(dev, txq); spin_lock(&txq->lock); @@ -198,11 +198,10 @@ static inline int qdisc_restart(struct netdev_queue *txq) void __qdisc_run(struct netdev_queue *txq) { - struct net_device *dev = txq->dev; unsigned long start_time = jiffies; while (qdisc_restart(txq)) { - if (netif_queue_stopped(dev)) + if (netif_tx_queue_stopped(txq)) break; /* diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 44a2c3451f4d..ade3372221c7 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -295,8 +295,7 @@ restart: slave_txq = netdev_get_tx_queue(slave, 0); if (slave_txq->qdisc_sleeping != q) continue; - if (netif_queue_stopped(slave) || - __netif_subqueue_stopped(slave, subq) || + if (__netif_subqueue_stopped(slave, subq) || !netif_running(slave)) { busy = 1; continue; @@ -305,8 +304,7 @@ restart: switch (teql_resolve(skb, skb_res, slave)) { case 0: if (netif_tx_trylock(slave)) { - if (!netif_queue_stopped(slave) && - !__netif_subqueue_stopped(slave, subq) && + if (!__netif_subqueue_stopped(slave, subq) && slave->hard_start_xmit(skb, slave) == 0) { netif_tx_unlock(slave); master->slaves = NEXT_SLAVE(q); -- cgit v1.2.3 From eae792b722fef08dcf3aee88266ee7def9710757 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 15 Jul 2008 03:03:33 -0700 Subject: netdev: Add netdev->select_queue() method. Devices or device layers can set this to control the queue selection performed by dev_pick_tx(). This function runs under RCU protection, which allows overriding functions to have some way of synchronizing with things like dynamic ->real_num_tx_queues adjustments. This makes the spinlock prefetch in dev_queue_xmit() a little bit less effective, but that's the price right now for correctness. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 3 +++ net/core/dev.c | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index fdac1159253e..9464e6452967 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -724,6 +724,9 @@ struct net_device void (*poll_controller)(struct net_device *dev); #endif + u16 (*select_queue)(struct net_device *dev, + struct sk_buff *skb); + #ifdef CONFIG_NET_NS /* Network namespace this network device is inside */ struct net *nd_net; diff --git a/net/core/dev.c b/net/core/dev.c index f027a1ac4fbb..7ca9564d2f44 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1670,6 +1670,9 @@ static struct netdev_queue *dev_pick_tx(struct net_device *dev, { u16 queue_index = 0; + if (dev->select_queue) + queue_index = dev->select_queue(dev, skb); + skb_set_queue_mapping(skb, queue_index); return netdev_get_tx_queue(dev, queue_index); } @@ -1710,14 +1713,14 @@ int dev_queue_xmit(struct sk_buff *skb) } gso: - txq = dev_pick_tx(dev, skb); - spin_lock_prefetch(&txq->lock); - /* Disable soft irqs for various locks below. Also * stops preemption for RCU. */ rcu_read_lock_bh(); + txq = dev_pick_tx(dev, skb); + spin_lock_prefetch(&txq->lock); + /* Updates of qdisc are serialized by queue->lock. * The struct Qdisc which is pointed to by qdisc is now a * rcu structure - it may be accessed without acquiring -- cgit v1.2.3 From 8f0f2223cc08a5ae9a77f40edfe02e8a9f1abd77 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 15 Jul 2008 03:47:03 -0700 Subject: net: Implement simple sw TX hashing. It just xor hashes over IPv4/IPv6 addresses and ports of transport. The only assumption it makes is that skb_network_header() is set correctly. With bug fixes from Eric Dumazet. Signed-off-by: David S. Miller --- net/core/dev.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'net/core/dev.c') diff --git a/net/core/dev.c b/net/core/dev.c index 7ca9564d2f44..467bfb325123 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -121,6 +121,9 @@ #include #include #include +#include +#include +#include #include "net-sysfs.h" @@ -1665,6 +1668,53 @@ out_kfree_skb: * --BLG */ +static u16 simple_tx_hash(struct net_device *dev, struct sk_buff *skb) +{ + u32 *addr, *ports, hash, ihl; + u8 ip_proto; + int alen; + + switch (skb->protocol) { + case __constant_htons(ETH_P_IP): + ip_proto = ip_hdr(skb)->protocol; + addr = &ip_hdr(skb)->saddr; + ihl = ip_hdr(skb)->ihl; + alen = 2; + break; + case __constant_htons(ETH_P_IPV6): + ip_proto = ipv6_hdr(skb)->nexthdr; + addr = &ipv6_hdr(skb)->saddr.s6_addr32[0]; + ihl = (40 >> 2); + alen = 8; + break; + default: + return 0; + } + + ports = (u32 *) (skb_network_header(skb) + (ihl * 4)); + + hash = 0; + while (alen--) + hash ^= *addr++; + + switch (ip_proto) { + case IPPROTO_TCP: + case IPPROTO_UDP: + case IPPROTO_DCCP: + case IPPROTO_ESP: + case IPPROTO_AH: + case IPPROTO_SCTP: + case IPPROTO_UDPLITE: + hash ^= *ports; + break; + + default: + break; + } + + return hash % dev->real_num_tx_queues; +} + static struct netdev_queue *dev_pick_tx(struct net_device *dev, struct sk_buff *skb) { @@ -1672,6 +1722,8 @@ static struct netdev_queue *dev_pick_tx(struct net_device *dev, if (dev->select_queue) queue_index = dev->select_queue(dev, skb); + else if (dev->real_num_tx_queues > 1) + queue_index = simple_tx_hash(dev, skb); skb_set_queue_mapping(skb, queue_index); return netdev_get_tx_queue(dev, queue_index); -- cgit v1.2.3 From 37437bb2e1ae8af470dfcd5b4ff454110894ccaf Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Wed, 16 Jul 2008 02:15:04 -0700 Subject: pkt_sched: Schedule qdiscs instead of netdev_queue. When we have shared qdiscs, packets come out of the qdiscs for multiple transmit queues. Therefore it doesn't make any sense to schedule the transmit queue when logically we cannot know ahead of time the TX queue of the SKB that the qdisc->dequeue() will give us. Just for sanity I added a BUG check to make sure we never get into a state where the noop_qdisc is scheduled. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 12 ++++----- include/net/pkt_sched.h | 11 +++----- include/net/sch_generic.h | 2 ++ net/core/dev.c | 68 +++++++++++++++++++---------------------------- net/sched/sch_api.c | 3 +-- net/sched/sch_cbq.c | 2 +- net/sched/sch_generic.c | 30 ++++++++++----------- 7 files changed, 55 insertions(+), 73 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 9240a95793be..1e839fa01434 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -275,7 +275,6 @@ enum netdev_state_t { __LINK_STATE_START, __LINK_STATE_PRESENT, - __LINK_STATE_SCHED, __LINK_STATE_NOCARRIER, __LINK_STATE_LINKWATCH_PENDING, __LINK_STATE_DORMANT, @@ -452,7 +451,6 @@ struct netdev_queue { int xmit_lock_owner; struct Qdisc *qdisc_sleeping; struct list_head qdisc_list; - struct netdev_queue *next_sched; } ____cacheline_aligned_in_smp; /* @@ -969,7 +967,7 @@ static inline int unregister_gifconf(unsigned int family) */ struct softnet_data { - struct netdev_queue *output_queue; + struct Qdisc *output_queue; struct sk_buff_head input_pkt_queue; struct list_head poll_list; struct sk_buff *completion_queue; @@ -984,12 +982,12 @@ DECLARE_PER_CPU(struct softnet_data,softnet_data); #define HAVE_NETIF_QUEUE -extern void __netif_schedule(struct netdev_queue *txq); +extern void __netif_schedule(struct Qdisc *q); static inline void netif_schedule_queue(struct netdev_queue *txq) { if (!test_bit(__QUEUE_STATE_XOFF, &txq->state)) - __netif_schedule(txq); + __netif_schedule(txq->qdisc); } static inline void netif_tx_schedule_all(struct net_device *dev) @@ -1042,7 +1040,7 @@ static inline void netif_tx_wake_queue(struct netdev_queue *dev_queue) } #endif if (test_and_clear_bit(__QUEUE_STATE_XOFF, &dev_queue->state)) - __netif_schedule(dev_queue); + __netif_schedule(dev_queue->qdisc); } static inline void netif_wake_queue(struct net_device *dev) @@ -1186,7 +1184,7 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index) return; #endif if (test_and_clear_bit(__QUEUE_STATE_XOFF, &txq->state)) - __netif_schedule(txq); + __netif_schedule(txq->qdisc); } /** diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 06a442d85186..e4e30052e4e2 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -84,15 +84,12 @@ extern struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab); extern void qdisc_put_rtab(struct qdisc_rate_table *tab); -extern void __qdisc_run(struct netdev_queue *txq); +extern void __qdisc_run(struct Qdisc *q); -static inline void qdisc_run(struct netdev_queue *txq) +static inline void qdisc_run(struct Qdisc *q) { - struct Qdisc *q = txq->qdisc; - - if (!netif_tx_queue_stopped(txq) && - !test_and_set_bit(__QDISC_STATE_RUNNING, &q->state)) - __qdisc_run(txq); + if (!test_and_set_bit(__QDISC_STATE_RUNNING, &q->state)) + __qdisc_run(q); } extern int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp, diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 92417825d387..3cc4b5cd8c6a 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -26,6 +26,7 @@ struct qdisc_rate_table enum qdisc_state_t { __QDISC_STATE_RUNNING, + __QDISC_STATE_SCHED, }; struct Qdisc @@ -45,6 +46,7 @@ struct Qdisc struct sk_buff *gso_skb; struct sk_buff_head q; struct netdev_queue *dev_queue; + struct Qdisc *next_sched; struct list_head list; struct gnet_stats_basic bstats; diff --git a/net/core/dev.c b/net/core/dev.c index 467bfb325123..0b909b74f698 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1323,18 +1323,18 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) } -void __netif_schedule(struct netdev_queue *txq) +void __netif_schedule(struct Qdisc *q) { - struct net_device *dev = txq->dev; + BUG_ON(q == &noop_qdisc); - if (!test_and_set_bit(__LINK_STATE_SCHED, &dev->state)) { + if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) { struct softnet_data *sd; unsigned long flags; local_irq_save(flags); sd = &__get_cpu_var(softnet_data); - txq->next_sched = sd->output_queue; - sd->output_queue = txq; + q->next_sched = sd->output_queue; + sd->output_queue = q; raise_softirq_irqoff(NET_TX_SOFTIRQ); local_irq_restore(flags); } @@ -1771,37 +1771,23 @@ gso: rcu_read_lock_bh(); txq = dev_pick_tx(dev, skb); - spin_lock_prefetch(&txq->lock); - - /* Updates of qdisc are serialized by queue->lock. - * The struct Qdisc which is pointed to by qdisc is now a - * rcu structure - it may be accessed without acquiring - * a lock (but the structure may be stale.) The freeing of the - * qdisc will be deferred until it's known that there are no - * more references to it. - * - * If the qdisc has an enqueue function, we still need to - * hold the queue->lock before calling it, since queue->lock - * also serializes access to the device queue. - */ - q = rcu_dereference(txq->qdisc); + #ifdef CONFIG_NET_CLS_ACT skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS); #endif if (q->enqueue) { - /* Grab device queue */ - spin_lock(&txq->lock); - q = txq->qdisc; - if (q->enqueue) { - rc = q->enqueue(skb, q); - qdisc_run(txq); - spin_unlock(&txq->lock); - - rc = rc == NET_XMIT_BYPASS ? NET_XMIT_SUCCESS : rc; - goto out; - } - spin_unlock(&txq->lock); + spinlock_t *root_lock = qdisc_root_lock(q); + + spin_lock(root_lock); + + rc = q->enqueue(skb, q); + qdisc_run(q); + + spin_unlock(root_lock); + + rc = rc == NET_XMIT_BYPASS ? NET_XMIT_SUCCESS : rc; + goto out; } /* The device has no queue. Common case for software devices: @@ -1974,7 +1960,7 @@ static void net_tx_action(struct softirq_action *h) } if (sd->output_queue) { - struct netdev_queue *head; + struct Qdisc *head; local_irq_disable(); head = sd->output_queue; @@ -1982,18 +1968,20 @@ static void net_tx_action(struct softirq_action *h) local_irq_enable(); while (head) { - struct netdev_queue *txq = head; - struct net_device *dev = txq->dev; + struct Qdisc *q = head; + spinlock_t *root_lock; + head = head->next_sched; smp_mb__before_clear_bit(); - clear_bit(__LINK_STATE_SCHED, &dev->state); + clear_bit(__QDISC_STATE_SCHED, &q->state); - if (spin_trylock(&txq->lock)) { - qdisc_run(txq); - spin_unlock(&txq->lock); + root_lock = qdisc_root_lock(q); + if (spin_trylock(root_lock)) { + qdisc_run(q); + spin_unlock(root_lock); } else { - netif_schedule_queue(txq); + __netif_schedule(q); } } } @@ -4459,7 +4447,7 @@ static int dev_cpu_callback(struct notifier_block *nfb, void *ocpu) { struct sk_buff **list_skb; - struct netdev_queue **list_net; + struct Qdisc **list_net; struct sk_buff *skb; unsigned int cpu, oldcpu = (unsigned long)ocpu; struct softnet_data *sd, *oldsd; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 19c244a00839..8e8c5becc348 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -294,11 +294,10 @@ static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) { struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog, timer); - struct netdev_queue *txq = wd->qdisc->dev_queue; wd->qdisc->flags &= ~TCQ_F_THROTTLED; smp_wmb(); - netif_schedule_queue(txq); + __netif_schedule(wd->qdisc); return HRTIMER_NORESTART; } diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c index 37ae653db683..a3953bbe2d79 100644 --- a/net/sched/sch_cbq.c +++ b/net/sched/sch_cbq.c @@ -650,7 +650,7 @@ static enum hrtimer_restart cbq_undelay(struct hrtimer *timer) } sch->flags &= ~TCQ_F_THROTTLED; - netif_schedule_queue(sch->dev_queue); + __netif_schedule(sch); return HRTIMER_NORESTART; } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 739a8711ab30..dd5c4e70abe4 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -72,16 +72,14 @@ static inline int qdisc_qlen(struct Qdisc *q) return q->q.qlen; } -static inline int dev_requeue_skb(struct sk_buff *skb, - struct netdev_queue *dev_queue, - struct Qdisc *q) +static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) { if (unlikely(skb->next)) q->gso_skb = skb; else q->ops->requeue(skb, q); - netif_schedule_queue(dev_queue); + __netif_schedule(q); return 0; } @@ -121,7 +119,7 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, * some time. */ __get_cpu_var(netdev_rx_stat).cpu_collision++; - ret = dev_requeue_skb(skb, dev_queue, q); + ret = dev_requeue_skb(skb, q); } return ret; @@ -146,9 +144,9 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, * >0 - queue is not empty. * */ -static inline int qdisc_restart(struct netdev_queue *txq, - struct Qdisc *q) +static inline int qdisc_restart(struct Qdisc *q) { + struct netdev_queue *txq; int ret = NETDEV_TX_BUSY; struct net_device *dev; spinlock_t *root_lock; @@ -163,7 +161,8 @@ static inline int qdisc_restart(struct netdev_queue *txq, /* And release qdisc */ spin_unlock(root_lock); - dev = txq->dev; + dev = qdisc_dev(q); + txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); HARD_TX_LOCK(dev, txq, smp_processor_id()); if (!netif_subqueue_stopped(dev, skb)) @@ -189,29 +188,28 @@ static inline int qdisc_restart(struct netdev_queue *txq, printk(KERN_WARNING "BUG %s code %d qlen %d\n", dev->name, ret, q->q.qlen); - ret = dev_requeue_skb(skb, txq, q); + ret = dev_requeue_skb(skb, q); break; } + if (ret && netif_tx_queue_stopped(txq)) + ret = 0; + return ret; } -void __qdisc_run(struct netdev_queue *txq) +void __qdisc_run(struct Qdisc *q) { unsigned long start_time = jiffies; - struct Qdisc *q = txq->qdisc; - - while (qdisc_restart(txq, q)) { - if (netif_tx_queue_stopped(txq)) - break; + while (qdisc_restart(q)) { /* * Postpone processing if * 1. another process needs the CPU; * 2. we've been doing it for too long. */ if (need_resched() || jiffies != start_time) { - netif_schedule_queue(txq); + __netif_schedule(q); break; } } -- cgit v1.2.3 From ead81cc5fc6d996db6afb20f211241612610a07a Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Thu, 17 Jul 2008 00:50:32 -0700 Subject: netdevice: Move qdisc_list back into net_device proper. And give it it's own lock. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 3 ++- net/core/dev.c | 2 ++ net/sched/sch_api.c | 31 +++++++------------------------ net/sched/sch_generic.c | 9 +++++++-- 4 files changed, 18 insertions(+), 27 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 1e839fa01434..3170bcef734b 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -450,7 +450,6 @@ struct netdev_queue { spinlock_t _xmit_lock; int xmit_lock_owner; struct Qdisc *qdisc_sleeping; - struct list_head qdisc_list; } ____cacheline_aligned_in_smp; /* @@ -638,6 +637,8 @@ struct net_device unsigned int real_num_tx_queues; unsigned long tx_queue_len; /* Max frames per queue allowed */ + spinlock_t qdisc_list_lock; + struct list_head qdisc_list; /* * One part is mostly used on xmit path (device) diff --git a/net/core/dev.c b/net/core/dev.c index 0b909b74f698..6741e344ac59 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3886,6 +3886,8 @@ int register_netdevice(struct net_device *dev) net = dev_net(dev); spin_lock_init(&dev->addr_list_lock); + spin_lock_init(&dev->qdisc_list_lock); + INIT_LIST_HEAD(&dev->qdisc_list); netdev_init_queue_locks(dev); dev->iflink = -1; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 8e8c5becc348..6958fe7c9a77 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -183,30 +183,17 @@ EXPORT_SYMBOL(unregister_qdisc); (root qdisc, all its children, children of children etc.) */ -static struct Qdisc *__qdisc_lookup(struct netdev_queue *dev_queue, u32 handle) +struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) { struct Qdisc *q; - list_for_each_entry(q, &dev_queue->qdisc_list, list) { + list_for_each_entry(q, &dev->qdisc_list, list) { if (q->handle == handle) return q; } return NULL; } -struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) -{ - unsigned int i; - - for (i = 0; i < dev->num_tx_queues; i++) { - struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - struct Qdisc *q = __qdisc_lookup(txq, handle); - if (q) - return q; - } - return NULL; -} - static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid) { unsigned long cl; @@ -645,9 +632,9 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, goto err_out3; } } - qdisc_lock_tree(dev); - list_add_tail(&sch->list, &dev_queue->qdisc_list); - qdisc_unlock_tree(dev); + spin_lock_bh(&dev->qdisc_list_lock); + list_add_tail(&sch->list, &dev->qdisc_list); + spin_unlock_bh(&dev->qdisc_list_lock); return sch; } @@ -1032,14 +1019,12 @@ static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) read_lock(&dev_base_lock); idx = 0; for_each_netdev(&init_net, dev) { - struct netdev_queue *dev_queue; if (idx < s_idx) goto cont; if (idx > s_idx) s_q_idx = 0; q_idx = 0; - dev_queue = netdev_get_tx_queue(dev, 0); - list_for_each_entry(q, &dev_queue->qdisc_list, list) { + list_for_each_entry(q, &dev->qdisc_list, list) { if (q_idx < s_q_idx) { q_idx++; continue; @@ -1269,7 +1254,6 @@ static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walk static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) { struct net *net = sock_net(skb->sk); - struct netdev_queue *dev_queue; int t; int s_t; struct net_device *dev; @@ -1288,8 +1272,7 @@ static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) s_t = cb->args[0]; t = 0; - dev_queue = netdev_get_tx_queue(dev, 0); - list_for_each_entry(q, &dev_queue->qdisc_list, list) { + list_for_each_entry(q, &dev->qdisc_list, list) { if (t < s_t || !q->ops->cl_ops || (tcm->tcm_parent && TC_H_MAJ(tcm->tcm_parent) != q->handle)) { diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index efa418a1b34e..8cdf0b4a6a5a 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -563,11 +563,15 @@ static void __qdisc_destroy(struct rcu_head *head) void qdisc_destroy(struct Qdisc *qdisc) { + struct net_device *dev = qdisc_dev(qdisc); + if (qdisc->flags & TCQ_F_BUILTIN || !atomic_dec_and_test(&qdisc->refcnt)) return; + spin_lock_bh(&dev->qdisc_list_lock); list_del(&qdisc->list); + spin_unlock_bh(&dev->qdisc_list_lock); call_rcu(&qdisc->q_rcu, __qdisc_destroy); } @@ -599,7 +603,9 @@ static void attach_one_default_qdisc(struct net_device *dev, printk(KERN_INFO "%s: activation failed\n", dev->name); return; } - list_add_tail(&qdisc->list, &dev_queue->qdisc_list); + spin_lock_bh(&dev->qdisc_list_lock); + list_add_tail(&qdisc->list, &dev->qdisc_list); + spin_unlock_bh(&dev->qdisc_list_lock); } else { qdisc = &noqueue_qdisc; } @@ -738,7 +744,6 @@ static void dev_init_scheduler_queue(struct net_device *dev, dev_queue->qdisc = qdisc; dev_queue->qdisc_sleeping = qdisc; - INIT_LIST_HEAD(&dev_queue->qdisc_list); } void dev_init_scheduler(struct net_device *dev) -- cgit v1.2.3 From 83874000929ed63aef30b44083a9f713135ff040 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Thu, 17 Jul 2008 00:53:03 -0700 Subject: pkt_sched: Kill netdev_queue lock. We can simply use the qdisc->q.lock for all of the qdisc tree synchronization. Signed-off-by: David S. Miller --- drivers/net/ifb.c | 20 -------------------- include/linux/netdevice.h | 1 - include/net/sch_generic.h | 7 ++++++- net/core/dev.c | 9 +++++---- net/mac80211/wme.c | 19 ++++++++++++------- net/sched/sch_generic.c | 32 +++++++++++++++----------------- net/sched/sch_teql.c | 7 +++++-- 7 files changed, 43 insertions(+), 52 deletions(-) (limited to 'net/core/dev.c') diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c index 897b05e79ed0..0960e69b2da4 100644 --- a/drivers/net/ifb.c +++ b/drivers/net/ifb.c @@ -35,7 +35,6 @@ #include #include #include -#include #define TX_TIMEOUT (2*HZ) @@ -228,22 +227,6 @@ static struct rtnl_link_ops ifb_link_ops __read_mostly = { module_param(numifbs, int, 0); MODULE_PARM_DESC(numifbs, "Number of ifb devices"); -/* - * dev_ifb's TX queue lock is usually taken after dev->rx_queue.lock, - * reversely to e.g. qdisc_lock_tree(). It should be safe until - * ifb doesn't take dev's TX queue lock with dev_ifb->rx_queue.lock. - * But lockdep should know that ifb has different locks from dev. - */ -static struct lock_class_key ifb_tx_queue_lock_key; -static struct lock_class_key ifb_rx_queue_lock_key; - -static void set_tx_lockdep_key(struct net_device *dev, - struct netdev_queue *txq, - void *_unused) -{ - lockdep_set_class(&txq->lock, &ifb_tx_queue_lock_key); -} - static int __init ifb_init_one(int index) { struct net_device *dev_ifb; @@ -264,9 +247,6 @@ static int __init ifb_init_one(int index) if (err < 0) goto err; - netdev_for_each_tx_queue(dev_ifb, set_tx_lockdep_key, NULL); - lockdep_set_class(&dev_ifb->rx_queue.lock, &ifb_rx_queue_lock_key); - return 0; err: diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 3170bcef734b..9c5a68850114 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -443,7 +443,6 @@ enum netdev_queue_state_t }; struct netdev_queue { - spinlock_t lock; struct net_device *dev; struct Qdisc *qdisc; unsigned long state; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 1eef8d0c9990..2902a42564f0 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -163,6 +163,11 @@ struct tcf_proto struct tcf_proto_ops *ops; }; +static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) +{ + return &qdisc->q.lock; +} + static inline struct Qdisc *qdisc_root(struct Qdisc *qdisc) { return qdisc->dev_queue->qdisc; @@ -172,7 +177,7 @@ static inline spinlock_t *qdisc_root_lock(struct Qdisc *qdisc) { struct Qdisc *root = qdisc_root(qdisc); - return &root->dev_queue->lock; + return qdisc_lock(root); } static inline struct net_device *qdisc_dev(struct Qdisc *qdisc) diff --git a/net/core/dev.c b/net/core/dev.c index 6741e344ac59..32a13772c1cb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2080,10 +2080,12 @@ static int ing_filter(struct sk_buff *skb) rxq = &dev->rx_queue; - spin_lock(&rxq->lock); - if ((q = rxq->qdisc) != NULL) + q = rxq->qdisc; + if (q) { + spin_lock(qdisc_lock(q)); result = q->enqueue(skb, q); - spin_unlock(&rxq->lock); + spin_unlock(qdisc_lock(q)); + } return result; } @@ -4173,7 +4175,6 @@ static void netdev_init_one_queue(struct net_device *dev, struct netdev_queue *queue, void *_unused) { - spin_lock_init(&queue->lock); queue->dev = dev; } diff --git a/net/mac80211/wme.c b/net/mac80211/wme.c index b21cfec4b6ce..6e8099e77043 100644 --- a/net/mac80211/wme.c +++ b/net/mac80211/wme.c @@ -237,12 +237,14 @@ void ieee80211_ht_agg_queue_remove(struct ieee80211_local *local, ieee80211_requeue(local, agg_queue); } else { struct netdev_queue *txq; + spinlock_t *root_lock; txq = netdev_get_tx_queue(local->mdev, agg_queue); + root_lock = qdisc_root_lock(txq->qdisc); - spin_lock_bh(&txq->lock); + spin_lock_bh(root_lock); qdisc_reset(txq->qdisc); - spin_unlock_bh(&txq->lock); + spin_unlock_bh(root_lock); } } @@ -250,6 +252,7 @@ void ieee80211_requeue(struct ieee80211_local *local, int queue) { struct netdev_queue *txq = netdev_get_tx_queue(local->mdev, queue); struct sk_buff_head list; + spinlock_t *root_lock; struct Qdisc *qdisc; u32 len; @@ -261,14 +264,15 @@ void ieee80211_requeue(struct ieee80211_local *local, int queue) skb_queue_head_init(&list); - spin_lock(&txq->lock); + root_lock = qdisc_root_lock(qdisc); + spin_lock(root_lock); for (len = qdisc->q.qlen; len > 0; len--) { struct sk_buff *skb = qdisc->dequeue(qdisc); if (skb) __skb_queue_tail(&list, skb); } - spin_unlock(&txq->lock); + spin_unlock(root_lock); for (len = list.qlen; len > 0; len--) { struct sk_buff *skb = __skb_dequeue(&list); @@ -280,12 +284,13 @@ void ieee80211_requeue(struct ieee80211_local *local, int queue) txq = netdev_get_tx_queue(local->mdev, new_queue); - spin_lock(&txq->lock); qdisc = rcu_dereference(txq->qdisc); - qdisc->enqueue(skb, qdisc); + root_lock = qdisc_root_lock(qdisc); - spin_unlock(&txq->lock); + spin_lock(root_lock); + qdisc->enqueue(skb, qdisc); + spin_unlock(root_lock); } out_unlock: diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 3d53e92ad9c8..8fc580b3e173 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -96,15 +96,15 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, } /* - * NOTE: Called under queue->lock with locally disabled BH. + * NOTE: Called under qdisc_lock(q) with locally disabled BH. * * __QDISC_STATE_RUNNING guarantees only one CPU can process - * this qdisc at a time. queue->lock serializes queue accesses for - * this queue AND txq->qdisc pointer itself. + * this qdisc at a time. qdisc_lock(q) serializes queue accesses for + * this queue. * * netif_tx_lock serializes accesses to device driver. * - * queue->lock and netif_tx_lock are mutually exclusive, + * qdisc_lock(q) and netif_tx_lock are mutually exclusive, * if one is grabbed, another must be free. * * Note, that this procedure can be called by a watchdog timer @@ -317,7 +317,6 @@ struct Qdisc_ops noop_qdisc_ops __read_mostly = { }; static struct netdev_queue noop_netdev_queue = { - .lock = __SPIN_LOCK_UNLOCKED(noop_netdev_queue.lock), .qdisc = &noop_qdisc, }; @@ -327,6 +326,7 @@ struct Qdisc noop_qdisc = { .flags = TCQ_F_BUILTIN, .ops = &noop_qdisc_ops, .list = LIST_HEAD_INIT(noop_qdisc.list), + .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock), .dev_queue = &noop_netdev_queue, }; EXPORT_SYMBOL(noop_qdisc); @@ -498,7 +498,7 @@ errout: } EXPORT_SYMBOL(qdisc_create_dflt); -/* Under queue->lock and BH! */ +/* Under qdisc_root_lock(qdisc) and BH! */ void qdisc_reset(struct Qdisc *qdisc) { @@ -526,10 +526,12 @@ static void __qdisc_destroy(struct rcu_head *head) module_put(ops->owner); dev_put(qdisc_dev(qdisc)); + kfree_skb(qdisc->gso_skb); + kfree((char *) qdisc - qdisc->padded); } -/* Under queue->lock and BH! */ +/* Under qdisc_root_lock(qdisc) and BH! */ void qdisc_destroy(struct Qdisc *qdisc) { @@ -586,13 +588,12 @@ static void transition_one_qdisc(struct net_device *dev, struct netdev_queue *dev_queue, void *_need_watchdog) { + struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping; int *need_watchdog_p = _need_watchdog; - spin_lock_bh(&dev_queue->lock); - rcu_assign_pointer(dev_queue->qdisc, dev_queue->qdisc_sleeping); - if (dev_queue->qdisc != &noqueue_qdisc) + rcu_assign_pointer(dev_queue->qdisc, new_qdisc); + if (new_qdisc != &noqueue_qdisc) *need_watchdog_p = 1; - spin_unlock_bh(&dev_queue->lock); } void dev_activate(struct net_device *dev) @@ -629,19 +630,16 @@ static void dev_deactivate_queue(struct net_device *dev, struct sk_buff *skb = NULL; struct Qdisc *qdisc; - spin_lock_bh(&dev_queue->lock); - qdisc = dev_queue->qdisc; if (qdisc) { + spin_lock_bh(qdisc_lock(qdisc)); + dev_queue->qdisc = qdisc_default; qdisc_reset(qdisc); - skb = qdisc->gso_skb; - qdisc->gso_skb = NULL; + spin_unlock_bh(qdisc_lock(qdisc)); } - spin_unlock_bh(&dev_queue->lock); - kfree_skb(skb); } diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index ade3372221c7..8b0ff345f9da 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -156,12 +156,15 @@ teql_destroy(struct Qdisc* sch) master->slaves = NEXT_SLAVE(q); if (q == master->slaves) { struct netdev_queue *txq; + spinlock_t *root_lock; txq = netdev_get_tx_queue(master->dev, 0); master->slaves = NULL; - spin_lock_bh(&txq->lock); + + root_lock = qdisc_root_lock(txq->qdisc); + spin_lock_bh(root_lock); qdisc_reset(txq->qdisc); - spin_unlock_bh(&txq->lock); + spin_unlock_bh(root_lock); } } skb_queue_purge(&dat->q); -- cgit v1.2.3 From 3072367300aa8c779e3a14ee8e89de079e90f3ad Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Fri, 18 Jul 2008 22:50:15 -0700 Subject: pkt_sched: Manage qdisc list inside of root qdisc. Idea is from Patrick McHardy. Instead of managing the list of qdiscs on the device level, manage it in the root qdisc of a netdev_queue. This solves all kinds of visibility issues during qdisc destruction. The way to iterate over all qdiscs of a netdev_queue is to visit the netdev_queue->qdisc, and then traverse it's list. The only special case is to ignore builting qdiscs at the root when dumping or doing a qdisc_lookup(). That was not needed previously because builtin qdiscs were not added to the device's qdisc_list. Signed-off-by: David S. Miller --- include/linux/netdevice.h | 2 - net/core/dev.c | 2 - net/sched/sch_api.c | 175 ++++++++++++++++++++++++++++++++++------------ net/sched/sch_generic.c | 10 +-- 4 files changed, 133 insertions(+), 56 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 9c5a68850114..812bcd8b4363 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -636,8 +636,6 @@ struct net_device unsigned int real_num_tx_queues; unsigned long tx_queue_len; /* Max frames per queue allowed */ - spinlock_t qdisc_list_lock; - struct list_head qdisc_list; /* * One part is mostly used on xmit path (device) diff --git a/net/core/dev.c b/net/core/dev.c index e54acde839da..065b9817e209 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3888,8 +3888,6 @@ int register_netdevice(struct net_device *dev) net = dev_net(dev); spin_lock_init(&dev->addr_list_lock); - spin_lock_init(&dev->qdisc_list_lock); - INIT_LIST_HEAD(&dev->qdisc_list); netdev_init_queue_locks(dev); dev->iflink = -1; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index b3ef8307204e..fb43731c9860 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -185,11 +185,20 @@ EXPORT_SYMBOL(unregister_qdisc); struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) { - struct Qdisc *q; + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + struct Qdisc *q, *txq_root = txq->qdisc; - list_for_each_entry(q, &dev->qdisc_list, list) { - if (q->handle == handle) - return q; + if (!(txq_root->flags & TCQ_F_BUILTIN) && + txq_root->handle == handle) + return txq_root; + + list_for_each_entry(q, &txq_root->list, list) { + if (q->handle == handle) + return q; + } } return NULL; } @@ -676,9 +685,8 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, goto err_out3; } } - spin_lock_bh(&dev->qdisc_list_lock); - list_add_tail(&sch->list, &dev->qdisc_list); - spin_unlock_bh(&dev->qdisc_list_lock); + if (parent) + list_add_tail(&sch->list, &dev_queue->qdisc->list); return sch; } @@ -1037,13 +1045,57 @@ err_out: return -EINVAL; } +static bool tc_qdisc_dump_ignore(struct Qdisc *q) +{ + return (q->flags & TCQ_F_BUILTIN) ? true : false; +} + +static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb, + struct netlink_callback *cb, + int *q_idx_p, int s_q_idx) +{ + int ret = 0, q_idx = *q_idx_p; + struct Qdisc *q; + + if (!root) + return 0; + + q = root; + if (q_idx < s_q_idx) { + q_idx++; + } else { + if (!tc_qdisc_dump_ignore(q) && + tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid, + cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0) + goto done; + q_idx++; + } + list_for_each_entry(q, &root->list, list) { + if (q_idx < s_q_idx) { + q_idx++; + continue; + } + if (!tc_qdisc_dump_ignore(q) && + tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid, + cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0) + goto done; + q_idx++; + } + +out: + *q_idx_p = q_idx; + return ret; +done: + ret = -1; + goto out; +} + static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) { struct net *net = sock_net(skb->sk); int idx, q_idx; int s_idx, s_q_idx; struct net_device *dev; - struct Qdisc *q; if (net != &init_net) return 0; @@ -1053,21 +1105,22 @@ static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) read_lock(&dev_base_lock); idx = 0; for_each_netdev(&init_net, dev) { + struct netdev_queue *dev_queue; + if (idx < s_idx) goto cont; if (idx > s_idx) s_q_idx = 0; q_idx = 0; - list_for_each_entry(q, &dev->qdisc_list, list) { - if (q_idx < s_q_idx) { - q_idx++; - continue; - } - if (tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid, - cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0) - goto done; - q_idx++; - } + + dev_queue = netdev_get_tx_queue(dev, 0); + if (tc_dump_qdisc_root(dev_queue->qdisc, skb, cb, &q_idx, s_q_idx) < 0) + goto done; + + dev_queue = &dev->rx_queue; + if (tc_dump_qdisc_root(dev_queue->qdisc, skb, cb, &q_idx, s_q_idx) < 0) + goto done; + cont: idx++; } @@ -1285,15 +1338,62 @@ static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walk a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS); } +static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb, + struct tcmsg *tcm, struct netlink_callback *cb, + int *t_p, int s_t) +{ + struct qdisc_dump_args arg; + + if (tc_qdisc_dump_ignore(q) || + *t_p < s_t || !q->ops->cl_ops || + (tcm->tcm_parent && + TC_H_MAJ(tcm->tcm_parent) != q->handle)) { + (*t_p)++; + return 0; + } + if (*t_p > s_t) + memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0])); + arg.w.fn = qdisc_class_dump; + arg.skb = skb; + arg.cb = cb; + arg.w.stop = 0; + arg.w.skip = cb->args[1]; + arg.w.count = 0; + q->ops->cl_ops->walk(q, &arg.w); + cb->args[1] = arg.w.count; + if (arg.w.stop) + return -1; + (*t_p)++; + return 0; +} + +static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, + struct tcmsg *tcm, struct netlink_callback *cb, + int *t_p, int s_t) +{ + struct Qdisc *q; + + if (!root) + return 0; + + if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0) + return -1; + + list_for_each_entry(q, &root->list, list) { + if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) + return -1; + } + + return 0; +} + static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) { + struct tcmsg *tcm = (struct tcmsg*)NLMSG_DATA(cb->nlh); struct net *net = sock_net(skb->sk); - int t; - int s_t; + struct netdev_queue *dev_queue; struct net_device *dev; - struct Qdisc *q; - struct tcmsg *tcm = (struct tcmsg*)NLMSG_DATA(cb->nlh); - struct qdisc_dump_args arg; + int t, s_t; if (net != &init_net) return 0; @@ -1306,28 +1406,15 @@ static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) s_t = cb->args[0]; t = 0; - list_for_each_entry(q, &dev->qdisc_list, list) { - if (t < s_t || !q->ops->cl_ops || - (tcm->tcm_parent && - TC_H_MAJ(tcm->tcm_parent) != q->handle)) { - t++; - continue; - } - if (t > s_t) - memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0])); - arg.w.fn = qdisc_class_dump; - arg.skb = skb; - arg.cb = cb; - arg.w.stop = 0; - arg.w.skip = cb->args[1]; - arg.w.count = 0; - q->ops->cl_ops->walk(q, &arg.w); - cb->args[1] = arg.w.count; - if (arg.w.stop) - break; - t++; - } + dev_queue = netdev_get_tx_queue(dev, 0); + if (tc_dump_tclass_root(dev_queue->qdisc, skb, tcm, cb, &t, s_t) < 0) + goto done; + + dev_queue = &dev->rx_queue; + if (tc_dump_tclass_root(dev_queue->qdisc, skb, tcm, cb, &t, s_t) < 0) + goto done; +done: cb->args[0] = t; dev_put(dev); diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index e244c462e6bd..14cc443d0490 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -480,15 +480,12 @@ static void __qdisc_destroy(struct rcu_head *head) void qdisc_destroy(struct Qdisc *qdisc) { - struct net_device *dev = qdisc_dev(qdisc); - if (qdisc->flags & TCQ_F_BUILTIN || !atomic_dec_and_test(&qdisc->refcnt)) return; - spin_lock_bh(&dev->qdisc_list_lock); - list_del(&qdisc->list); - spin_unlock_bh(&dev->qdisc_list_lock); + if (qdisc->parent) + list_del(&qdisc->list); call_rcu(&qdisc->q_rcu, __qdisc_destroy); } @@ -520,9 +517,6 @@ static void attach_one_default_qdisc(struct net_device *dev, printk(KERN_INFO "%s: activation failed\n", dev->name); return; } - spin_lock_bh(&dev->qdisc_list_lock); - list_add_tail(&qdisc->list, &dev->qdisc_list); - spin_unlock_bh(&dev->qdisc_list_lock); } else { qdisc = &noqueue_qdisc; } -- cgit v1.2.3 From 5f86173bdf15981ca49d0434f638b68f70a35644 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Sun, 20 Jul 2008 00:08:04 -0700 Subject: net_sched: Add qdisc_enqueue wrapper Signed-off-by: Jussi Kivilinna Signed-off-by: David S. Miller --- include/net/sch_generic.h | 10 ++++++++++ net/core/dev.c | 4 ++-- net/mac80211/wme.c | 2 +- net/sched/sch_atm.c | 2 +- net/sched/sch_cbq.c | 5 +++-- net/sched/sch_dsmark.c | 2 +- net/sched/sch_hfsc.c | 2 +- net/sched/sch_htb.c | 3 +-- net/sched/sch_netem.c | 20 ++++++++++++-------- net/sched/sch_prio.c | 3 ++- net/sched/sch_red.c | 2 +- net/sched/sch_tbf.c | 3 ++- 12 files changed, 37 insertions(+), 21 deletions(-) (limited to 'net/core/dev.c') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 8a44386b35cf..f396dff335a3 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -306,6 +306,16 @@ static inline bool qdisc_tx_is_noop(const struct net_device *dev) return true; } +static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch) +{ + return sch->enqueue(skb, sch); +} + +static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch) +{ + return qdisc_enqueue(skb, sch); +} + static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff_head *list) { diff --git a/net/core/dev.c b/net/core/dev.c index 065b9817e209..2eed17bcb2dd 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1781,7 +1781,7 @@ gso: spin_lock(root_lock); - rc = q->enqueue(skb, q); + rc = qdisc_enqueue_root(skb, q); qdisc_run(q); spin_unlock(root_lock); @@ -2083,7 +2083,7 @@ static int ing_filter(struct sk_buff *skb) q = rxq->qdisc; if (q) { spin_lock(qdisc_lock(q)); - result = q->enqueue(skb, q); + result = qdisc_enqueue_root(skb, q); spin_unlock(qdisc_lock(q)); } diff --git a/net/mac80211/wme.c b/net/mac80211/wme.c index 6e8099e77043..07edda0b8a5c 100644 --- a/net/mac80211/wme.c +++ b/net/mac80211/wme.c @@ -289,7 +289,7 @@ void ieee80211_requeue(struct ieee80211_local *local, int queue) root_lock = qdisc_root_lock(qdisc); spin_lock(root_lock); - qdisc->enqueue(skb, qdisc); + qdisc_enqueue_root(skb, qdisc); spin_unlock(root_lock); } diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c index 0de757e3be4a..68ed35e2a763 100644 --- a/net/sched/sch_atm.c +++ b/net/sched/sch_atm.c @@ -429,7 +429,7 @@ static int atm_tc_enqueue(struct sk_buff *skb, struct Qdisc *sch) #endif } - ret = flow->q->enqueue(skb, flow->q); + ret = qdisc_enqueue(skb, flow->q); if (ret != 0) { drop: __maybe_unused sch->qstats.drops++; diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c index a3953bbe2d79..1afe3eece627 100644 --- a/net/sched/sch_cbq.c +++ b/net/sched/sch_cbq.c @@ -387,7 +387,8 @@ cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch) #ifdef CONFIG_NET_CLS_ACT cl->q->__parent = sch; #endif - if ((ret = cl->q->enqueue(skb, cl->q)) == NET_XMIT_SUCCESS) { + ret = qdisc_enqueue(skb, cl->q); + if (ret == NET_XMIT_SUCCESS) { sch->q.qlen++; sch->bstats.packets++; sch->bstats.bytes+=len; @@ -671,7 +672,7 @@ static int cbq_reshape_fail(struct sk_buff *skb, struct Qdisc *child) q->rx_class = cl; cl->q->__parent = sch; - if (cl->q->enqueue(skb, cl->q) == 0) { + if (qdisc_enqueue(skb, cl->q) == 0) { sch->q.qlen++; sch->bstats.packets++; sch->bstats.bytes+=len; diff --git a/net/sched/sch_dsmark.c b/net/sched/sch_dsmark.c index 3aafbd17393a..44d347e831cf 100644 --- a/net/sched/sch_dsmark.c +++ b/net/sched/sch_dsmark.c @@ -252,7 +252,7 @@ static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch) } } - err = p->q->enqueue(skb, p->q); + err = qdisc_enqueue(skb, p->q); if (err != NET_XMIT_SUCCESS) { sch->qstats.drops++; return err; diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 5090708ba384..fd61ed6ee1e7 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -1586,7 +1586,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch) } len = skb->len; - err = cl->qdisc->enqueue(skb, cl->qdisc); + err = qdisc_enqueue(skb, cl->qdisc); if (unlikely(err != NET_XMIT_SUCCESS)) { cl->qstats.drops++; sch->qstats.drops++; diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index ee48457eaa4a..72b5a946178f 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -572,8 +572,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch) kfree_skb(skb); return ret; #endif - } else if (cl->un.leaf.q->enqueue(skb, cl->un.leaf.q) != - NET_XMIT_SUCCESS) { + } else if (qdisc_enqueue(skb, cl->un.leaf.q) != NET_XMIT_SUCCESS) { sch->qstats.drops++; cl->qstats.drops++; return NET_XMIT_DROP; diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index c5ea40c9eb21..13c4821e42b8 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -82,6 +82,12 @@ struct netem_skb_cb { psched_time_t time_to_send; }; +static inline struct netem_skb_cb *netem_skb_cb(struct sk_buff *skb) +{ + BUILD_BUG_ON(sizeof(skb->cb) < sizeof(struct netem_skb_cb)); + return (struct netem_skb_cb *)skb->cb; +} + /* init_crandom - initialize correlated random number generator * Use entropy source for initial seed. */ @@ -184,7 +190,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch) u32 dupsave = q->duplicate; /* prevent duplicating a dup... */ q->duplicate = 0; - rootq->enqueue(skb2, rootq); + qdisc_enqueue_root(skb2, rootq); q->duplicate = dupsave; } @@ -205,7 +211,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch) skb->data[net_random() % skb_headlen(skb)] ^= 1<<(net_random() % 8); } - cb = (struct netem_skb_cb *)skb->cb; + cb = netem_skb_cb(skb); if (q->gap == 0 /* not doing reordering */ || q->counter < q->gap /* inside last reordering gap */ || q->reorder < get_crandom(&q->reorder_cor)) { @@ -218,7 +224,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch) now = psched_get_time(); cb->time_to_send = now + delay; ++q->counter; - ret = q->qdisc->enqueue(skb, q->qdisc); + ret = qdisc_enqueue(skb, q->qdisc); } else { /* * Do re-ordering by putting one out of N packets at the front @@ -277,8 +283,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch) skb = q->qdisc->dequeue(q->qdisc); if (skb) { - const struct netem_skb_cb *cb - = (const struct netem_skb_cb *)skb->cb; + const struct netem_skb_cb *cb = netem_skb_cb(skb); psched_time_t now = psched_get_time(); /* if more time remaining? */ @@ -457,7 +462,7 @@ static int tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch) { struct fifo_sched_data *q = qdisc_priv(sch); struct sk_buff_head *list = &sch->q; - psched_time_t tnext = ((struct netem_skb_cb *)nskb->cb)->time_to_send; + psched_time_t tnext = netem_skb_cb(nskb)->time_to_send; struct sk_buff *skb; if (likely(skb_queue_len(list) < q->limit)) { @@ -468,8 +473,7 @@ static int tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch) } skb_queue_reverse_walk(list, skb) { - const struct netem_skb_cb *cb - = (const struct netem_skb_cb *)skb->cb; + const struct netem_skb_cb *cb = netem_skb_cb(skb); if (tnext >= cb->time_to_send) break; diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 536ca474dc69..d29c2f87fc0b 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -81,7 +81,8 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch) } #endif - if ((ret = qdisc->enqueue(skb, qdisc)) == NET_XMIT_SUCCESS) { + ret = qdisc_enqueue(skb, qdisc); + if (ret == NET_XMIT_SUCCESS) { sch->bstats.bytes += skb->len; sch->bstats.packets++; sch->q.qlen++; diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 77098acf0adc..b48a391bc129 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -92,7 +92,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc* sch) break; } - ret = child->enqueue(skb, child); + ret = qdisc_enqueue(skb, child); if (likely(ret == NET_XMIT_SUCCESS)) { sch->bstats.bytes += skb->len; sch->bstats.packets++; diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c index 444c227fcb6b..7d705b86dae5 100644 --- a/net/sched/sch_tbf.c +++ b/net/sched/sch_tbf.c @@ -133,7 +133,8 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc* sch) return NET_XMIT_DROP; } - if ((ret = q->qdisc->enqueue(skb, q->qdisc)) != 0) { + ret = qdisc_enqueue(skb, q->qdisc); + if (ret != 0) { sch->qstats.drops++; return ret; } -- cgit v1.2.3