summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/netdevice.h4
-rw-r--r--net/core/dev.c149
2 files changed, 82 insertions, 71 deletions
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 83ab3da149ad..3c5ed5f5274e 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1401,10 +1401,10 @@ struct softnet_data {
struct napi_struct backlog;
};
-static inline void incr_input_queue_head(struct softnet_data *queue)
+static inline void input_queue_head_incr(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
- queue->input_queue_head++;
+ sd->input_queue_head++;
#endif
}
diff --git a/net/core/dev.c b/net/core/dev.c
index 05a2b294906b..7f5755b0a57c 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -208,17 +208,17 @@ static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
}
-static inline void rps_lock(struct softnet_data *queue)
+static inline void rps_lock(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
- spin_lock(&queue->input_pkt_queue.lock);
+ spin_lock(&sd->input_pkt_queue.lock);
#endif
}
-static inline void rps_unlock(struct softnet_data *queue)
+static inline void rps_unlock(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
- spin_unlock(&queue->input_pkt_queue.lock);
+ spin_unlock(&sd->input_pkt_queue.lock);
#endif
}
@@ -2346,63 +2346,74 @@ done:
}
/* Called from hardirq (IPI) context */
-static void trigger_softirq(void *data)
+static void rps_trigger_softirq(void *data)
{
- struct softnet_data *queue = data;
- __napi_schedule(&queue->backlog);
+ struct softnet_data *sd = data;
+
+ __napi_schedule(&sd->backlog);
__get_cpu_var(netdev_rx_stat).received_rps++;
}
+
#endif /* CONFIG_RPS */
/*
+ * Check if this softnet_data structure is another cpu one
+ * If yes, queue it to our IPI list and return 1
+ * If no, return 0
+ */
+static int rps_ipi_queued(struct softnet_data *sd)
+{
+#ifdef CONFIG_RPS
+ struct softnet_data *mysd = &__get_cpu_var(softnet_data);
+
+ if (sd != mysd) {
+ sd->rps_ipi_next = mysd->rps_ipi_list;
+ mysd->rps_ipi_list = sd;
+
+ __raise_softirq_irqoff(NET_RX_SOFTIRQ);
+ return 1;
+ }
+#endif /* CONFIG_RPS */
+ return 0;
+}
+
+/*
* enqueue_to_backlog is called to queue an skb to a per CPU backlog
* queue (may be a remote CPU queue).
*/
static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
unsigned int *qtail)
{
- struct softnet_data *queue;
+ struct softnet_data *sd;
unsigned long flags;
- queue = &per_cpu(softnet_data, cpu);
+ sd = &per_cpu(softnet_data, cpu);
local_irq_save(flags);
__get_cpu_var(netdev_rx_stat).total++;
- rps_lock(queue);
- if (queue->input_pkt_queue.qlen <= netdev_max_backlog) {
- if (queue->input_pkt_queue.qlen) {
+ rps_lock(sd);
+ if (sd->input_pkt_queue.qlen <= netdev_max_backlog) {
+ if (sd->input_pkt_queue.qlen) {
enqueue:
- __skb_queue_tail(&queue->input_pkt_queue, skb);
+ __skb_queue_tail(&sd->input_pkt_queue, skb);
#ifdef CONFIG_RPS
- *qtail = queue->input_queue_head +
- queue->input_pkt_queue.qlen;
+ *qtail = sd->input_queue_head + sd->input_pkt_queue.qlen;
#endif
- rps_unlock(queue);
+ rps_unlock(sd);
local_irq_restore(flags);
return NET_RX_SUCCESS;
}
/* Schedule NAPI for backlog device */
- if (napi_schedule_prep(&queue->backlog)) {
-#ifdef CONFIG_RPS
- if (cpu != smp_processor_id()) {
- struct softnet_data *myqueue;
-
- myqueue = &__get_cpu_var(softnet_data);
- queue->rps_ipi_next = myqueue->rps_ipi_list;
- myqueue->rps_ipi_list = queue;
-
- __raise_softirq_irqoff(NET_RX_SOFTIRQ);
- goto enqueue;
- }
-#endif
- __napi_schedule(&queue->backlog);
+ if (napi_schedule_prep(&sd->backlog)) {
+ if (!rps_ipi_queued(sd))
+ __napi_schedule(&sd->backlog);
}
goto enqueue;
}
- rps_unlock(queue);
+ rps_unlock(sd);
__get_cpu_var(netdev_rx_stat).dropped++;
local_irq_restore(flags);
@@ -2903,17 +2914,17 @@ EXPORT_SYMBOL(netif_receive_skb);
static void flush_backlog(void *arg)
{
struct net_device *dev = arg;
- struct softnet_data *queue = &__get_cpu_var(softnet_data);
+ struct softnet_data *sd = &__get_cpu_var(softnet_data);
struct sk_buff *skb, *tmp;
- rps_lock(queue);
- skb_queue_walk_safe(&queue->input_pkt_queue, skb, tmp)
+ rps_lock(sd);
+ skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp)
if (skb->dev == dev) {
- __skb_unlink(skb, &queue->input_pkt_queue);
+ __skb_unlink(skb, &sd->input_pkt_queue);
kfree_skb(skb);
- incr_input_queue_head(queue);
+ input_queue_head_incr(sd);
}
- rps_unlock(queue);
+ rps_unlock(sd);
}
static int napi_gro_complete(struct sk_buff *skb)
@@ -3219,23 +3230,23 @@ EXPORT_SYMBOL(napi_gro_frags);
static int process_backlog(struct napi_struct *napi, int quota)
{
int work = 0;
- struct softnet_data *queue = &__get_cpu_var(softnet_data);
+ struct softnet_data *sd = &__get_cpu_var(softnet_data);
napi->weight = weight_p;
do {
struct sk_buff *skb;
local_irq_disable();
- rps_lock(queue);
- skb = __skb_dequeue(&queue->input_pkt_queue);
+ rps_lock(sd);
+ skb = __skb_dequeue(&sd->input_pkt_queue);
if (!skb) {
__napi_complete(napi);
- rps_unlock(queue);
+ rps_unlock(sd);
local_irq_enable();
break;
}
- incr_input_queue_head(queue);
- rps_unlock(queue);
+ input_queue_head_incr(sd);
+ rps_unlock(sd);
local_irq_enable();
__netif_receive_skb(skb);
@@ -3331,24 +3342,25 @@ EXPORT_SYMBOL(netif_napi_del);
* net_rps_action sends any pending IPI's for rps.
* Note: called with local irq disabled, but exits with local irq enabled.
*/
-static void net_rps_action(void)
+static void net_rps_action_and_irq_disable(void)
{
#ifdef CONFIG_RPS
- struct softnet_data *locqueue = &__get_cpu_var(softnet_data);
- struct softnet_data *remqueue = locqueue->rps_ipi_list;
+ struct softnet_data *sd = &__get_cpu_var(softnet_data);
+ struct softnet_data *remsd = sd->rps_ipi_list;
- if (remqueue) {
- locqueue->rps_ipi_list = NULL;
+ if (remsd) {
+ sd->rps_ipi_list = NULL;
local_irq_enable();
/* Send pending IPI's to kick RPS processing on remote cpus. */
- while (remqueue) {
- struct softnet_data *next = remqueue->rps_ipi_next;
- if (cpu_online(remqueue->cpu))
- __smp_call_function_single(remqueue->cpu,
- &remqueue->csd, 0);
- remqueue = next;
+ while (remsd) {
+ struct softnet_data *next = remsd->rps_ipi_next;
+
+ if (cpu_online(remsd->cpu))
+ __smp_call_function_single(remsd->cpu,
+ &remsd->csd, 0);
+ remsd = next;
}
} else
#endif
@@ -3423,7 +3435,7 @@ static void net_rx_action(struct softirq_action *h)
netpoll_poll_unlock(have);
}
out:
- net_rps_action();
+ net_rps_action_and_irq_disable();
#ifdef CONFIG_NET_DMA
/*
@@ -5595,7 +5607,7 @@ static int dev_cpu_callback(struct notifier_block *nfb,
/* Process offline CPU's input_pkt_queue */
while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
netif_rx(skb);
- incr_input_queue_head(oldsd);
+ input_queue_head_incr(oldsd);
}
return NOTIFY_OK;
@@ -5812,24 +5824,23 @@ static int __init net_dev_init(void)
*/
for_each_possible_cpu(i) {
- struct softnet_data *queue;
+ struct softnet_data *sd = &per_cpu(softnet_data, i);
- queue = &per_cpu(softnet_data, i);
- skb_queue_head_init(&queue->input_pkt_queue);
- queue->completion_queue = NULL;
- INIT_LIST_HEAD(&queue->poll_list);
+ skb_queue_head_init(&sd->input_pkt_queue);
+ sd->completion_queue = NULL;
+ INIT_LIST_HEAD(&sd->poll_list);
#ifdef CONFIG_RPS
- queue->csd.func = trigger_softirq;
- queue->csd.info = queue;
- queue->csd.flags = 0;
- queue->cpu = i;
+ sd->csd.func = rps_trigger_softirq;
+ sd->csd.info = sd;
+ sd->csd.flags = 0;
+ sd->cpu = i;
#endif
- queue->backlog.poll = process_backlog;
- queue->backlog.weight = weight_p;
- queue->backlog.gro_list = NULL;
- queue->backlog.gro_count = 0;
+ sd->backlog.poll = process_backlog;
+ sd->backlog.weight = weight_p;
+ sd->backlog.gro_list = NULL;
+ sd->backlog.gro_count = 0;
}
dev_boot_phase = 0;
generated by cgit v1.2.3 (git 2.39.0) at 2024-11-16 19:41:54 +0300