diff options
| author | David S. Miller <davem@davemloft.net> | 2018-05-17 19:46:55 +0300 |
|---|---|---|
| committer | David S. Miller <davem@davemloft.net> | 2018-05-17 19:46:55 +0300 |
| commit | 4b9c77680375f5ecb7d2c4bc4c0db67cdf531c09 (patch) | |
| tree | 6e6b0d4339e495a4c94e8489f3b06a00033fa435 /include | |
| parent | b9f672af148bf7a08a6031743156faffd58dbc7e (diff) | |
| parent | 021a17ed796b62383f7623f4fea73787abddad77 (diff) | |
| download | linux-4b9c77680375f5ecb7d2c4bc4c0db67cdf531c09.tar.xz | |
Merge branch 'sched-refactor-NOLOCK-qdiscs'
Paolo Abeni says:
====================
sched: refactor NOLOCK qdiscs
With the introduction of NOLOCK qdiscs, pfifo_fast performances in the
uncontended scenario degraded measurably, especially after the commit
eb82a9944792 ("net: sched, fix OOO packets with pfifo_fast").
This series restore the pfifo_fast performances in such scenario back the
previous level, mainly reducing the number of atomic operations required to
perform the qdisc_run() call. Even performances in the contended scenario
increase measurably.
Note: This series is on top of:
sched: manipulate __QDISC_STATE_RUNNING in qdisc_run_* helpers
====================
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'include')
| -rw-r--r-- | include/linux/skb_array.h | 5 | ||||
| -rw-r--r-- | include/net/sch_generic.h | 10 |
2 files changed, 10 insertions, 5 deletions
diff --git a/include/linux/skb_array.h b/include/linux/skb_array.h index a6b6e8bb3d7b..62d9b0a6329f 100644 --- a/include/linux/skb_array.h +++ b/include/linux/skb_array.h @@ -97,6 +97,11 @@ static inline bool skb_array_empty_any(struct skb_array *a) return ptr_ring_empty_any(&a->ring); } +static inline struct sk_buff *__skb_array_consume(struct skb_array *a) +{ + return __ptr_ring_consume(&a->ring); +} + static inline struct sk_buff *skb_array_consume(struct skb_array *a) { return ptr_ring_consume(&a->ring); diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 4d2b37226e75..98c10a28cd01 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -30,7 +30,6 @@ struct qdisc_rate_table { enum qdisc_state_t { __QDISC_STATE_SCHED, __QDISC_STATE_DEACTIVATED, - __QDISC_STATE_RUNNING, }; struct qdisc_size_table { @@ -102,6 +101,7 @@ struct Qdisc { refcount_t refcnt; spinlock_t busylock ____cacheline_aligned_in_smp; + spinlock_t seqlock; }; static inline void qdisc_refcount_inc(struct Qdisc *qdisc) @@ -111,17 +111,17 @@ static inline void qdisc_refcount_inc(struct Qdisc *qdisc) refcount_inc(&qdisc->refcnt); } -static inline bool qdisc_is_running(const struct Qdisc *qdisc) +static inline bool qdisc_is_running(struct Qdisc *qdisc) { if (qdisc->flags & TCQ_F_NOLOCK) - return test_bit(__QDISC_STATE_RUNNING, &qdisc->state); + return spin_is_locked(&qdisc->seqlock); return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; } static inline bool qdisc_run_begin(struct Qdisc *qdisc) { if (qdisc->flags & TCQ_F_NOLOCK) { - if (test_and_set_bit(__QDISC_STATE_RUNNING, &qdisc->state)) + if (!spin_trylock(&qdisc->seqlock)) return false; } else if (qdisc_is_running(qdisc)) { return false; @@ -138,7 +138,7 @@ static inline void qdisc_run_end(struct Qdisc *qdisc) { write_seqcount_end(&qdisc->running); if (qdisc->flags & TCQ_F_NOLOCK) - clear_bit(__QDISC_STATE_RUNNING, &qdisc->state); + spin_unlock(&qdisc->seqlock); } static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) |