From a993800655ee516b6f6a6fc4c2ee13fedfb0590b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 20 Apr 2007 08:55:52 +0200 Subject: cfq-iosched: fix sequential write regression We have a 10-15% performance regression for sequential writes on TCQ/NCQ enabled drives in 2.6.21-rcX after the CFQ update went in. It has been reported by Valerie Clement and the Intel testing folks. The regression is because of CFQ's now more aggressive queue control, limiting the depth available to the device. This patches fixes that regression by allowing a greater depth when only one queue is busy. It has been tested to not impact sync-vs-async workloads too much - we still do a lot better than 2.6.20. Signed-off-by: Jens Axboe Signed-off-by: Linus Torvalds --- block/cfq-iosched.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) (limited to 'block/cfq-iosched.c') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index b6491c020f26..9e3797167c81 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -986,9 +986,9 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq, * expire an async queue immediately if it has used up its slice. idle * queue always expire after 1 dispatch round. */ - if ((!cfq_cfqq_sync(cfqq) && + if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) && cfqd->dispatch_slice >= cfq_prio_to_maxrq(cfqd, cfqq)) || - cfq_class_idle(cfqq)) { + cfq_class_idle(cfqq))) { cfqq->slice_end = jiffies + 1; cfq_slice_expired(cfqd, 0, 0); } @@ -1051,19 +1051,21 @@ cfq_dispatch_requests(request_queue_t *q, int force) while ((cfqq = cfq_select_queue(cfqd)) != NULL) { int max_dispatch; - /* - * Don't repeat dispatch from the previous queue. - */ - if (prev_cfqq == cfqq) - break; + if (cfqd->busy_queues > 1) { + /* + * Don't repeat dispatch from the previous queue. + */ + if (prev_cfqq == cfqq) + break; - /* - * So we have dispatched before in this round, if the - * next queue has idling enabled (must be sync), don't - * allow it service until the previous have continued. - */ - if (cfqd->rq_in_driver && cfq_cfqq_idle_window(cfqq)) - break; + /* + * So we have dispatched before in this round, if the + * next queue has idling enabled (must be sync), don't + * allow it service until the previous have continued. + */ + if (cfqd->rq_in_driver && cfq_cfqq_idle_window(cfqq)) + break; + } cfq_clear_cfqq_must_dispatch(cfqq); cfq_clear_cfqq_wait_request(cfqq); @@ -1370,7 +1372,9 @@ retry: atomic_set(&cfqq->ref, 0); cfqq->cfqd = cfqd; - cfq_mark_cfqq_idle_window(cfqq); + if (key != CFQ_KEY_ASYNC) + cfq_mark_cfqq_idle_window(cfqq); + cfq_mark_cfqq_prio_changed(cfqq); cfq_mark_cfqq_queue_new(cfqq); cfq_init_prio_data(cfqq); -- cgit v1.2.3 From 5044eed48886b105a123333fe7ca97c6bd496120 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 25 Apr 2007 11:53:48 +0200 Subject: cfq-iosched: fix alias + front merge bug There's a really rare and obscure bug in CFQ, that causes a crash in cfq_dispatch_insert() due to rq == NULL. One example of the resulting oops is seen here: http://lkml.org/lkml/2007/4/15/41 Neil correctly diagnosed the situation for how this can happen: if two concurrent requests with the exact same sector number (due to direct IO or aliasing between MD and the raw device access), the alias handling will add the request to the sortlist, but next_rq remains NULL. Read the more complete analysis at: http://lkml.org/lkml/2007/4/25/57 This looks like it requires md to trigger, even though it should potentially be possible to due with O_DIRECT (at least if you edit the kernel and doctor some of the unplug calls). The fix is to move the ->next_rq update to when we add a request to the rbtree. Then we remove the possibility for a request to exist in the rbtree code, but not have ->next_rq correctly updated. Signed-off-by: Jens Axboe Signed-off-by: Linus Torvalds --- block/cfq-iosched.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'block/cfq-iosched.c') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 9e3797167c81..f92ba2a869b4 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -532,6 +532,12 @@ static void cfq_add_rq_rb(struct request *rq) if (!cfq_cfqq_on_rr(cfqq)) cfq_add_cfqq_rr(cfqd, cfqq); + + /* + * check if this request is a better next-serve candidate + */ + cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq); + BUG_ON(!cfqq->next_rq); } static inline void @@ -1638,12 +1644,6 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq, if (rq_is_meta(rq)) cfqq->meta_pending++; - /* - * check if this request is a better next-serve candidate)) { - */ - cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq); - BUG_ON(!cfqq->next_rq); - /* * we never wait for an async request and we don't allow preemption * of an async request. so just return early -- cgit v1.2.3