From 50470fc5723ae0adb2f429a8b27ff6bf1a41913e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:19 +0000 Subject: io_uring: return normal tw run linking optimisation io_submit_flush_completions() may produce new task_work items, so it's a good idea to recheck the task_work list after flushing completions. The optimisation is not new and was accidentially removed by f88262e60bb9 ("io_uring: lockless task list") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a7ed5ede84de190832cc33ebbcdd6e91cd90f5b6.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 55101013f3ee..9c92ca081c11 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1238,6 +1238,15 @@ void tctx_task_work(struct callback_head *cb) loops++; node = io_llist_xchg(&tctx->task_list, &fake); count += handle_tw_list(node, &ctx, &uring_locked, &fake); + + /* skip expensive cmpxchg if there are items in the list */ + if (READ_ONCE(tctx->task_list.first) != &fake) + continue; + if (uring_locked && !wq_list_empty(&ctx->submit_state.compl_reqs)) { + io_submit_flush_completions(ctx); + if (READ_ONCE(tctx->task_list.first) != &fake) + continue; + } node = io_llist_cmpxchg(&tctx->task_list, &fake, NULL); } while (node != &fake); -- cgit v1.2.3