diff options
Diffstat (limited to 'io_uring')
| -rw-r--r-- | io_uring/io-wq.c | 2 | ||||
| -rw-r--r-- | io_uring/rw.c | 15 | ||||
| -rw-r--r-- | io_uring/waitid.c | 6 |
3 files changed, 15 insertions, 8 deletions
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 9fd9f6ab722c..2fa7d3601edb 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -598,9 +598,9 @@ static void io_worker_handle_work(struct io_wq_acct *acct, __releases(&acct->lock) { struct io_wq *wq = worker->wq; - bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); do { + bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); struct io_wq_work *work; /* diff --git a/io_uring/rw.c b/io_uring/rw.c index 70ca88cc1f54..28555bc85ba0 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -144,19 +144,22 @@ static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, return 0; } -static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) +static bool io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_rw *rw = req->async_data; if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) - return; + return false; io_alloc_cache_vec_kasan(&rw->vec); if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP) io_vec_free(&rw->vec); - if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) + if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) { io_req_async_data_clear(req, 0); + return true; + } + return false; } static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) @@ -190,7 +193,11 @@ static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) */ if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) { req->flags &= ~REQ_F_NEED_CLEANUP; - io_rw_recycle(req, issue_flags); + if (!io_rw_recycle(req, issue_flags)) { + struct io_async_rw *rw = req->async_data; + + io_vec_free(&rw->vec); + } } } diff --git a/io_uring/waitid.c b/io_uring/waitid.c index 2d4cbd47c67c..d25d60aed6af 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -114,11 +114,11 @@ static void io_waitid_remove_wq(struct io_kiocb *req) struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); struct wait_queue_head *head; - head = READ_ONCE(iw->head); + head = smp_load_acquire(&iw->head); if (head) { struct io_waitid_async *iwa = req->async_data; - iw->head = NULL; + smp_store_release(&iw->head, NULL); spin_lock_irq(&head->lock); list_del_init(&iwa->wo.child_wait.entry); spin_unlock_irq(&head->lock); @@ -246,7 +246,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, return 0; list_del_init(&wait->entry); - iw->head = NULL; + smp_store_release(&iw->head, NULL); /* cancel is in progress */ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) |