summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/io_uring.c271
-rw-r--r--io_uring/io_uring.h5
2 files changed, 142 insertions, 134 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 92e51bbb769c..7b8f1c9b7b48 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -625,7 +625,6 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
bool cancel_all);
static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
-static void __io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags);
static void io_dismantle_req(struct io_kiocb *req);
static void io_queue_linked_timeout(struct io_kiocb *req);
static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
@@ -1126,7 +1125,7 @@ static inline void req_set_fail(struct io_kiocb *req)
static inline void req_fail_link_node(struct io_kiocb *req, int res)
{
req_set_fail(req);
- req->cqe.res = res;
+ io_req_set_res(req, res, 0);
}
static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx)
@@ -1855,50 +1854,37 @@ static void __io_req_complete_put(struct io_kiocb *req)
}
}
-static void __io_req_complete_post(struct io_kiocb *req, s32 res,
- u32 cflags)
+static void __io_req_complete_post(struct io_kiocb *req)
{
- if (!(req->flags & REQ_F_CQE_SKIP)) {
- req->cqe.res = res;
- req->cqe.flags = cflags;
+ if (!(req->flags & REQ_F_CQE_SKIP))
__io_fill_cqe_req(req->ctx, req);
- }
__io_req_complete_put(req);
}
-static void io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags)
+static void io_req_complete_post(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
spin_lock(&ctx->completion_lock);
- __io_req_complete_post(req, res, cflags);
+ __io_req_complete_post(req);
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
io_cqring_ev_posted(ctx);
}
-static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
- s32 res, u32 cflags)
+static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)
{
- if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
- io_req_set_res(req, res, cflags);
+ if (issue_flags & IO_URING_F_COMPLETE_DEFER)
req->flags |= REQ_F_COMPLETE_INLINE;
- } else {
- io_req_complete_post(req, res, cflags);
- }
-}
-
-static inline void io_req_complete(struct io_kiocb *req, s32 res)
-{
- if (res < 0)
- req_set_fail(req);
- __io_req_complete(req, 0, res, 0);
+ else
+ io_req_complete_post(req);
}
static void io_req_complete_failed(struct io_kiocb *req, s32 res)
{
req_set_fail(req);
- io_req_complete_post(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));
+ io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));
+ io_req_complete_post(req);
}
/*
@@ -2071,7 +2057,8 @@ static void io_fail_links(struct io_kiocb *req)
link->flags |= REQ_F_CQE_SKIP;
else
link->flags &= ~REQ_F_CQE_SKIP;
- __io_req_complete_post(link, res, 0);
+ io_req_set_res(link, res, 0);
+ __io_req_complete_post(link);
link = nxt;
}
}
@@ -2185,11 +2172,12 @@ static void handle_prev_tw_list(struct io_wq_work_node *node,
if (unlikely(!*uring_locked))
spin_lock(&(*ctx)->completion_lock);
}
- if (likely(*uring_locked))
+ if (likely(*uring_locked)) {
req->io_task_work.func(req, uring_locked);
- else
- __io_req_complete_post(req, req->cqe.res,
- io_put_kbuf_comp(req));
+ } else {
+ req->cqe.flags = io_put_kbuf_comp(req);
+ __io_req_complete_post(req);
+ }
node = next;
} while (node);
@@ -2317,13 +2305,12 @@ static void io_req_task_prio_work_add(struct io_kiocb *req)
static void io_req_tw_post(struct io_kiocb *req, bool *locked)
{
- io_req_complete_post(req, req->cqe.res, req->cqe.flags);
+ io_req_complete_post(req);
}
static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags)
{
- req->cqe.res = res;
- req->cqe.flags = cflags;
+ io_req_set_res(req, res, cflags);
req->io_task_work.func = io_req_tw_post;
io_req_task_work_add(req);
}
@@ -2347,7 +2334,7 @@ static void io_req_task_submit(struct io_kiocb *req, bool *locked)
static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
{
- req->cqe.res = ret;
+ io_req_set_res(req, ret, 0);
req->io_task_work.func = io_req_task_cancel;
io_req_task_work_add(req);
}
@@ -2741,15 +2728,13 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res)
static inline void io_req_task_complete(struct io_kiocb *req, bool *locked)
{
- int res = req->cqe.res;
-
if (*locked) {
- io_req_set_res(req, res, io_put_kbuf(req, 0));
+ req->cqe.flags |= io_put_kbuf(req, 0);
req->flags |= REQ_F_COMPLETE_INLINE;
io_req_add_compl_list(req);
} else {
- io_req_complete_post(req, res,
- io_put_kbuf(req, IO_URING_F_UNLOCKED));
+ req->cqe.flags |= io_put_kbuf(req, IO_URING_F_UNLOCKED);
+ io_req_complete_post(req);
}
}
@@ -2758,8 +2743,8 @@ static void __io_complete_rw(struct io_kiocb *req, long res,
{
if (__io_complete_rw_common(req, res))
return;
- __io_req_complete(req, issue_flags, req->cqe.res,
- io_put_kbuf(req, issue_flags));
+ io_req_set_res(req, req->cqe.res, io_put_kbuf(req, issue_flags));
+ __io_req_complete(req, issue_flags);
}
static void io_complete_rw(struct kiocb *kiocb, long res)
@@ -2769,7 +2754,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res)
if (__io_complete_rw_common(req, res))
return;
- req->cqe.res = res;
+ io_req_set_res(req, res, 0);
req->io_task_work.func = io_req_task_complete;
io_req_task_prio_work_add(req);
}
@@ -3745,7 +3730,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags)
*/
ret = io_iter_do_read(rw, &s->iter);
if (ret == -EIOCBQUEUED)
- return 0;
+ return IOU_ISSUE_SKIP_COMPLETE;
/* we got some bytes, but not all. retry. */
kiocb->ki_flags &= ~IOCB_WAITQ;
iov_iter_restore(&s->iter, &s->iter_state);
@@ -3756,7 +3741,7 @@ out_free:
/* it's faster to check here then delegate to kfree */
if (iovec)
kfree(iovec);
- return 0;
+ return IOU_ISSUE_SKIP_COMPLETE;
}
static int io_write(struct io_kiocb *req, unsigned int issue_flags)
@@ -3850,6 +3835,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags)
goto copy_iov;
done:
kiocb_done(req, ret2, issue_flags);
+ ret = IOU_ISSUE_SKIP_COMPLETE;
} else {
copy_iov:
iov_iter_restore(&s->iter, &s->iter_state);
@@ -3906,8 +3892,8 @@ static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
ren->newpath, ren->flags);
req->flags &= ~REQ_F_NEED_CLEANUP;
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static void io_renameat_cleanup(struct io_kiocb *req)
@@ -3934,7 +3920,7 @@ static void io_xattr_finish(struct io_kiocb *req, int ret)
req->flags &= ~REQ_F_NEED_CLEANUP;
io_xattr_cleanup(req);
- io_req_complete(req, ret);
+ io_req_set_res(req, ret, 0);
}
static int __io_getxattr_prep(struct io_kiocb *req,
@@ -4015,7 +4001,7 @@ static int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags)
&ix->ctx);
io_xattr_finish(req, ret);
- return 0;
+ return IOU_OK;
}
static int io_getxattr(struct io_kiocb *req, unsigned int issue_flags)
@@ -4043,7 +4029,7 @@ retry:
}
io_xattr_finish(req, ret);
- return 0;
+ return IOU_OK;
}
static int __io_setxattr_prep(struct io_kiocb *req,
@@ -4129,8 +4115,7 @@ static int io_fsetxattr(struct io_kiocb *req, unsigned int issue_flags)
ret = __io_setxattr(req, issue_flags, &req->file->f_path);
io_xattr_finish(req, ret);
-
- return 0;
+ return IOU_OK;
}
static int io_setxattr(struct io_kiocb *req, unsigned int issue_flags)
@@ -4155,7 +4140,7 @@ retry:
}
io_xattr_finish(req, ret);
- return 0;
+ return IOU_OK;
}
static int io_unlinkat_prep(struct io_kiocb *req,
@@ -4198,8 +4183,8 @@ static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
ret = do_unlinkat(un->dfd, un->filename);
req->flags &= ~REQ_F_NEED_CLEANUP;
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static void io_unlinkat_cleanup(struct io_kiocb *req)
@@ -4243,8 +4228,8 @@ static int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags)
ret = do_mkdirat(mkd->dfd, mkd->filename, mkd->mode);
req->flags &= ~REQ_F_NEED_CLEANUP;
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static void io_mkdirat_cleanup(struct io_kiocb *req)
@@ -4294,8 +4279,8 @@ static int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags)
ret = do_symlinkat(sl->oldpath, sl->new_dfd, sl->newpath);
req->flags &= ~REQ_F_NEED_CLEANUP;
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_linkat_prep(struct io_kiocb *req,
@@ -4341,8 +4326,8 @@ static int io_linkat(struct io_kiocb *req, unsigned int issue_flags)
lnk->newpath, lnk->flags);
req->flags &= ~REQ_F_NEED_CLEANUP;
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static void io_link_cleanup(struct io_kiocb *req)
@@ -4393,7 +4378,7 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
io_req_set_res(req, 0, ret);
if (req->ctx->flags & IORING_SETUP_CQE32)
io_req_set_cqe32_extra(req, res2, 0);
- io_req_complete(req, ret);
+ __io_req_complete(req, 0);
}
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
@@ -4450,9 +4435,12 @@ static int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
return -EAGAIN;
}
- if (ret != -EIOCBQUEUED)
+ if (ret != -EIOCBQUEUED) {
io_uring_cmd_done(ioucmd, ret, 0);
- return 0;
+ return IOU_OK;
+ }
+
+ return IOU_ISSUE_SKIP_COMPLETE;
}
static int __io_splice_prep(struct io_kiocb *req,
@@ -4505,8 +4493,8 @@ static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
done:
if (ret != sp->len)
req_set_fail(req);
- __io_req_complete(req, 0, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -4550,8 +4538,8 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
done:
if (ret != sp->len)
req_set_fail(req);
- __io_req_complete(req, 0, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -4564,8 +4552,8 @@ static int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
*/
static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
{
- __io_req_complete(req, issue_flags, 0, 0);
- return 0;
+ io_req_set_res(req, 0, 0);
+ return IOU_OK;
}
static int io_msg_ring_prep(struct io_kiocb *req,
@@ -4609,11 +4597,11 @@ static int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
done:
if (ret < 0)
req_set_fail(req);
- __io_req_complete(req, issue_flags, ret, 0);
+ io_req_set_res(req, ret, 0);
/* put file to avoid an attempt to IOPOLL the req */
io_put_file(req->file);
req->file = NULL;
- return 0;
+ return IOU_OK;
}
static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -4644,8 +4632,8 @@ static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
ret = vfs_fsync_range(req->file, sync->off, end > 0 ? end : LLONG_MAX,
sync->flags & IORING_FSYNC_DATASYNC);
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_fallocate_prep(struct io_kiocb *req,
@@ -4673,8 +4661,8 @@ static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
ret = vfs_fallocate(req->file, sync->mode, sync->off, sync->len);
if (ret >= 0)
fsnotify_modify(req->file);
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -4855,8 +4843,8 @@ err:
req->flags &= ~REQ_F_NEED_CLEANUP;
if (ret < 0)
req_set_fail(req);
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
@@ -4951,9 +4939,10 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
req_set_fail(req);
/* complete before unlock, IOPOLL may need the lock */
- __io_req_complete(req, issue_flags, ret, 0);
+ io_req_set_res(req, ret, 0);
+ __io_req_complete(req, issue_flags);
io_ring_submit_unlock(ctx, issue_flags);
- return 0;
+ return IOU_ISSUE_SKIP_COMPLETE;
}
static int io_provide_buffers_prep(struct io_kiocb *req,
@@ -5117,9 +5106,10 @@ err:
if (ret < 0)
req_set_fail(req);
/* complete before unlock, IOPOLL may need the lock */
- __io_req_complete(req, issue_flags, ret, 0);
+ io_req_set_res(req, ret, 0);
+ __io_req_complete(req, issue_flags);
io_ring_submit_unlock(ctx, issue_flags);
- return 0;
+ return IOU_ISSUE_SKIP_COMPLETE;
}
static int io_epoll_ctl_prep(struct io_kiocb *req,
@@ -5162,8 +5152,8 @@ static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
if (ret < 0)
req_set_fail(req);
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
#else
return -EOPNOTSUPP;
#endif
@@ -5196,8 +5186,8 @@ static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
return -EAGAIN;
ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
#else
return -EOPNOTSUPP;
#endif
@@ -5235,8 +5225,8 @@ static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
if (ret < 0)
req_set_fail(req);
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -5279,8 +5269,8 @@ static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
return -EAGAIN;
ret = do_statx(sx->dfd, sx->filename, sx->flags, sx->mask, sx->buffer);
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static void io_statx_cleanup(struct io_kiocb *req)
@@ -5350,8 +5340,8 @@ static int io_close(struct io_kiocb *req, unsigned int issue_flags)
err:
if (ret < 0)
req_set_fail(req);
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -5377,8 +5367,8 @@ static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
return -EAGAIN;
ret = sync_file_range(req->file, sync->off, sync->len, sync->flags);
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
#if defined(CONFIG_NET)
@@ -5409,8 +5399,8 @@ static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
return -ENOTSOCK;
ret = __sys_shutdown_sock(sock, shutdown->how);
- io_req_complete(req, ret);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static bool io_net_retry(struct socket *sock, int flags)
@@ -5548,8 +5538,8 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
ret += sr->done_io;
else if (sr->done_io)
ret = sr->done_io;
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_send(struct io_kiocb *req, unsigned int issue_flags)
@@ -5605,8 +5595,8 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags)
ret += sr->done_io;
else if (sr->done_io)
ret = sr->done_io;
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
@@ -5805,8 +5795,8 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
cflags = io_put_kbuf(req, issue_flags);
if (kmsg->msg.msg_inq)
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
- __io_req_complete(req, issue_flags, ret, cflags);
- return 0;
+ io_req_set_res(req, ret, cflags);
+ return IOU_OK;
}
static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
@@ -5881,8 +5871,8 @@ out_free:
cflags = io_put_kbuf(req, issue_flags);
if (msg.msg_inq)
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
- __io_req_complete(req, issue_flags, ret, cflags);
- return 0;
+ io_req_set_res(req, ret, cflags);
+ return IOU_OK;
}
static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -5948,7 +5938,7 @@ retry:
*/
if ((req->flags & IO_APOLL_MULTI_POLLED) ==
IO_APOLL_MULTI_POLLED)
- ret = 0;
+ ret = IOU_ISSUE_SKIP_COMPLETE;
return ret;
}
if (ret == -ERESTARTSYS)
@@ -5963,8 +5953,8 @@ retry:
}
if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
if (ret >= 0) {
bool filled;
@@ -6034,8 +6024,8 @@ static int io_socket(struct io_kiocb *req, unsigned int issue_flags)
ret = io_fixed_fd_install(req, issue_flags, file,
sock->file_slot);
}
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_connect_prep_async(struct io_kiocb *req)
@@ -6096,8 +6086,8 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
out:
if (ret < 0)
req_set_fail(req);
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
#else /* !CONFIG_NET */
#define IO_NETOP_FN(op) \
@@ -6328,7 +6318,8 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)
io_poll_remove_entries(req);
spin_lock(&ctx->completion_lock);
hash_del(&req->hash_node);
- __io_req_complete_post(req, req->cqe.res, 0);
+ req->cqe.flags = 0;
+ __io_req_complete_post(req);
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
io_cqring_ev_posted(ctx);
@@ -6357,7 +6348,7 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked)
static void __io_poll_execute(struct io_kiocb *req, int mask,
__poll_t __maybe_unused events)
{
- req->cqe.res = mask;
+ io_req_set_res(req, mask, 0);
/*
* This is useful for poll that is armed on behalf of another
* request, and where the wakeup path could be on a different
@@ -6810,12 +6801,16 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
ipt.pt._qproc = io_poll_queue_proc;
ret = __io_arm_poll_handler(req, poll, &ipt, poll->events);
- if (!ret && ipt.error)
+ if (ret) {
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
+ }
+ if (ipt.error) {
req_set_fail(req);
- ret = ret ?: ipt.error;
- if (ret)
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ return ipt.error;
+ }
+
+ return IOU_ISSUE_SKIP_COMPLETE;
}
static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
@@ -6850,20 +6845,22 @@ static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
ret2 = io_poll_add(preq, issue_flags);
/* successfully updated, don't complete poll request */
- if (!ret2)
+ if (!ret2 || ret2 == -EIOCBQUEUED)
goto out;
}
req_set_fail(preq);
- preq->cqe.res = -ECANCELED;
+ io_req_set_res(preq, -ECANCELED, 0);
locked = !(issue_flags & IO_URING_F_UNLOCKED);
io_req_task_complete(preq, &locked);
out:
- if (ret < 0)
+ if (ret < 0) {
req_set_fail(req);
+ return ret;
+ }
/* complete update request, we're done with it */
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
@@ -6884,7 +6881,7 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
req_set_fail(req);
- req->cqe.res = -ETIME;
+ io_req_set_res(req, -ETIME, 0);
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
return HRTIMER_NORESTART;
@@ -7069,8 +7066,8 @@ static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
if (ret < 0)
req_set_fail(req);
- io_req_complete_post(req, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int __io_timeout_prep(struct io_kiocb *req,
@@ -7191,7 +7188,7 @@ add:
data->timer.function = io_timeout_fn;
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
spin_unlock_irq(&ctx->timeout_lock);
- return 0;
+ return IOU_ISSUE_SKIP_COMPLETE;
}
static bool io_cancel_cb(struct io_wq_work *work, void *data)
@@ -7359,8 +7356,8 @@ static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
done:
if (ret < 0)
req_set_fail(req);
- io_req_complete_post(req, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_files_update_prep(struct io_kiocb *req,
@@ -7445,8 +7442,8 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
if (ret < 0)
req_set_fail(req);
- __io_req_complete(req, issue_flags, ret, 0);
- return 0;
+ io_req_set_res(req, ret, 0);
+ return IOU_OK;
}
static int io_req_prep_async(struct io_kiocb *req)
@@ -7590,8 +7587,12 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
if (creds)
revert_creds(creds);
- if (ret)
+
+ if (ret == IOU_OK)
+ __io_req_complete(req, issue_flags);
+ else if (ret != IOU_ISSUE_SKIP_COMPLETE)
return ret;
+
/* If the op doesn't have a file, we're not polling for it */
if ((req->ctx->flags & IORING_SETUP_IOPOLL) && req->file)
io_iopoll_req_issued(req, issue_flags);
@@ -7668,7 +7669,7 @@ fail:
} while (1);
/* avoid locking problems by failing it from a clean context */
- if (ret)
+ if (ret < 0)
io_req_task_queue_fail(req, ret);
}
@@ -7745,10 +7746,12 @@ static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
ret = io_try_cancel(req, &cd);
}
- io_req_complete_post(req, ret ?: -ETIME, 0);
+ io_req_set_res(req, ret ?: -ETIME, 0);
+ io_req_complete_post(req);
io_put_req(prev);
} else {
- io_req_complete_post(req, -ETIME, 0);
+ io_req_set_res(req, -ETIME, 0);
+ io_req_complete_post(req);
}
}
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 522e65219757..73943dbe884e 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -4,6 +4,11 @@
#include <linux/errno.h>
#include "io_uring_types.h"
+enum {
+ IOU_OK = 0,
+ IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED,
+};
+
static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
{
req->cqe.res = res;
generated by cgit v1.2.3 (git 2.39.0) at 2024-12-28 21:39:29 +0300