diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2021-08-11 21:28:31 +0300 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2021-08-23 22:10:37 +0300 |
| commit | fd08e5309bba8672c1190362dff6c92bfd59218d (patch) | |
| tree | d49461dc811db2fd896507ee24386cb19ba8ec09 /fs/io_uring.c | |
| parent | 20e60a3832089741d6b25c13d291050c5d00b4e7 (diff) | |
| download | linux-fd08e5309bba8672c1190362dff6c92bfd59218d.tar.xz | |
io_uring: optimise hot path of ltimeout prep
io_prep_linked_timeout() grew too heavy and compiler now refuse to
inline the function. Help it by splitting in two and annotating with
inline.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/560636717a32e9513724f09b9ecaace942dde4d4.1628705069.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'fs/io_uring.c')
| -rw-r--r-- | fs/io_uring.c | 45 |
1 files changed, 25 insertions, 20 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index dae87c576943..4f5a00707644 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1046,7 +1046,6 @@ static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data, static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req); static void io_dismantle_req(struct io_kiocb *req); -static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); static void io_queue_linked_timeout(struct io_kiocb *req); static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, struct io_uring_rsrc_update2 *up, @@ -1299,6 +1298,31 @@ static void io_req_track_inflight(struct io_kiocb *req) } } +static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req) +{ + struct io_kiocb *nxt = req->link; + + if (req->flags & REQ_F_LINK_TIMEOUT) + return NULL; + + /* linked timeouts should have two refs once prep'ed */ + io_req_refcount(req); + io_req_refcount(nxt); + req_ref_get(nxt); + + nxt->timeout.head = req; + nxt->flags |= REQ_F_LTIMEOUT_ACTIVE; + req->flags |= REQ_F_LINK_TIMEOUT; + return nxt; +} + +static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) +{ + if (likely(!req->link || req->link->opcode != IORING_OP_LINK_TIMEOUT)) + return NULL; + return __io_prep_linked_timeout(req); +} + static void io_prep_async_work(struct io_kiocb *req) { const struct io_op_def *def = &io_op_defs[req->opcode]; @@ -6453,25 +6477,6 @@ static void io_queue_linked_timeout(struct io_kiocb *req) io_put_req(req); } -static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) -{ - struct io_kiocb *nxt = req->link; - - if (!nxt || (req->flags & REQ_F_LINK_TIMEOUT) || - nxt->opcode != IORING_OP_LINK_TIMEOUT) - return NULL; - - /* linked timeouts should have two refs once prep'ed */ - io_req_refcount(req); - io_req_refcount(nxt); - req_ref_get(nxt); - - nxt->timeout.head = req; - nxt->flags |= REQ_F_LTIMEOUT_ACTIVE; - req->flags |= REQ_F_LINK_TIMEOUT; - return nxt; -} - static void __io_queue_sqe(struct io_kiocb *req) __must_hold(&req->ctx->uring_lock) { |