Merge tag 'for-6.3/io_uring-2023-02-16' of git://git.kernel.dk/linux

Pull io_uring updates from Jens Axboe:

 - Cleanup series making the async prep and handling of
   REQ_F_FORCE_ASYNC easier to follow and verify (Dylan)

 - Enable specifying specific flags for OP_MSG_RING (Breno)

 - Enable use of KASAN with the internal request cache (Breno)

 - Split the opcode definition structs into a hot and cold part (Breno)

 - OP_MSG_RING fixes (Pavel, me)

 - Fix an issue with IOPOLL cancelation and PREEMPT_NONE (me)

 - Handle TIF_NOTIFY_RESUME for the io-wq threads that never return to
   userspace (me)

 - Add support for using io_uring_register() with a registered ring fd
   (Josh)

 - Improve handling of poll on the ring fd (Pavel)

 - Series improving the task_work handling (Pavel)

 - Misc cleanups, fixes, improvements (Dmitrii, Quanfa, Richard, Pavel,
   me)

* tag 'for-6.3/io_uring-2023-02-16' of git://git.kernel.dk/linux: (51 commits)
  io_uring: Support calling io_uring_register with a registered ring fd
  io_uring,audit: don't log IORING_OP_MADVISE
  io_uring: mark task TASK_RUNNING before handling resume/task work
  io_uring: always go async for unsupported open flags
  io_uring: always go async for unsupported fadvise flags
  io_uring: for requests that require async, force it
  io_uring: if a linked request has REQ_F_FORCE_ASYNC then run it async
  io_uring: add reschedule point to handle_tw_list()
  io_uring: add a conditional reschedule to the IOPOLL cancelation loop
  io_uring: return normal tw run linking optimisation
  io_uring: refactor tctx_task_work
  io_uring: refactor io_put_task helpers
  io_uring: refactor req allocation
  io_uring: improve io_get_sqe
  io_uring: kill outdated comment about overflow flush
  io_uring: use user visible tail in io_uring_poll()
  io_uring: pass in io_issue_def to io_assign_file()
  io_uring: Enable KASAN for request cache
  io_uring: handle TIF_NOTIFY_RESUME when checking for task_work
  io_uring/msg-ring: ensure flags passing works for task_work completions
  ...

1 files changed, 229 insertions, 111 deletions

diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index 3aa0d65c50e3..cca7c5b55208 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -46,11 +46,10 @@ static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
 	return -EOPNOTSUPP;
 }
 
-const struct io_op_def io_op_defs[] = {
+const struct io_issue_def io_issue_defs[] = {
 	[IORING_OP_NOP] = {
 		.audit_skip		= 1,
 		.iopoll			= 1,
-		.name			= "NOP",
 		.prep			= io_nop_prep,
 		.issue			= io_nop,
 	},
@@ -64,13 +63,8 @@ const struct io_op_def io_op_defs[] = {
 		.ioprio			= 1,
 		.iopoll			= 1,
 		.iopoll_queue		= 1,
-		.async_size		= sizeof(struct io_async_rw),
-		.name			= "READV",
 		.prep			= io_prep_rw,
 		.issue			= io_read,
-		.prep_async		= io_readv_prep_async,
-		.cleanup		= io_readv_writev_cleanup,
-		.fail			= io_rw_fail,
 	},
 	[IORING_OP_WRITEV] = {
 		.needs_file		= 1,
@@ -82,18 +76,12 @@ const struct io_op_def io_op_defs[] = {
 		.ioprio			= 1,
 		.iopoll			= 1,
 		.iopoll_queue		= 1,
-		.async_size		= sizeof(struct io_async_rw),
-		.name			= "WRITEV",
 		.prep			= io_prep_rw,
 		.issue			= io_write,
-		.prep_async		= io_writev_prep_async,
-		.cleanup		= io_readv_writev_cleanup,
-		.fail			= io_rw_fail,
 	},
 	[IORING_OP_FSYNC] = {
 		.needs_file		= 1,
 		.audit_skip		= 1,
-		.name			= "FSYNC",
 		.prep			= io_fsync_prep,
 		.issue			= io_fsync,
 	},
@@ -106,11 +94,8 @@ const struct io_op_def io_op_defs[] = {
 		.ioprio			= 1,
 		.iopoll			= 1,
 		.iopoll_queue		= 1,
-		.async_size		= sizeof(struct io_async_rw),
-		.name			= "READ_FIXED",
 		.prep			= io_prep_rw,
 		.issue			= io_read,
-		.fail			= io_rw_fail,
 	},
 	[IORING_OP_WRITE_FIXED] = {
 		.needs_file		= 1,
@@ -122,30 +107,24 @@ const struct io_op_def io_op_defs[] = {
 		.ioprio			= 1,
 		.iopoll			= 1,
 		.iopoll_queue		= 1,
-		.async_size		= sizeof(struct io_async_rw),
-		.name			= "WRITE_FIXED",
 		.prep			= io_prep_rw,
 		.issue			= io_write,
-		.fail			= io_rw_fail,
 	},
 	[IORING_OP_POLL_ADD] = {
 		.needs_file		= 1,
 		.unbound_nonreg_file	= 1,
 		.audit_skip		= 1,
-		.name			= "POLL_ADD",
 		.prep			= io_poll_add_prep,
 		.issue			= io_poll_add,
 	},
 	[IORING_OP_POLL_REMOVE] = {
 		.audit_skip		= 1,
-		.name			= "POLL_REMOVE",
 		.prep			= io_poll_remove_prep,
 		.issue			= io_poll_remove,
 	},
 	[IORING_OP_SYNC_FILE_RANGE] = {
 		.needs_file		= 1,
 		.audit_skip		= 1,
-		.name			= "SYNC_FILE_RANGE",
 		.prep			= io_sfr_prep,
 		.issue			= io_sync_file_range,
 	},
@@ -155,14 +134,9 @@ const struct io_op_def io_op_defs[] = {
 		.pollout		= 1,
 		.ioprio			= 1,
 		.manual_alloc		= 1,
-		.name			= "SENDMSG",
 #if defined(CONFIG_NET)
-		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_sendmsg_prep,
 		.issue			= io_sendmsg,
-		.prep_async		= io_sendmsg_prep_async,
-		.cleanup		= io_sendmsg_recvmsg_cleanup,
-		.fail			= io_sendrecv_fail,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
@@ -174,29 +148,21 @@ const struct io_op_def io_op_defs[] = {
 		.buffer_select		= 1,
 		.ioprio			= 1,
 		.manual_alloc		= 1,
-		.name			= "RECVMSG",
 #if defined(CONFIG_NET)
-		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_recvmsg_prep,
 		.issue			= io_recvmsg,
-		.prep_async		= io_recvmsg_prep_async,
-		.cleanup		= io_sendmsg_recvmsg_cleanup,
-		.fail			= io_sendrecv_fail,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
 	},
 	[IORING_OP_TIMEOUT] = {
 		.audit_skip		= 1,
-		.async_size		= sizeof(struct io_timeout_data),
-		.name			= "TIMEOUT",
 		.prep			= io_timeout_prep,
 		.issue			= io_timeout,
 	},
 	[IORING_OP_TIMEOUT_REMOVE] = {
 		/* used by timeout updates' prep() */
 		.audit_skip		= 1,
-		.name			= "TIMEOUT_REMOVE",
 		.prep			= io_timeout_remove_prep,
 		.issue			= io_timeout_remove,
 	},
@@ -206,7 +172,6 @@ const struct io_op_def io_op_defs[] = {
 		.pollin			= 1,
 		.poll_exclusive		= 1,
 		.ioprio			= 1,	/* used for flags */
-		.name			= "ACCEPT",
 #if defined(CONFIG_NET)
 		.prep			= io_accept_prep,
 		.issue			= io_accept,
@@ -216,14 +181,11 @@ const struct io_op_def io_op_defs[] = {
 	},
 	[IORING_OP_ASYNC_CANCEL] = {
 		.audit_skip		= 1,
-		.name			= "ASYNC_CANCEL",
 		.prep			= io_async_cancel_prep,
 		.issue			= io_async_cancel,
 	},
 	[IORING_OP_LINK_TIMEOUT] = {
 		.audit_skip		= 1,
-		.async_size		= sizeof(struct io_timeout_data),
-		.name			= "LINK_TIMEOUT",
 		.prep			= io_link_timeout_prep,
 		.issue			= io_no_issue,
 	},
@@ -231,46 +193,36 @@ const struct io_op_def io_op_defs[] = {
 		.needs_file		= 1,
 		.unbound_nonreg_file	= 1,
 		.pollout		= 1,
-		.name			= "CONNECT",
 #if defined(CONFIG_NET)
-		.async_size		= sizeof(struct io_async_connect),
 		.prep			= io_connect_prep,
 		.issue			= io_connect,
-		.prep_async		= io_connect_prep_async,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
 	},
 	[IORING_OP_FALLOCATE] = {
 		.needs_file		= 1,
-		.name			= "FALLOCATE",
 		.prep			= io_fallocate_prep,
 		.issue			= io_fallocate,
 	},
 	[IORING_OP_OPENAT] = {
-		.name			= "OPENAT",
 		.prep			= io_openat_prep,
 		.issue			= io_openat,
-		.cleanup		= io_open_cleanup,
 	},
 	[IORING_OP_CLOSE] = {
-		.name			= "CLOSE",
 		.prep			= io_close_prep,
 		.issue			= io_close,
 	},
 	[IORING_OP_FILES_UPDATE] = {
 		.audit_skip		= 1,
 		.iopoll			= 1,
-		.name			= "FILES_UPDATE",
 		.prep			= io_files_update_prep,
 		.issue			= io_files_update,
 	},
 	[IORING_OP_STATX] = {
 		.audit_skip		= 1,
-		.name			= "STATX",
 		.prep			= io_statx_prep,
 		.issue			= io_statx,
-		.cleanup		= io_statx_cleanup,
 	},
 	[IORING_OP_READ] = {
 		.needs_file		= 1,
@@ -282,11 +234,8 @@ const struct io_op_def io_op_defs[] = {
 		.ioprio			= 1,
 		.iopoll			= 1,
 		.iopoll_queue		= 1,
-		.async_size		= sizeof(struct io_async_rw),
-		.name			= "READ",
 		.prep			= io_prep_rw,
 		.issue			= io_read,
-		.fail			= io_rw_fail,
 	},
 	[IORING_OP_WRITE] = {
 		.needs_file		= 1,
@@ -298,21 +247,17 @@ const struct io_op_def io_op_defs[] = {
 		.ioprio			= 1,
 		.iopoll			= 1,
 		.iopoll_queue		= 1,
-		.async_size		= sizeof(struct io_async_rw),
-		.name			= "WRITE",
 		.prep			= io_prep_rw,
 		.issue			= io_write,
-		.fail			= io_rw_fail,
 	},
 	[IORING_OP_FADVISE] = {
 		.needs_file		= 1,
 		.audit_skip		= 1,
-		.name			= "FADVISE",
 		.prep			= io_fadvise_prep,
 		.issue			= io_fadvise,
 	},
 	[IORING_OP_MADVISE] = {
-		.name			= "MADVISE",
+		.audit_skip		= 1,
 		.prep			= io_madvise_prep,
 		.issue			= io_madvise,
 	},
@@ -323,13 +268,9 @@ const struct io_op_def io_op_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.manual_alloc		= 1,
-		.name			= "SEND",
 #if defined(CONFIG_NET)
-		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_sendmsg_prep,
 		.issue			= io_send,
-		.fail			= io_sendrecv_fail,
-		.prep_async		= io_send_prep_async,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
@@ -341,25 +282,20 @@ const struct io_op_def io_op_defs[] = {
 		.buffer_select		= 1,
 		.audit_skip		= 1,
 		.ioprio			= 1,
-		.name			= "RECV",
 #if defined(CONFIG_NET)
 		.prep			= io_recvmsg_prep,
 		.issue			= io_recv,
-		.fail			= io_sendrecv_fail,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
 	},
 	[IORING_OP_OPENAT2] = {
-		.name			= "OPENAT2",
 		.prep			= io_openat2_prep,
 		.issue			= io_openat2,
-		.cleanup		= io_open_cleanup,
 	},
 	[IORING_OP_EPOLL_CTL] = {
 		.unbound_nonreg_file	= 1,
 		.audit_skip		= 1,
-		.name			= "EPOLL",
 #if defined(CONFIG_EPOLL)
 		.prep			= io_epoll_ctl_prep,
 		.issue			= io_epoll_ctl,
@@ -372,21 +308,18 @@ const struct io_op_def io_op_defs[] = {
 		.hash_reg_file		= 1,
 		.unbound_nonreg_file	= 1,
 		.audit_skip		= 1,
-		.name			= "SPLICE",
 		.prep			= io_splice_prep,
 		.issue			= io_splice,
 	},
 	[IORING_OP_PROVIDE_BUFFERS] = {
 		.audit_skip		= 1,
 		.iopoll			= 1,
-		.name			= "PROVIDE_BUFFERS",
 		.prep			= io_provide_buffers_prep,
 		.issue			= io_provide_buffers,
 	},
 	[IORING_OP_REMOVE_BUFFERS] = {
 		.audit_skip		= 1,
 		.iopoll			= 1,
-		.name			= "REMOVE_BUFFERS",
 		.prep			= io_remove_buffers_prep,
 		.issue			= io_remove_buffers,
 	},
@@ -395,13 +328,11 @@ const struct io_op_def io_op_defs[] = {
 		.hash_reg_file		= 1,
 		.unbound_nonreg_file	= 1,
 		.audit_skip		= 1,
-		.name			= "TEE",
 		.prep			= io_tee_prep,
 		.issue			= io_tee,
 	},
 	[IORING_OP_SHUTDOWN] = {
 		.needs_file		= 1,
-		.name			= "SHUTDOWN",
 #if defined(CONFIG_NET)
 		.prep			= io_shutdown_prep,
 		.issue			= io_shutdown,
@@ -410,72 +341,51 @@ const struct io_op_def io_op_defs[] = {
 #endif
 	},
 	[IORING_OP_RENAMEAT] = {
-		.name			= "RENAMEAT",
 		.prep			= io_renameat_prep,
 		.issue			= io_renameat,
-		.cleanup		= io_renameat_cleanup,
 	},
 	[IORING_OP_UNLINKAT] = {
-		.name			= "UNLINKAT",
 		.prep			= io_unlinkat_prep,
 		.issue			= io_unlinkat,
-		.cleanup		= io_unlinkat_cleanup,
 	},
 	[IORING_OP_MKDIRAT] = {
-		.name			= "MKDIRAT",
 		.prep			= io_mkdirat_prep,
 		.issue			= io_mkdirat,
-		.cleanup		= io_mkdirat_cleanup,
 	},
 	[IORING_OP_SYMLINKAT] = {
-		.name			= "SYMLINKAT",
 		.prep			= io_symlinkat_prep,
 		.issue			= io_symlinkat,
-		.cleanup		= io_link_cleanup,
 	},
 	[IORING_OP_LINKAT] = {
-		.name			= "LINKAT",
 		.prep			= io_linkat_prep,
 		.issue			= io_linkat,
-		.cleanup		= io_link_cleanup,
 	},
 	[IORING_OP_MSG_RING] = {
 		.needs_file		= 1,
 		.iopoll			= 1,
-		.name			= "MSG_RING",
 		.prep			= io_msg_ring_prep,
 		.issue			= io_msg_ring,
-		.cleanup		= io_msg_ring_cleanup,
 	},
 	[IORING_OP_FSETXATTR] = {
 		.needs_file = 1,
-		.name			= "FSETXATTR",
 		.prep			= io_fsetxattr_prep,
 		.issue			= io_fsetxattr,
-		.cleanup		= io_xattr_cleanup,
 	},
 	[IORING_OP_SETXATTR] = {
-		.name			= "SETXATTR",
 		.prep			= io_setxattr_prep,
 		.issue			= io_setxattr,
-		.cleanup		= io_xattr_cleanup,
 	},
 	[IORING_OP_FGETXATTR] = {
 		.needs_file = 1,
-		.name			= "FGETXATTR",
 		.prep			= io_fgetxattr_prep,
 		.issue			= io_fgetxattr,
-		.cleanup		= io_xattr_cleanup,
 	},
 	[IORING_OP_GETXATTR] = {
-		.name			= "GETXATTR",
 		.prep			= io_getxattr_prep,
 		.issue			= io_getxattr,
-		.cleanup		= io_xattr_cleanup,
 	},
 	[IORING_OP_SOCKET] = {
 		.audit_skip		= 1,
-		.name			= "SOCKET",
 #if defined(CONFIG_NET)
 		.prep			= io_socket_prep,
 		.issue			= io_socket,
@@ -486,16 +396,12 @@ const struct io_op_def io_op_defs[] = {
 	[IORING_OP_URING_CMD] = {
 		.needs_file		= 1,
 		.plug			= 1,
-		.name			= "URING_CMD",
 		.iopoll			= 1,
 		.iopoll_queue		= 1,
-		.async_size		= uring_cmd_pdu_size(1),
 		.prep			= io_uring_cmd_prep,
 		.issue			= io_uring_cmd,
-		.prep_async		= io_uring_cmd_prep_async,
 	},
 	[IORING_OP_SEND_ZC] = {
-		.name			= "SEND_ZC",
 		.needs_file		= 1,
 		.unbound_nonreg_file	= 1,
 		.pollout		= 1,
@@ -503,32 +409,243 @@ const struct io_op_def io_op_defs[] = {
 		.ioprio			= 1,
 		.manual_alloc		= 1,
 #if defined(CONFIG_NET)
-		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_send_zc_prep,
 		.issue			= io_send_zc,
-		.prep_async		= io_send_prep_async,
-		.cleanup		= io_send_zc_cleanup,
-		.fail			= io_sendrecv_fail,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
 	},
 	[IORING_OP_SENDMSG_ZC] = {
-		.name			= "SENDMSG_ZC",
 		.needs_file		= 1,
 		.unbound_nonreg_file	= 1,
 		.pollout		= 1,
 		.ioprio			= 1,
 		.manual_alloc		= 1,
 #if defined(CONFIG_NET)
-		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_send_zc_prep,
 		.issue			= io_sendmsg_zc,
+#else
+		.prep			= io_eopnotsupp_prep,
+#endif
+	},
+};
+
+
+const struct io_cold_def io_cold_defs[] = {
+	[IORING_OP_NOP] = {
+		.name			= "NOP",
+	},
+	[IORING_OP_READV] = {
+		.async_size		= sizeof(struct io_async_rw),
+		.name			= "READV",
+		.prep_async		= io_readv_prep_async,
+		.cleanup		= io_readv_writev_cleanup,
+		.fail			= io_rw_fail,
+	},
+	[IORING_OP_WRITEV] = {
+		.async_size		= sizeof(struct io_async_rw),
+		.name			= "WRITEV",
+		.prep_async		= io_writev_prep_async,
+		.cleanup		= io_readv_writev_cleanup,
+		.fail			= io_rw_fail,
+	},
+	[IORING_OP_FSYNC] = {
+		.name			= "FSYNC",
+	},
+	[IORING_OP_READ_FIXED] = {
+		.async_size		= sizeof(struct io_async_rw),
+		.name			= "READ_FIXED",
+		.fail			= io_rw_fail,
+	},
+	[IORING_OP_WRITE_FIXED] = {
+		.async_size		= sizeof(struct io_async_rw),
+		.name			= "WRITE_FIXED",
+		.fail			= io_rw_fail,
+	},
+	[IORING_OP_POLL_ADD] = {
+		.name			= "POLL_ADD",
+	},
+	[IORING_OP_POLL_REMOVE] = {
+		.name			= "POLL_REMOVE",
+	},
+	[IORING_OP_SYNC_FILE_RANGE] = {
+		.name			= "SYNC_FILE_RANGE",
+	},
+	[IORING_OP_SENDMSG] = {
+		.name			= "SENDMSG",
+#if defined(CONFIG_NET)
+		.async_size		= sizeof(struct io_async_msghdr),
+		.prep_async		= io_sendmsg_prep_async,
+		.cleanup		= io_sendmsg_recvmsg_cleanup,
+		.fail			= io_sendrecv_fail,
+#endif
+	},
+	[IORING_OP_RECVMSG] = {
+		.name			= "RECVMSG",
+#if defined(CONFIG_NET)
+		.async_size		= sizeof(struct io_async_msghdr),
+		.prep_async		= io_recvmsg_prep_async,
+		.cleanup		= io_sendmsg_recvmsg_cleanup,
+		.fail			= io_sendrecv_fail,
+#endif
+	},
+	[IORING_OP_TIMEOUT] = {
+		.async_size		= sizeof(struct io_timeout_data),
+		.name			= "TIMEOUT",
+	},
+	[IORING_OP_TIMEOUT_REMOVE] = {
+		.name			= "TIMEOUT_REMOVE",
+	},
+	[IORING_OP_ACCEPT] = {
+		.name			= "ACCEPT",
+	},
+	[IORING_OP_ASYNC_CANCEL] = {
+		.name			= "ASYNC_CANCEL",
+	},
+	[IORING_OP_LINK_TIMEOUT] = {
+		.async_size		= sizeof(struct io_timeout_data),
+		.name			= "LINK_TIMEOUT",
+	},
+	[IORING_OP_CONNECT] = {
+		.name			= "CONNECT",
+#if defined(CONFIG_NET)
+		.async_size		= sizeof(struct io_async_connect),
+		.prep_async		= io_connect_prep_async,
+#endif
+	},
+	[IORING_OP_FALLOCATE] = {
+		.name			= "FALLOCATE",
+	},
+	[IORING_OP_OPENAT] = {
+		.name			= "OPENAT",
+		.cleanup		= io_open_cleanup,
+	},
+	[IORING_OP_CLOSE] = {
+		.name			= "CLOSE",
+	},
+	[IORING_OP_FILES_UPDATE] = {
+		.name			= "FILES_UPDATE",
+	},
+	[IORING_OP_STATX] = {
+		.name			= "STATX",
+		.cleanup		= io_statx_cleanup,
+	},
+	[IORING_OP_READ] = {
+		.async_size		= sizeof(struct io_async_rw),
+		.name			= "READ",
+		.fail			= io_rw_fail,
+	},
+	[IORING_OP_WRITE] = {
+		.async_size		= sizeof(struct io_async_rw),
+		.name			= "WRITE",
+		.fail			= io_rw_fail,
+	},
+	[IORING_OP_FADVISE] = {
+		.name			= "FADVISE",
+	},
+	[IORING_OP_MADVISE] = {
+		.name			= "MADVISE",
+	},
+	[IORING_OP_SEND] = {
+		.name			= "SEND",
+#if defined(CONFIG_NET)
+		.async_size		= sizeof(struct io_async_msghdr),
+		.fail			= io_sendrecv_fail,
+		.prep_async		= io_send_prep_async,
+#endif
+	},
+	[IORING_OP_RECV] = {
+		.name			= "RECV",
+#if defined(CONFIG_NET)
+		.fail			= io_sendrecv_fail,
+#endif
+	},
+	[IORING_OP_OPENAT2] = {
+		.name			= "OPENAT2",
+		.cleanup		= io_open_cleanup,
+	},
+	[IORING_OP_EPOLL_CTL] = {
+		.name			= "EPOLL",
+	},
+	[IORING_OP_SPLICE] = {
+		.name			= "SPLICE",
+	},
+	[IORING_OP_PROVIDE_BUFFERS] = {
+		.name			= "PROVIDE_BUFFERS",
+	},
+	[IORING_OP_REMOVE_BUFFERS] = {
+		.name			= "REMOVE_BUFFERS",
+	},
+	[IORING_OP_TEE] = {
+		.name			= "TEE",
+	},
+	[IORING_OP_SHUTDOWN] = {
+		.name			= "SHUTDOWN",
+	},
+	[IORING_OP_RENAMEAT] = {
+		.name			= "RENAMEAT",
+		.cleanup		= io_renameat_cleanup,
+	},
+	[IORING_OP_UNLINKAT] = {
+		.name			= "UNLINKAT",
+		.cleanup		= io_unlinkat_cleanup,
+	},
+	[IORING_OP_MKDIRAT] = {
+		.name			= "MKDIRAT",
+		.cleanup		= io_mkdirat_cleanup,
+	},
+	[IORING_OP_SYMLINKAT] = {
+		.name			= "SYMLINKAT",
+		.cleanup		= io_link_cleanup,
+	},
+	[IORING_OP_LINKAT] = {
+		.name			= "LINKAT",
+		.cleanup		= io_link_cleanup,
+	},
+	[IORING_OP_MSG_RING] = {
+		.name			= "MSG_RING",
+		.cleanup		= io_msg_ring_cleanup,
+	},
+	[IORING_OP_FSETXATTR] = {
+		.name			= "FSETXATTR",
+		.cleanup		= io_xattr_cleanup,
+	},
+	[IORING_OP_SETXATTR] = {
+		.name			= "SETXATTR",
+		.cleanup		= io_xattr_cleanup,
+	},
+	[IORING_OP_FGETXATTR] = {
+		.name			= "FGETXATTR",
+		.cleanup		= io_xattr_cleanup,
+	},
+	[IORING_OP_GETXATTR] = {
+		.name			= "GETXATTR",
+		.cleanup		= io_xattr_cleanup,
+	},
+	[IORING_OP_SOCKET] = {
+		.name			= "SOCKET",
+	},
+	[IORING_OP_URING_CMD] = {
+		.name			= "URING_CMD",
+		.async_size		= uring_cmd_pdu_size(1),
+		.prep_async		= io_uring_cmd_prep_async,
+	},
+	[IORING_OP_SEND_ZC] = {
+		.name			= "SEND_ZC",
+#if defined(CONFIG_NET)
+		.async_size		= sizeof(struct io_async_msghdr),
+		.prep_async		= io_send_prep_async,
+		.cleanup		= io_send_zc_cleanup,
+		.fail			= io_sendrecv_fail,
+#endif
+	},
+	[IORING_OP_SENDMSG_ZC] = {
+		.name			= "SENDMSG_ZC",
+#if defined(CONFIG_NET)
+		.async_size		= sizeof(struct io_async_msghdr),
 		.prep_async		= io_sendmsg_prep_async,
 		.cleanup		= io_send_zc_cleanup,
 		.fail			= io_sendrecv_fail,
-#else
-		.prep			= io_eopnotsupp_prep,
 #endif
 	},
 };
@@ -536,7 +653,7 @@ const struct io_op_def io_op_defs[] = {
 const char *io_uring_get_opcode(u8 opcode)
 {
 	if (opcode < IORING_OP_LAST)
-		return io_op_defs[opcode].name;
+		return io_cold_defs[opcode].name;
 	return "INVALID";
 }
 
@@ -544,12 +661,13 @@ void __init io_uring_optable_init(void)
 {
 	int i;
 
-	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
+	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
+	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 
-	for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
-		BUG_ON(!io_op_defs[i].prep);
-		if (io_op_defs[i].prep != io_eopnotsupp_prep)
-			BUG_ON(!io_op_defs[i].issue);
-		WARN_ON_ONCE(!io_op_defs[i].name);
+	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
+		BUG_ON(!io_issue_defs[i].prep);
+		if (io_issue_defs[i].prep != io_eopnotsupp_prep)
+			BUG_ON(!io_issue_defs[i].issue);
+		WARN_ON_ONCE(!io_cold_defs[i].name);
 	}
 }