2022-06-15 22:27:42 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
|
/*
|
|
|
|
|
* io_uring opcode handling table
|
|
|
|
|
*/
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
|
#include <linux/errno.h>
|
|
|
|
|
#include <linux/fs.h>
|
|
|
|
|
#include <linux/file.h>
|
|
|
|
|
#include <linux/io_uring.h>
|
|
|
|
|
|
|
|
|
|
#include "io_uring.h"
|
|
|
|
|
#include "opdef.h"
|
|
|
|
|
#include "refs.h"
|
|
|
|
|
#include "tctx.h"
|
|
|
|
|
#include "sqpoll.h"
|
|
|
|
|
#include "fdinfo.h"
|
|
|
|
|
#include "kbuf.h"
|
|
|
|
|
#include "rsrc.h"
|
|
|
|
|
|
|
|
|
|
#include "xattr.h"
|
|
|
|
|
#include "nop.h"
|
|
|
|
|
#include "fs.h"
|
|
|
|
|
#include "splice.h"
|
|
|
|
|
#include "sync.h"
|
|
|
|
|
#include "advise.h"
|
|
|
|
|
#include "openclose.h"
|
|
|
|
|
#include "uring_cmd.h"
|
|
|
|
|
#include "epoll.h"
|
|
|
|
|
#include "statx.h"
|
|
|
|
|
#include "net.h"
|
|
|
|
|
#include "msg_ring.h"
|
|
|
|
|
#include "timeout.h"
|
|
|
|
|
#include "poll.h"
|
|
|
|
|
#include "cancel.h"
|
|
|
|
|
#include "rw.h"
|
2023-07-10 22:14:37 +00:00
|
|
|
#include "waitid.h"
|
io_uring: add support for futex wake and wait
JIRA: https://issues.redhat.com/browse/RHEL-27755
commit 194bb58c6090e39bd7d9b9c888a079213628e1f6
Author: Jens Axboe <axboe@kernel.dk>
Date: Thu Jun 8 11:57:40 2023 -0600
io_uring: add support for futex wake and wait
Add support for FUTEX_WAKE/WAIT primitives.
IORING_OP_FUTEX_WAKE is mix of FUTEX_WAKE and FUTEX_WAKE_BITSET, as
it does support passing in a bitset.
Similary, IORING_OP_FUTEX_WAIT is a mix of FUTEX_WAIT and
FUTEX_WAIT_BITSET.
For both of them, they are using the futex2 interface.
FUTEX_WAKE is straight forward, as those can always be done directly from
the io_uring submission without needing async handling. For FUTEX_WAIT,
things are a bit more complicated. If the futex isn't ready, then we
rely on a callback via futex_queue->wake() when someone wakes up the
futex. From that calback, we queue up task_work with the original task,
which will post a CQE and wake it, if necessary.
Cancelations are supported, both from the application point-of-view,
but also to be able to cancel pending waits if the ring exits before
all events have occurred. The return value of futex_unqueue() is used
to gate who wins the potential race between cancelation and futex
wakeups. Whomever gets a 'ret == 1' return from that claims ownership
of the io_uring futex request.
This is just the barebones wait/wake support. PI or REQUEUE support is
not added at this point, unclear if we might look into that later.
Likewise, explicit timeouts are not supported either. It is expected
that users that need timeouts would do so via the usual io_uring
mechanism to do that using linked timeouts.
The SQE format is as follows:
`addr` Address of futex
`fd` futex2(2) FUTEX2_* flags
`futex_flags` io_uring specific command flags. None valid now.
`addr2` Value of futex
`addr3` Mask to wake/wait
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
2023-06-08 17:57:40 +00:00
|
|
|
#include "futex.h"
|
2022-06-15 22:27:42 +00:00
|
|
|
|
|
|
|
|
static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
|
|
|
|
|
{
|
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
|
return -ECANCELED;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
|
|
|
|
|
const struct io_uring_sqe *sqe)
|
|
|
|
|
{
|
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-12 14:44:10 +00:00
|
|
|
const struct io_issue_def io_issue_defs[] = {
|
2022-06-15 22:27:42 +00:00
|
|
|
[IORING_OP_NOP] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.iopoll = 1,
|
|
|
|
|
.prep = io_nop_prep,
|
|
|
|
|
.issue = io_nop,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_READV] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollin = 1,
|
|
|
|
|
.buffer_select = 1,
|
|
|
|
|
.plug = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
.iopoll = 1,
|
2022-12-07 03:53:26 +00:00
|
|
|
.iopoll_queue = 1,
|
2023-09-11 19:46:07 +00:00
|
|
|
.vectored = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_prep_rw,
|
|
|
|
|
.issue = io_read,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_WRITEV] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.hash_reg_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
.plug = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
.iopoll = 1,
|
2022-12-07 03:53:26 +00:00
|
|
|
.iopoll_queue = 1,
|
2023-09-11 19:46:07 +00:00
|
|
|
.vectored = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_prep_rw,
|
|
|
|
|
.issue = io_write,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FSYNC] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_fsync_prep,
|
|
|
|
|
.issue = io_fsync,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_READ_FIXED] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollin = 1,
|
|
|
|
|
.plug = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
.iopoll = 1,
|
2022-12-07 03:53:26 +00:00
|
|
|
.iopoll_queue = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_prep_rw,
|
|
|
|
|
.issue = io_read,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_WRITE_FIXED] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.hash_reg_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
.plug = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
.iopoll = 1,
|
2022-12-07 03:53:26 +00:00
|
|
|
.iopoll_queue = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_prep_rw,
|
|
|
|
|
.issue = io_write,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_POLL_ADD] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_poll_add_prep,
|
|
|
|
|
.issue = io_poll_add,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_POLL_REMOVE] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_poll_remove_prep,
|
|
|
|
|
.issue = io_poll_remove,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SYNC_FILE_RANGE] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_sfr_prep,
|
|
|
|
|
.issue = io_sync_file_range,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SENDMSG] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
.ioprio = 1,
|
2022-09-08 12:20:30 +00:00
|
|
|
.manual_alloc = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_sendmsg_prep,
|
|
|
|
|
.issue = io_sendmsg,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_RECVMSG] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollin = 1,
|
|
|
|
|
.buffer_select = 1,
|
|
|
|
|
.ioprio = 1,
|
2022-09-08 12:20:30 +00:00
|
|
|
.manual_alloc = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_recvmsg_prep,
|
|
|
|
|
.issue = io_recvmsg,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_TIMEOUT] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_timeout_prep,
|
|
|
|
|
.issue = io_timeout,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_TIMEOUT_REMOVE] = {
|
|
|
|
|
/* used by timeout updates' prep() */
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_timeout_remove_prep,
|
|
|
|
|
.issue = io_timeout_remove,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_ACCEPT] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollin = 1,
|
|
|
|
|
.poll_exclusive = 1,
|
|
|
|
|
.ioprio = 1, /* used for flags */
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_accept_prep,
|
|
|
|
|
.issue = io_accept,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_ASYNC_CANCEL] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_async_cancel_prep,
|
|
|
|
|
.issue = io_async_cancel,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_LINK_TIMEOUT] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_link_timeout_prep,
|
|
|
|
|
.issue = io_no_issue,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_CONNECT] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_connect_prep,
|
|
|
|
|
.issue = io_connect,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FALLOCATE] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.prep = io_fallocate_prep,
|
|
|
|
|
.issue = io_fallocate,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_OPENAT] = {
|
|
|
|
|
.prep = io_openat_prep,
|
|
|
|
|
.issue = io_openat,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_CLOSE] = {
|
|
|
|
|
.prep = io_close_prep,
|
|
|
|
|
.issue = io_close,
|
|
|
|
|
},
|
2022-09-01 10:54:02 +00:00
|
|
|
[IORING_OP_FILES_UPDATE] = {
|
2022-06-15 22:27:42 +00:00
|
|
|
.audit_skip = 1,
|
|
|
|
|
.iopoll = 1,
|
2022-09-01 10:54:02 +00:00
|
|
|
.prep = io_files_update_prep,
|
|
|
|
|
.issue = io_files_update,
|
2022-06-15 22:27:42 +00:00
|
|
|
},
|
|
|
|
|
[IORING_OP_STATX] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_statx_prep,
|
|
|
|
|
.issue = io_statx,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_READ] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollin = 1,
|
|
|
|
|
.buffer_select = 1,
|
|
|
|
|
.plug = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
.iopoll = 1,
|
2022-12-07 03:53:26 +00:00
|
|
|
.iopoll_queue = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_prep_rw,
|
|
|
|
|
.issue = io_read,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_WRITE] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.hash_reg_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
.plug = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
.iopoll = 1,
|
2022-12-07 03:53:26 +00:00
|
|
|
.iopoll_queue = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_prep_rw,
|
|
|
|
|
.issue = io_write,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FADVISE] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_fadvise_prep,
|
|
|
|
|
.issue = io_fadvise,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_MADVISE] = {
|
2023-02-01 20:33:33 +00:00
|
|
|
.audit_skip = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_madvise_prep,
|
|
|
|
|
.issue = io_madvise,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SEND] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
2022-09-21 11:17:51 +00:00
|
|
|
.manual_alloc = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_sendmsg_prep,
|
|
|
|
|
.issue = io_send,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_RECV] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollin = 1,
|
|
|
|
|
.buffer_select = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_recvmsg_prep,
|
|
|
|
|
.issue = io_recv,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_OPENAT2] = {
|
|
|
|
|
.prep = io_openat2_prep,
|
|
|
|
|
.issue = io_openat2,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_EPOLL_CTL] = {
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
#if defined(CONFIG_EPOLL)
|
|
|
|
|
.prep = io_epoll_ctl_prep,
|
|
|
|
|
.issue = io_epoll_ctl,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SPLICE] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.hash_reg_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_splice_prep,
|
|
|
|
|
.issue = io_splice,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_PROVIDE_BUFFERS] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.iopoll = 1,
|
|
|
|
|
.prep = io_provide_buffers_prep,
|
|
|
|
|
.issue = io_provide_buffers,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_REMOVE_BUFFERS] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.iopoll = 1,
|
|
|
|
|
.prep = io_remove_buffers_prep,
|
|
|
|
|
.issue = io_remove_buffers,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_TEE] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.hash_reg_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_tee_prep,
|
|
|
|
|
.issue = io_tee,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SHUTDOWN] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_shutdown_prep,
|
|
|
|
|
.issue = io_shutdown,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_RENAMEAT] = {
|
|
|
|
|
.prep = io_renameat_prep,
|
|
|
|
|
.issue = io_renameat,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_UNLINKAT] = {
|
|
|
|
|
.prep = io_unlinkat_prep,
|
|
|
|
|
.issue = io_unlinkat,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_MKDIRAT] = {
|
|
|
|
|
.prep = io_mkdirat_prep,
|
|
|
|
|
.issue = io_mkdirat,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SYMLINKAT] = {
|
|
|
|
|
.prep = io_symlinkat_prep,
|
|
|
|
|
.issue = io_symlinkat,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_LINKAT] = {
|
|
|
|
|
.prep = io_linkat_prep,
|
|
|
|
|
.issue = io_linkat,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_MSG_RING] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.iopoll = 1,
|
|
|
|
|
.prep = io_msg_ring_prep,
|
|
|
|
|
.issue = io_msg_ring,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FSETXATTR] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.prep = io_fsetxattr_prep,
|
|
|
|
|
.issue = io_fsetxattr,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SETXATTR] = {
|
|
|
|
|
.prep = io_setxattr_prep,
|
|
|
|
|
.issue = io_setxattr,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FGETXATTR] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.prep = io_fgetxattr_prep,
|
|
|
|
|
.issue = io_fgetxattr,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_GETXATTR] = {
|
|
|
|
|
.prep = io_getxattr_prep,
|
|
|
|
|
.issue = io_getxattr,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SOCKET] = {
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_socket_prep,
|
|
|
|
|
.issue = io_socket,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_URING_CMD] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.plug = 1,
|
2022-08-23 16:14:41 +00:00
|
|
|
.iopoll = 1,
|
2022-12-07 03:53:26 +00:00
|
|
|
.iopoll_queue = 1,
|
2022-06-15 22:27:42 +00:00
|
|
|
.prep = io_uring_cmd_prep,
|
|
|
|
|
.issue = io_uring_cmd,
|
|
|
|
|
},
|
2022-09-01 10:54:04 +00:00
|
|
|
[IORING_OP_SEND_ZC] = {
|
2022-07-12 20:52:43 +00:00
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.ioprio = 1,
|
2022-08-24 12:07:43 +00:00
|
|
|
.manual_alloc = 1,
|
2022-07-12 20:52:43 +00:00
|
|
|
#if defined(CONFIG_NET)
|
2022-09-21 11:17:52 +00:00
|
|
|
.prep = io_send_zc_prep,
|
|
|
|
|
.issue = io_send_zc,
|
2022-07-12 20:52:43 +00:00
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
2022-09-21 11:17:54 +00:00
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SENDMSG_ZC] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollout = 1,
|
|
|
|
|
.ioprio = 1,
|
|
|
|
|
.manual_alloc = 1,
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.prep = io_send_zc_prep,
|
|
|
|
|
.issue = io_sendmsg_zc,
|
2023-01-12 14:44:11 +00:00
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
2023-09-11 19:35:42 +00:00
|
|
|
[IORING_OP_READ_MULTISHOT] = {
|
|
|
|
|
.needs_file = 1,
|
|
|
|
|
.unbound_nonreg_file = 1,
|
|
|
|
|
.pollin = 1,
|
|
|
|
|
.buffer_select = 1,
|
|
|
|
|
.audit_skip = 1,
|
|
|
|
|
.prep = io_read_mshot_prep,
|
|
|
|
|
.issue = io_read_mshot,
|
|
|
|
|
},
|
2023-07-10 22:14:37 +00:00
|
|
|
[IORING_OP_WAITID] = {
|
|
|
|
|
.prep = io_waitid_prep,
|
|
|
|
|
.issue = io_waitid,
|
|
|
|
|
},
|
io_uring: add support for futex wake and wait
JIRA: https://issues.redhat.com/browse/RHEL-27755
commit 194bb58c6090e39bd7d9b9c888a079213628e1f6
Author: Jens Axboe <axboe@kernel.dk>
Date: Thu Jun 8 11:57:40 2023 -0600
io_uring: add support for futex wake and wait
Add support for FUTEX_WAKE/WAIT primitives.
IORING_OP_FUTEX_WAKE is mix of FUTEX_WAKE and FUTEX_WAKE_BITSET, as
it does support passing in a bitset.
Similary, IORING_OP_FUTEX_WAIT is a mix of FUTEX_WAIT and
FUTEX_WAIT_BITSET.
For both of them, they are using the futex2 interface.
FUTEX_WAKE is straight forward, as those can always be done directly from
the io_uring submission without needing async handling. For FUTEX_WAIT,
things are a bit more complicated. If the futex isn't ready, then we
rely on a callback via futex_queue->wake() when someone wakes up the
futex. From that calback, we queue up task_work with the original task,
which will post a CQE and wake it, if necessary.
Cancelations are supported, both from the application point-of-view,
but also to be able to cancel pending waits if the ring exits before
all events have occurred. The return value of futex_unqueue() is used
to gate who wins the potential race between cancelation and futex
wakeups. Whomever gets a 'ret == 1' return from that claims ownership
of the io_uring futex request.
This is just the barebones wait/wake support. PI or REQUEUE support is
not added at this point, unclear if we might look into that later.
Likewise, explicit timeouts are not supported either. It is expected
that users that need timeouts would do so via the usual io_uring
mechanism to do that using linked timeouts.
The SQE format is as follows:
`addr` Address of futex
`fd` futex2(2) FUTEX2_* flags
`futex_flags` io_uring specific command flags. None valid now.
`addr2` Value of futex
`addr3` Mask to wake/wait
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
2023-06-08 17:57:40 +00:00
|
|
|
[IORING_OP_FUTEX_WAIT] = {
|
|
|
|
|
#if defined(CONFIG_FUTEX)
|
|
|
|
|
.prep = io_futex_prep,
|
|
|
|
|
.issue = io_futex_wait,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FUTEX_WAKE] = {
|
|
|
|
|
#if defined(CONFIG_FUTEX)
|
|
|
|
|
.prep = io_futex_prep,
|
|
|
|
|
.issue = io_futex_wake,
|
|
|
|
|
#else
|
|
|
|
|
.prep = io_eopnotsupp_prep,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
2023-01-12 14:44:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const struct io_cold_def io_cold_defs[] = {
|
|
|
|
|
[IORING_OP_NOP] = {
|
|
|
|
|
.name = "NOP",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_READV] = {
|
|
|
|
|
.async_size = sizeof(struct io_async_rw),
|
|
|
|
|
.name = "READV",
|
|
|
|
|
.prep_async = io_readv_prep_async,
|
|
|
|
|
.cleanup = io_readv_writev_cleanup,
|
|
|
|
|
.fail = io_rw_fail,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_WRITEV] = {
|
|
|
|
|
.async_size = sizeof(struct io_async_rw),
|
|
|
|
|
.name = "WRITEV",
|
|
|
|
|
.prep_async = io_writev_prep_async,
|
|
|
|
|
.cleanup = io_readv_writev_cleanup,
|
|
|
|
|
.fail = io_rw_fail,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FSYNC] = {
|
|
|
|
|
.name = "FSYNC",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_READ_FIXED] = {
|
|
|
|
|
.async_size = sizeof(struct io_async_rw),
|
|
|
|
|
.name = "READ_FIXED",
|
|
|
|
|
.fail = io_rw_fail,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_WRITE_FIXED] = {
|
|
|
|
|
.async_size = sizeof(struct io_async_rw),
|
|
|
|
|
.name = "WRITE_FIXED",
|
|
|
|
|
.fail = io_rw_fail,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_POLL_ADD] = {
|
|
|
|
|
.name = "POLL_ADD",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_POLL_REMOVE] = {
|
|
|
|
|
.name = "POLL_REMOVE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SYNC_FILE_RANGE] = {
|
|
|
|
|
.name = "SYNC_FILE_RANGE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SENDMSG] = {
|
|
|
|
|
.name = "SENDMSG",
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.async_size = sizeof(struct io_async_msghdr),
|
|
|
|
|
.prep_async = io_sendmsg_prep_async,
|
|
|
|
|
.cleanup = io_sendmsg_recvmsg_cleanup,
|
|
|
|
|
.fail = io_sendrecv_fail,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_RECVMSG] = {
|
|
|
|
|
.name = "RECVMSG",
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.async_size = sizeof(struct io_async_msghdr),
|
|
|
|
|
.prep_async = io_recvmsg_prep_async,
|
|
|
|
|
.cleanup = io_sendmsg_recvmsg_cleanup,
|
|
|
|
|
.fail = io_sendrecv_fail,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_TIMEOUT] = {
|
|
|
|
|
.async_size = sizeof(struct io_timeout_data),
|
|
|
|
|
.name = "TIMEOUT",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_TIMEOUT_REMOVE] = {
|
|
|
|
|
.name = "TIMEOUT_REMOVE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_ACCEPT] = {
|
|
|
|
|
.name = "ACCEPT",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_ASYNC_CANCEL] = {
|
|
|
|
|
.name = "ASYNC_CANCEL",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_LINK_TIMEOUT] = {
|
|
|
|
|
.async_size = sizeof(struct io_timeout_data),
|
|
|
|
|
.name = "LINK_TIMEOUT",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_CONNECT] = {
|
|
|
|
|
.name = "CONNECT",
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.async_size = sizeof(struct io_async_connect),
|
|
|
|
|
.prep_async = io_connect_prep_async,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FALLOCATE] = {
|
|
|
|
|
.name = "FALLOCATE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_OPENAT] = {
|
|
|
|
|
.name = "OPENAT",
|
|
|
|
|
.cleanup = io_open_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_CLOSE] = {
|
|
|
|
|
.name = "CLOSE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FILES_UPDATE] = {
|
|
|
|
|
.name = "FILES_UPDATE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_STATX] = {
|
|
|
|
|
.name = "STATX",
|
|
|
|
|
.cleanup = io_statx_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_READ] = {
|
|
|
|
|
.async_size = sizeof(struct io_async_rw),
|
|
|
|
|
.name = "READ",
|
|
|
|
|
.fail = io_rw_fail,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_WRITE] = {
|
|
|
|
|
.async_size = sizeof(struct io_async_rw),
|
|
|
|
|
.name = "WRITE",
|
|
|
|
|
.fail = io_rw_fail,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FADVISE] = {
|
|
|
|
|
.name = "FADVISE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_MADVISE] = {
|
|
|
|
|
.name = "MADVISE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SEND] = {
|
|
|
|
|
.name = "SEND",
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.async_size = sizeof(struct io_async_msghdr),
|
|
|
|
|
.fail = io_sendrecv_fail,
|
|
|
|
|
.prep_async = io_send_prep_async,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_RECV] = {
|
|
|
|
|
.name = "RECV",
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.fail = io_sendrecv_fail,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_OPENAT2] = {
|
|
|
|
|
.name = "OPENAT2",
|
|
|
|
|
.cleanup = io_open_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_EPOLL_CTL] = {
|
|
|
|
|
.name = "EPOLL",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SPLICE] = {
|
|
|
|
|
.name = "SPLICE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_PROVIDE_BUFFERS] = {
|
|
|
|
|
.name = "PROVIDE_BUFFERS",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_REMOVE_BUFFERS] = {
|
|
|
|
|
.name = "REMOVE_BUFFERS",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_TEE] = {
|
|
|
|
|
.name = "TEE",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SHUTDOWN] = {
|
|
|
|
|
.name = "SHUTDOWN",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_RENAMEAT] = {
|
|
|
|
|
.name = "RENAMEAT",
|
|
|
|
|
.cleanup = io_renameat_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_UNLINKAT] = {
|
|
|
|
|
.name = "UNLINKAT",
|
|
|
|
|
.cleanup = io_unlinkat_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_MKDIRAT] = {
|
|
|
|
|
.name = "MKDIRAT",
|
|
|
|
|
.cleanup = io_mkdirat_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SYMLINKAT] = {
|
|
|
|
|
.name = "SYMLINKAT",
|
|
|
|
|
.cleanup = io_link_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_LINKAT] = {
|
|
|
|
|
.name = "LINKAT",
|
|
|
|
|
.cleanup = io_link_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_MSG_RING] = {
|
|
|
|
|
.name = "MSG_RING",
|
|
|
|
|
.cleanup = io_msg_ring_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FSETXATTR] = {
|
|
|
|
|
.name = "FSETXATTR",
|
|
|
|
|
.cleanup = io_xattr_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SETXATTR] = {
|
|
|
|
|
.name = "SETXATTR",
|
|
|
|
|
.cleanup = io_xattr_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FGETXATTR] = {
|
|
|
|
|
.name = "FGETXATTR",
|
|
|
|
|
.cleanup = io_xattr_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_GETXATTR] = {
|
|
|
|
|
.name = "GETXATTR",
|
|
|
|
|
.cleanup = io_xattr_cleanup,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SOCKET] = {
|
|
|
|
|
.name = "SOCKET",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_URING_CMD] = {
|
|
|
|
|
.name = "URING_CMD",
|
io_uring: Pass whole sqe to commands
JIRA: https://issues.redhat.com/browse/RHEL-12076
Conflicts: RHEL does not yet support ublk, so this backport removes
those changes from the patch.
commit fd9b8547bc5c34186dc42ea05fb4380d21695374
Author: Breno Leitao <leitao@debian.org>
Date: Thu May 4 05:18:55 2023 -0700
io_uring: Pass whole sqe to commands
Currently uring CMD operation relies on having large SQEs, but future
operations might want to use normal SQE.
The io_uring_cmd currently only saves the payload (cmd) part of the SQE,
but, for commands that use normal SQE size, it might be necessary to
access the initial SQE fields outside of the payload/cmd block. So,
saves the whole SQE other than just the pdu.
This changes slightly how the io_uring_cmd works, since the cmd
structures and callbacks are not opaque to io_uring anymore. I.e, the
callbacks can look at the SQE entries, not only, in the cmd structure.
The main advantage is that we don't need to create custom structures for
simple commands.
Creates io_uring_sqe_cmd() that returns the cmd private data as a null
pointer and avoids casting in the callee side.
Also, make most of ublk_drv's sqe->cmd priv structure into const, and use
io_uring_sqe_cmd() to get the private structure, removing the unwanted
cast. (There is one case where the cast is still needed since the
header->{len,addr} is updated in the private structure)
Suggested-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Breno Leitao <leitao@debian.org>
Reviewed-by: Keith Busch <kbusch@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/20230504121856.904491-3-leitao@debian.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
2023-05-04 12:18:55 +00:00
|
|
|
.async_size = 2 * sizeof(struct io_uring_sqe),
|
2023-01-12 14:44:11 +00:00
|
|
|
.prep_async = io_uring_cmd_prep_async,
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SEND_ZC] = {
|
|
|
|
|
.name = "SEND_ZC",
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.async_size = sizeof(struct io_async_msghdr),
|
|
|
|
|
.prep_async = io_send_prep_async,
|
|
|
|
|
.cleanup = io_send_zc_cleanup,
|
|
|
|
|
.fail = io_sendrecv_fail,
|
|
|
|
|
#endif
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_SENDMSG_ZC] = {
|
|
|
|
|
.name = "SENDMSG_ZC",
|
|
|
|
|
#if defined(CONFIG_NET)
|
|
|
|
|
.async_size = sizeof(struct io_async_msghdr),
|
2022-09-21 11:17:54 +00:00
|
|
|
.prep_async = io_sendmsg_prep_async,
|
|
|
|
|
.cleanup = io_send_zc_cleanup,
|
|
|
|
|
.fail = io_sendrecv_fail,
|
2022-07-12 20:52:43 +00:00
|
|
|
#endif
|
|
|
|
|
},
|
2023-09-11 19:35:42 +00:00
|
|
|
[IORING_OP_READ_MULTISHOT] = {
|
|
|
|
|
.name = "READ_MULTISHOT",
|
|
|
|
|
},
|
2023-07-10 22:14:37 +00:00
|
|
|
[IORING_OP_WAITID] = {
|
|
|
|
|
.name = "WAITID",
|
|
|
|
|
.async_size = sizeof(struct io_waitid_async),
|
|
|
|
|
},
|
io_uring: add support for futex wake and wait
JIRA: https://issues.redhat.com/browse/RHEL-27755
commit 194bb58c6090e39bd7d9b9c888a079213628e1f6
Author: Jens Axboe <axboe@kernel.dk>
Date: Thu Jun 8 11:57:40 2023 -0600
io_uring: add support for futex wake and wait
Add support for FUTEX_WAKE/WAIT primitives.
IORING_OP_FUTEX_WAKE is mix of FUTEX_WAKE and FUTEX_WAKE_BITSET, as
it does support passing in a bitset.
Similary, IORING_OP_FUTEX_WAIT is a mix of FUTEX_WAIT and
FUTEX_WAIT_BITSET.
For both of them, they are using the futex2 interface.
FUTEX_WAKE is straight forward, as those can always be done directly from
the io_uring submission without needing async handling. For FUTEX_WAIT,
things are a bit more complicated. If the futex isn't ready, then we
rely on a callback via futex_queue->wake() when someone wakes up the
futex. From that calback, we queue up task_work with the original task,
which will post a CQE and wake it, if necessary.
Cancelations are supported, both from the application point-of-view,
but also to be able to cancel pending waits if the ring exits before
all events have occurred. The return value of futex_unqueue() is used
to gate who wins the potential race between cancelation and futex
wakeups. Whomever gets a 'ret == 1' return from that claims ownership
of the io_uring futex request.
This is just the barebones wait/wake support. PI or REQUEUE support is
not added at this point, unclear if we might look into that later.
Likewise, explicit timeouts are not supported either. It is expected
that users that need timeouts would do so via the usual io_uring
mechanism to do that using linked timeouts.
The SQE format is as follows:
`addr` Address of futex
`fd` futex2(2) FUTEX2_* flags
`futex_flags` io_uring specific command flags. None valid now.
`addr2` Value of futex
`addr3` Mask to wake/wait
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
2023-06-08 17:57:40 +00:00
|
|
|
[IORING_OP_FUTEX_WAIT] = {
|
|
|
|
|
.name = "FUTEX_WAIT",
|
|
|
|
|
},
|
|
|
|
|
[IORING_OP_FUTEX_WAKE] = {
|
|
|
|
|
.name = "FUTEX_WAKE",
|
|
|
|
|
},
|
2022-06-15 22:27:42 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const char *io_uring_get_opcode(u8 opcode)
|
|
|
|
|
{
|
|
|
|
|
if (opcode < IORING_OP_LAST)
|
2023-01-12 14:44:11 +00:00
|
|
|
return io_cold_defs[opcode].name;
|
2022-06-15 22:27:42 +00:00
|
|
|
return "INVALID";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void __init io_uring_optable_init(void)
|
|
|
|
|
{
|
|
|
|
|
int i;
|
|
|
|
|
|
2023-01-12 14:44:11 +00:00
|
|
|
BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
|
2023-01-12 14:44:10 +00:00
|
|
|
BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
|
2022-06-15 22:27:42 +00:00
|
|
|
|
2023-01-12 14:44:10 +00:00
|
|
|
for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
|
|
|
|
|
BUG_ON(!io_issue_defs[i].prep);
|
|
|
|
|
if (io_issue_defs[i].prep != io_eopnotsupp_prep)
|
|
|
|
|
BUG_ON(!io_issue_defs[i].issue);
|
2023-01-12 14:44:11 +00:00
|
|
|
WARN_ON_ONCE(!io_cold_defs[i].name);
|
2022-06-15 22:27:42 +00:00
|
|
|
}
|
|
|
|
|
}
|