qiurui/Centos-kernel-stream-10
1
0
Fork 0
mirror of https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-10.git synced 2026-09-09 00:07:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

io_uring: introduce type alias for io_tw_state

Browse Source 
JIRA: https://issues.redhat.com/browse/RHEL-106845

commit bcf8a0293a019bb0c4aebafdebe9a1e7a923249a
Author: Caleb Sander Mateos <csander@purestorage.com>
Date:   Sun Feb 16 19:25:04 2025 -0700

    io_uring: introduce type alias for io_tw_state

    In preparation for changing how io_tw_state is passed, introduce a type
    alias io_tw_token_t for struct io_tw_state *. This allows for changing
    the representation in one place, without having to update the many
    functions that just forward their struct io_tw_state * argument.

    Also add a comment to struct io_tw_state to explain its purpose.

    Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
    Link: https://lore.kernel.org/r/20250217022511.1150145-1-csander@purestorage.com
    Signed-off-by: Jens Axboe <axboe@kernel.dk>

Signed-off-by: Ming Lei <ming.lei@redhat.com>
This commit is contained in:
Ming Lei
2025-08-26 12:16:52 +08:00
parent afd0f450e8
commit 288688eb82
13 changed files with 66 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/linux/io_uring_types.h
+8 -1
View File
@@ -436,8 +436,15 @@ struct io_ring_ctx {
struct io_mapped_region param_region; struct io_mapped_region param_region;
}; };
/*
* Token indicating function is called in task work context:
* ctx->uring_lock is held and any completions generated will be flushed.
* ONLY core io_uring.c should instantiate this struct.
*/
struct io_tw_state { struct io_tw_state {
}; };
/* Alias to use in code that doesn't instantiate struct io_tw_state */
typedef struct io_tw_state *io_tw_token_t;
enum { enum {
REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT, REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
@@ -563,7 +570,7 @@ enum {
REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT), REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
}; };
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts); typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
struct io_task_work { struct io_task_work {
struct llist_node node; struct llist_node node;
io_uring/futex.c
+8 -8
View File
@@ -44,30 +44,30 @@ void io_futex_cache_free(struct io_ring_ctx *ctx)
io_alloc_cache_free(&ctx->futex_cache, kfree); io_alloc_cache_free(&ctx->futex_cache, kfree);
} }
static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
{ {
req->async_data = NULL; req->async_data = NULL;
hlist_del_init(&req->hash_node); hlist_del_init(&req->hash_node);
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
} }
static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_futex_data *ifd = req->async_data; struct io_futex_data *ifd = req->async_data;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
io_tw_lock(ctx, ts); io_tw_lock(ctx, tw);
if (!io_alloc_cache_put(&ctx->futex_cache, ifd)) if (!io_alloc_cache_put(&ctx->futex_cache, ifd))
kfree(ifd); kfree(ifd);
__io_futex_complete(req, ts); __io_futex_complete(req, tw);
} }
static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts) static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct futex_vector *futexv = req->async_data; struct futex_vector *futexv = req->async_data;
io_tw_lock(req->ctx, ts); io_tw_lock(req->ctx, tw);
if (!iof->futexv_unqueued) { if (!iof->futexv_unqueued) {
int res; int res;
@@ -79,7 +79,7 @@ static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
kfree(req->async_data); kfree(req->async_data);
req->flags &= ~REQ_F_ASYNC_DATA; req->flags &= ~REQ_F_ASYNC_DATA;
__io_futex_complete(req, ts); __io_futex_complete(req, tw);
} }
static bool io_futexv_claim(struct io_futex *iof) static bool io_futexv_claim(struct io_futex *iof)
io_uring/io_uring.c
+14 -14
View File
@@ -544,7 +544,7 @@ static void io_queue_iowq(struct io_kiocb *req)
io_queue_linked_timeout(link); io_queue_linked_timeout(link);
} }
static void io_req_queue_iowq_tw(struct io_kiocb *req, struct io_tw_state *ts) static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)
{ {
io_queue_iowq(req); io_queue_iowq(req);
} }
@@ -1031,7 +1031,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
return nxt; return nxt;
} }
static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts) static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw)
{ {
if (!ctx) if (!ctx)
return; return;
@@ -1286,7 +1286,7 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,
} }
static int __io_run_local_work_loop(struct llist_node **node, static int __io_run_local_work_loop(struct llist_node **node,
struct io_tw_state *ts, io_tw_token_t tw,
int events) int events)
{ {
int ret = 0; int ret = 0;
@@ -1297,7 +1297,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
io_task_work.node); io_task_work.node);
INDIRECT_CALL_2(req->io_task_work.func, INDIRECT_CALL_2(req->io_task_work.func,
io_poll_task_func, io_req_rw_complete, io_poll_task_func, io_req_rw_complete,
req, ts); req, tw);
*node = next; *node = next;
if (++ret >= events) if (++ret >= events)
break; break;
@@ -1306,7 +1306,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
return ret; return ret;
} }
static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts, static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
int min_events, int max_events) int min_events, int max_events)
{ {
struct llist_node *node; struct llist_node *node;
@@ -1319,7 +1319,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
again: again:
min_events -= ret; min_events -= ret;
ret = __io_run_local_work_loop(&ctx->retry_llist.first, ts, max_events); ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
if (ctx->retry_llist.first) if (ctx->retry_llist.first)
goto retry_done; goto retry_done;
@@ -1328,7 +1328,7 @@ again:
* running the pending items. * running the pending items.
*/ */
node = llist_reverse_order(llist_del_all(&ctx->work_llist)); node = llist_reverse_order(llist_del_all(&ctx->work_llist));
ret += __io_run_local_work_loop(&node, ts, max_events - ret); ret += __io_run_local_work_loop(&node, tw, max_events - ret);
ctx->retry_llist.first = node; ctx->retry_llist.first = node;
loops++; loops++;
@@ -1366,15 +1366,15 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events,
return ret; return ret;
} }
static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts) static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)
{ {
io_tw_lock(req->ctx, ts); io_tw_lock(req->ctx, tw);
io_req_defer_failed(req, req->cqe.res); io_req_defer_failed(req, req->cqe.res);
} }
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts) void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
{ {
io_tw_lock(req->ctx, ts); io_tw_lock(req->ctx, tw);
if (unlikely(io_should_terminate_tw())) if (unlikely(io_should_terminate_tw()))
io_req_defer_failed(req, -EFAULT); io_req_defer_failed(req, -EFAULT);
else if (req->flags & REQ_F_FORCE_ASYNC) else if (req->flags & REQ_F_FORCE_ASYNC)
@@ -1592,7 +1592,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
return 0; return 0;
} }
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts) void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)
{ {
io_req_complete_defer(req); io_req_complete_defer(req);
} }
@@ -1772,9 +1772,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
return ret; return ret;
} }
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts) int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
{ {
io_tw_lock(req->ctx, ts); io_tw_lock(req->ctx, tw);
return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT| return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|
IO_URING_F_COMPLETE_DEFER); IO_URING_F_COMPLETE_DEFER);
} }
io_uring/io_uring.h
+4 -4
View File
@@ -90,9 +90,9 @@ void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
unsigned flags); unsigned flags);
bool io_alloc_async_data(struct io_kiocb *req); bool io_alloc_async_data(struct io_kiocb *req);
void io_req_task_queue(struct io_kiocb *req); void io_req_task_queue(struct io_kiocb *req);
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts); void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);
void io_req_task_queue_fail(struct io_kiocb *req, int ret); void io_req_task_queue_fail(struct io_kiocb *req, int ret);
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts); void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);
struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries); struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);
struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count); struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);
void tctx_task_work(struct callback_head *cb); void tctx_task_work(struct callback_head *cb);
@@ -104,7 +104,7 @@ int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
int start, int end); int start, int end);
void io_req_queue_iowq(struct io_kiocb *req); void io_req_queue_iowq(struct io_kiocb *req);
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts); int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);
int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
void __io_submit_flush_completions(struct io_ring_ctx *ctx); void __io_submit_flush_completions(struct io_ring_ctx *ctx);
@@ -376,7 +376,7 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
return task_work_pending(current) || io_local_work_pending(ctx); return task_work_pending(current) || io_local_work_pending(ctx);
} }
static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts) static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw)
{ {
lockdep_assert_held(&ctx->uring_lock); lockdep_assert_held(&ctx->uring_lock);
} }
io_uring/msg_ring.c
+1 -1
View File
@@ -71,7 +71,7 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)
return target_ctx->task_complete; return target_ctx->task_complete;
} }
static void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts) static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
io_uring/notif.c
+2 -2
View File
@@ -11,7 +11,7 @@
static const struct ubuf_info_ops io_ubuf_ops; static const struct ubuf_info_ops io_ubuf_ops;
static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts) static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw)
{ {
struct io_notif_data *nd = io_notif_to_data(notif); struct io_notif_data *nd = io_notif_to_data(notif);
@@ -29,7 +29,7 @@ static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
} }
nd = nd->next; nd = nd->next;
io_req_task_complete(notif, ts); io_req_task_complete(notif, tw);
} while (nd); } while (nd);
} }
io_uring/poll.c
+9 -9
View File
@@ -220,7 +220,7 @@ static inline void io_poll_execute(struct io_kiocb *req, int res)
* req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot * req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
* poll and that the result is stored in req->cqe. * poll and that the result is stored in req->cqe.
*/ */
static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts) static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
{ {
int v; int v;
@@ -288,7 +288,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
return IOU_POLL_REMOVE_POLL_USE_RES; return IOU_POLL_REMOVE_POLL_USE_RES;
} }
} else { } else {
int ret = io_poll_issue(req, ts); int ret = io_poll_issue(req, tw);
if (ret == IOU_STOP_MULTISHOT) if (ret == IOU_STOP_MULTISHOT)
return IOU_POLL_REMOVE_POLL_USE_RES; return IOU_POLL_REMOVE_POLL_USE_RES;
else if (ret == IOU_REQUEUE) else if (ret == IOU_REQUEUE)
@@ -311,11 +311,11 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
return IOU_POLL_NO_ACTION; return IOU_POLL_NO_ACTION;
} }
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts) void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
{ {
int ret; int ret;
ret = io_poll_check_events(req, ts); ret = io_poll_check_events(req, tw);
if (ret == IOU_POLL_NO_ACTION) { if (ret == IOU_POLL_NO_ACTION) {
io_kbuf_recycle(req, 0); io_kbuf_recycle(req, 0);
return; return;
@@ -335,7 +335,7 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
poll = io_kiocb_to_cmd(req, struct io_poll); poll = io_kiocb_to_cmd(req, struct io_poll);
req->cqe.res = mangle_poll(req->cqe.res & poll->events); req->cqe.res = mangle_poll(req->cqe.res & poll->events);
} else if (ret == IOU_POLL_REISSUE) { } else if (ret == IOU_POLL_REISSUE) {
io_req_task_submit(req, ts); io_req_task_submit(req, tw);
return; return;
} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) { } else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
req->cqe.res = ret; req->cqe.res = ret;
@@ -343,14 +343,14 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
} }
io_req_set_res(req, req->cqe.res, 0); io_req_set_res(req, req->cqe.res, 0);
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
} else { } else {
io_tw_lock(req->ctx, ts); io_tw_lock(req->ctx, tw);
if (ret == IOU_POLL_REMOVE_POLL_USE_RES) if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE) else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE)
io_req_task_submit(req, ts); io_req_task_submit(req, tw);
else else
io_req_defer_failed(req, ret); io_req_defer_failed(req, ret);
} }
io_uring/poll.h
+3 -1
View File
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 // SPDX-License-Identifier: GPL-2.0
#include <linux/io_uring_types.h>
#define IO_POLL_ALLOC_CACHE_MAX 32 #define IO_POLL_ALLOC_CACHE_MAX 32
enum { enum {
@@ -43,4 +45,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags);
bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
bool cancel_all); bool cancel_all);
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts); void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw);
io_uring/rw.c
+2 -2
View File
@@ -519,7 +519,7 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
return res; return res;
} }
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts) void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
struct kiocb *kiocb = &rw->kiocb; struct kiocb *kiocb = &rw->kiocb;
@@ -536,7 +536,7 @@ void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0); req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0);
io_req_rw_cleanup(req, 0); io_req_rw_cleanup(req, 0);
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
} }
static void io_complete_rw(struct kiocb *kiocb, long res) static void io_complete_rw(struct kiocb *kiocb, long res)
io_uring/rw.h
+2 -1
View File
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 // SPDX-License-Identifier: GPL-2.0
#include <linux/io_uring_types.h>
#include <linux/pagemap.h> #include <linux/pagemap.h>
struct io_meta_state { struct io_meta_state {
@@ -39,7 +40,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags);
int io_write(struct io_kiocb *req, unsigned int issue_flags); int io_write(struct io_kiocb *req, unsigned int issue_flags);
void io_readv_writev_cleanup(struct io_kiocb *req); void io_readv_writev_cleanup(struct io_kiocb *req);
void io_rw_fail(struct io_kiocb *req); void io_rw_fail(struct io_kiocb *req);
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts); void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw);
int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags); int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);
void io_rw_cache_free(const void *entry); void io_rw_cache_free(const void *entry);
io_uring/timeout.c
+8 -8
View File
@@ -65,7 +65,7 @@ static inline bool io_timeout_finish(struct io_timeout *timeout,
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer); static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts) static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_timeout_data *data = req->async_data; struct io_timeout_data *data = req->async_data;
@@ -82,7 +82,7 @@ static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
} }
} }
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
} }
static __cold bool io_flush_killed_timeouts(struct list_head *list, int err) static __cold bool io_flush_killed_timeouts(struct list_head *list, int err)
@@ -154,9 +154,9 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
io_flush_killed_timeouts(&list, 0); io_flush_killed_timeouts(&list, 0);
} }
static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts) static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw)
{ {
io_tw_lock(link->ctx, ts); io_tw_lock(link->ctx, tw);
while (link) { while (link) {
struct io_kiocb *nxt = link->link; struct io_kiocb *nxt = link->link;
long res = -ECANCELED; long res = -ECANCELED;
@@ -165,7 +165,7 @@ static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
res = link->cqe.res; res = link->cqe.res;
link->link = NULL; link->link = NULL;
io_req_set_res(link, res, 0); io_req_set_res(link, res, 0);
io_req_task_complete(link, ts); io_req_task_complete(link, tw);
link = nxt; link = nxt;
} }
} }
@@ -312,7 +312,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
return 0; return 0;
} }
static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts) static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_kiocb *prev = timeout->prev; struct io_kiocb *prev = timeout->prev;
@@ -330,11 +330,11 @@ static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *t
ret = -ECANCELED; ret = -ECANCELED;
} }
io_req_set_res(req, ret ?: -ETIME, 0); io_req_set_res(req, ret ?: -ETIME, 0);
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
io_put_req(prev); io_put_req(prev);
} else { } else {
io_req_set_res(req, -ETIME, 0); io_req_set_res(req, -ETIME, 0);
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
} }
} }
io_uring/uring_cmd.c
+1 -1
View File
@@ -102,7 +102,7 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
} }
EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable); EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
static void io_uring_cmd_work(struct io_kiocb *req, struct io_tw_state *ts) static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
unsigned int flags = IO_URING_F_COMPLETE_DEFER; unsigned int flags = IO_URING_F_COMPLETE_DEFER;
io_uring/waitid.c
+4 -4
View File
@@ -16,7 +16,7 @@
#include "waitid.h" #include "waitid.h"
#include "../kernel/exit.h" #include "../kernel/exit.h"
static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts); static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw);
#define IO_WAITID_CANCEL_FLAG BIT(31) #define IO_WAITID_CANCEL_FLAG BIT(31)
#define IO_WAITID_REF_MASK GENMASK(30, 0) #define IO_WAITID_REF_MASK GENMASK(30, 0)
@@ -185,13 +185,13 @@ static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
return true; return true;
} }
static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts) static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw)
{ {
struct io_waitid_async *iwa = req->async_data; struct io_waitid_async *iwa = req->async_data;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
int ret; int ret;
io_tw_lock(ctx, ts); io_tw_lock(ctx, tw);
ret = __do_wait(&iwa->wo); ret = __do_wait(&iwa->wo);
@@ -221,7 +221,7 @@ static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)
} }
io_waitid_complete(req, ret); io_waitid_complete(req, ret);
io_req_task_complete(req, ts); io_req_task_complete(req, tw);
} }
static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,