qiurui/Centos-kernel-stream-9
1
0
Fork 0
mirror of https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9.git synced 2026-09-09 00:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

io_uring/sqpoll: be smarter on when to update the stime usage

Browse Source 
JIRA: https://issues.redhat.com/browse/RHEL-141266

commit a94e0657269c5b8e1a90b17aa2c048b3d276e16d
Author: Jens Axboe <axboe@kernel.dk>
Date:   Tue Oct 21 11:44:39 2025 -0600

    io_uring/sqpoll: be smarter on when to update the stime usage
    
    The current approach is a bit naive, and hence calls the time querying
    way too often. Only start the "doing work" timer when there's actual
    work to do, and then use that information to terminate (and account) the
    work time once done. This greatly reduces the frequency of these calls,
    when they cannot have changed anyway.
    
    Running a basic random reader that is setup to use SQPOLL, a profile
    before this change shows these as the top cycle consumers:
    
    +   32.60%  iou-sqp-1074  [kernel.kallsyms]  [k] thread_group_cputime_adjusted
    +   19.97%  iou-sqp-1074  [kernel.kallsyms]  [k] thread_group_cputime
    +   12.20%  io_uring      io_uring           [.] submitter_uring_fn
    +    4.13%  iou-sqp-1074  [kernel.kallsyms]  [k] getrusage
    +    2.45%  iou-sqp-1074  [kernel.kallsyms]  [k] io_submit_sqes
    +    2.18%  iou-sqp-1074  [kernel.kallsyms]  [k] __pi_memset_generic
    +    2.09%  iou-sqp-1074  [kernel.kallsyms]  [k] cputime_adjust
    
    and after this change, top of profile looks as follows:
    
    +   36.23%  io_uring     io_uring           [.] submitter_uring_fn
    +   23.26%  iou-sqp-819  [kernel.kallsyms]  [k] io_sq_thread
    +   10.14%  iou-sqp-819  [kernel.kallsyms]  [k] io_sq_tw
    +    6.52%  iou-sqp-819  [kernel.kallsyms]  [k] tctx_task_work_run
    +    4.82%  iou-sqp-819  [kernel.kallsyms]  [k] nvme_submit_cmds.part.0
    +    2.91%  iou-sqp-819  [kernel.kallsyms]  [k] io_submit_sqes
    [...]
         0.02%  iou-sqp-819  [kernel.kallsyms]  [k] cputime_adjust
    
    where it's spending the cycles on things that actually matter.
    
    Reported-by: Fengnan Chang <changfengnan@bytedance.com>
    Cc: stable@vger.kernel.org
    Fixes: 3fcb9d17206e ("io_uring/sqpoll: statistics of the true utilization of sq threads")
    Signed-off-by: Jens Axboe <axboe@kernel.dk>

Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
This commit is contained in:
Jeff Moyer
2026-01-14 12:00:17 -05:00
parent 57a02079d2
commit b3f121e275
1 changed files with 32 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
io_uring/sqpoll.c
+32 -11
View File
@@ -177,6 +177,11 @@ static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
return READ_ONCE(sqd->state);
}
struct io_sq_time {
bool started;
u64 usec;
};
u64 io_sq_cpu_usec(struct task_struct *tsk)
{
u64 utime, stime;
@@ -186,12 +191,24 @@ u64 io_sq_cpu_usec(struct task_struct *tsk)
return stime;
}
static void io_sq_update_worktime(struct io_sq_data *sqd, u64 usec)
static void io_sq_update_worktime(struct io_sq_data *sqd, struct io_sq_time *ist)
{
sqd->work_time += io_sq_cpu_usec(current) - usec;
if (!ist->started)
return;
ist->started = false;
sqd->work_time += io_sq_cpu_usec(current) - ist->usec;
}
static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
static void io_sq_start_worktime(struct io_sq_time *ist)
{
if (ist->started)
return;
ist->started = true;
ist->usec = io_sq_cpu_usec(current);
}
static int __io_sq_thread(struct io_ring_ctx *ctx, struct io_sq_data *sqd,
bool cap_entries, struct io_sq_time *ist)
{
unsigned int to_submit;
int ret = 0;
@@ -204,6 +221,8 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
if (to_submit || !wq_list_empty(&ctx->iopoll_list)) {
const struct cred *creds = NULL;
io_sq_start_worktime(ist);
if (ctx->sq_creds != current_cred())
creds = override_creds(ctx->sq_creds);
@@ -285,7 +304,6 @@ static int io_sq_thread(void *data)
unsigned long timeout = 0;
char buf[TASK_COMM_LEN];
DEFINE_WAIT(wait);
u64 start;
/* offload context creation failed, just exit */
if (!current->io_uring) {
@@ -320,6 +338,7 @@ static int io_sq_thread(void *data)
mutex_lock(&sqd->lock);
while (1) {
bool cap_entries, sqt_spin = false;
struct io_sq_time ist = { };
if (io_sqd_events_pending(sqd) || signal_pending(current)) {
if (io_sqd_handle_event(sqd))
@@ -328,9 +347,8 @@ static int io_sq_thread(void *data)
}
cap_entries = !list_is_singular(&sqd->ctx_list);
start = io_sq_cpu_usec(current);
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
int ret = __io_sq_thread(ctx, cap_entries);
int ret = __io_sq_thread(ctx, sqd, cap_entries, &ist);
if (!sqt_spin && (ret > 0 || !wq_list_empty(&ctx->iopoll_list)))
sqt_spin = true;
@@ -338,15 +356,18 @@ static int io_sq_thread(void *data)
if (io_sq_tw(&retry_list, IORING_TW_CAP_ENTRIES_VALUE))
sqt_spin = true;
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
if (io_napi(ctx))
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
if (io_napi(ctx)) {
io_sq_start_worktime(&ist);
io_napi_sqpoll_busy_poll(ctx);
}
}
io_sq_update_worktime(sqd, &ist);
if (sqt_spin || !time_after(jiffies, timeout)) {
if (sqt_spin) {
io_sq_update_worktime(sqd, start);
if (sqt_spin)
timeout = jiffies + sqd->sq_thread_idle;
}
if (unlikely(need_resched())) {
mutex_unlock(&sqd->lock);
cond_resched();