qiurui
/
Centos-kernel-stream-9
mirror of https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

sched: Introduce affinity_context 

Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2107354
Upstream Status: tip commit 713a2e21a5137e96d2594f53d19784ffde3ddbd0

commit 713a2e21a5137e96d2594f53d19784ffde3ddbd0
Author: Waiman Long <longman@redhat.com>
Date:   Thu, 22 Sep 2022 14:00:40 -0400

    sched: Introduce affinity_context

    In order to prepare for passing through additional data through the
    affinity call-chains, convert the mask and flags argument into a
    structure.

    Suggested-by: Peter Zijlstra <peterz@infradead.org>
    Signed-off-by: Waiman Long <longman@redhat.com>
    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Link: https://lkml.kernel.org/r/20220922180041.1768141-5-longman@redhat.com

Signed-off-by: Waiman Long <longman@redhat.com>
This commit is contained in:
Waiman Long 2022-10-27 19:44:41 -04:00
parent 31d9c33c0e
commit 8a370625e9
3 changed files with 85 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
kernel/sched/core.c
View File

@ -2207,14 +2207,18 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static void static void
__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags); __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx);
static int __set_cpus_allowed_ptr(struct task_struct *p, static int __set_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *new_mask, struct affinity_context *ctx);
u32 flags);
static void migrate_disable_switch(struct rq *rq, struct task_struct *p) static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
{ {
struct affinity_context ac = {
.new_mask = cpumask_of(rq->cpu),
.flags = SCA_MIGRATE_DISABLE,
};
if (likely(!p->migration_disabled)) if (likely(!p->migration_disabled))
return; return;
@ -2224,7 +2228,7 @@ static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
/* /*
* Violates locking rules! see comment in __do_set_cpus_allowed(). * Violates locking rules! see comment in __do_set_cpus_allowed().
*/ */
__do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE); __do_set_cpus_allowed(p, &ac);
} }
void migrate_disable(void) void migrate_disable(void)
@ -2246,6 +2250,10 @@ EXPORT_SYMBOL_GPL(migrate_disable);
void migrate_enable(void) void migrate_enable(void)
{ {
struct task_struct *p = current; struct task_struct *p = current;
struct affinity_context ac = {
.new_mask = &p->cpus_mask,
.flags = SCA_MIGRATE_ENABLE,
};
if (p->migration_disabled > 1) { if (p->migration_disabled > 1) {
p->migration_disabled--; p->migration_disabled--;
@ -2261,7 +2269,7 @@ void migrate_enable(void)
*/ */
preempt_disable(); preempt_disable();
if (p->cpus_ptr != &p->cpus_mask) if (p->cpus_ptr != &p->cpus_mask)
__set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE); __set_cpus_allowed_ptr(p, &ac);
/* /*
* Mustn't clear migration_disabled() until cpus_ptr points back at the * Mustn't clear migration_disabled() until cpus_ptr points back at the
* regular cpus_mask, otherwise things that race (eg. * regular cpus_mask, otherwise things that race (eg.
@ -2541,19 +2549,19 @@ out_unlock:
* sched_class::set_cpus_allowed must do the below, but is not required to * sched_class::set_cpus_allowed must do the below, but is not required to
* actually call this function. * actually call this function.
*/ */
void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags) void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx)
{ {
if (flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
p->cpus_ptr = new_mask; p->cpus_ptr = ctx->new_mask;
return; return;
} }
cpumask_copy(&p->cpus_mask, new_mask); cpumask_copy(&p->cpus_mask, ctx->new_mask);
p->nr_cpus_allowed = cpumask_weight(new_mask); p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
} }
static void static void
__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags) __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
{ {
struct rq *rq = task_rq(p); struct rq *rq = task_rq(p);
bool queued, running; bool queued, running;
@ -2570,7 +2578,7 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
* *
* XXX do further audits, this smells like something putrid. * XXX do further audits, this smells like something putrid.
*/ */
if (flags & SCA_MIGRATE_DISABLE) if (ctx->flags & SCA_MIGRATE_DISABLE)
SCHED_WARN_ON(!p->on_cpu); SCHED_WARN_ON(!p->on_cpu);
else else
lockdep_assert_held(&p->pi_lock); lockdep_assert_held(&p->pi_lock);
@ -2589,7 +2597,7 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
if (running) if (running)
put_prev_task(rq, p); put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, new_mask, flags); p->sched_class->set_cpus_allowed(p, ctx);
if (queued) if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@ -2599,7 +2607,12 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{ {
__do_set_cpus_allowed(p, new_mask, 0); struct affinity_context ac = {
.new_mask = new_mask,
.flags = 0,
};
__do_set_cpus_allowed(p, &ac);
} }
int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src,
@ -2852,8 +2865,7 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag
* Called with both p->pi_lock and rq->lock held; drops both before returning. * Called with both p->pi_lock and rq->lock held; drops both before returning.
*/ */
static int __set_cpus_allowed_ptr_locked(struct task_struct *p, static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
const struct cpumask *new_mask, struct affinity_context *ctx,
u32 flags,
struct rq *rq, struct rq *rq,
struct rq_flags *rf) struct rq_flags *rf)
__releases(rq->lock) __releases(rq->lock)
@ -2882,7 +2894,7 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
cpu_valid_mask = cpu_online_mask; cpu_valid_mask = cpu_online_mask;
} }
if (!kthread && !cpumask_subset(new_mask, cpu_allowed_mask)) { if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
} }
@ -2891,18 +2903,18 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* Must re-check here, to close a race against __kthread_bind(), * Must re-check here, to close a race against __kthread_bind(),
* sched_setaffinity() is not guaranteed to observe the flag. * sched_setaffinity() is not guaranteed to observe the flag.
*/ */
if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
} }
if (!(flags & SCA_MIGRATE_ENABLE)) { if (!(ctx->flags & SCA_MIGRATE_ENABLE)) {
if (cpumask_equal(&p->cpus_mask, new_mask)) if (cpumask_equal(&p->cpus_mask, ctx->new_mask))
goto out; goto out;
if (WARN_ON_ONCE(p == current && if (WARN_ON_ONCE(p == current &&
is_migration_disabled(p) && is_migration_disabled(p) &&
!cpumask_test_cpu(task_cpu(p), new_mask))) { !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) {
ret = -EBUSY; ret = -EBUSY;
goto out; goto out;
} }
@ -2913,18 +2925,18 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* for groups of tasks (ie. cpuset), so that load balancing is not * for groups of tasks (ie. cpuset), so that load balancing is not
* immediately required to distribute the tasks within their new mask. * immediately required to distribute the tasks within their new mask.
*/ */
dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask); dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask);
if (dest_cpu >= nr_cpu_ids) { if (dest_cpu >= nr_cpu_ids) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
} }
__do_set_cpus_allowed(p, new_mask, flags); __do_set_cpus_allowed(p, ctx);
if (flags & SCA_USER) if (ctx->flags & SCA_USER)
user_mask = clear_user_cpus_ptr(p); user_mask = clear_user_cpus_ptr(p);
ret = affine_move_task(rq, p, rf, dest_cpu, flags); ret = affine_move_task(rq, p, rf, dest_cpu, ctx->flags);
kfree(user_mask); kfree(user_mask);
@ -2946,18 +2958,23 @@ out:
* call is not atomic; no spinlocks may be held. * call is not atomic; no spinlocks may be held.
*/ */
static int __set_cpus_allowed_ptr(struct task_struct *p, static int __set_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *new_mask, u32 flags) struct affinity_context *ctx)
{ {
struct rq_flags rf; struct rq_flags rf;
struct rq *rq; struct rq *rq;
rq = task_rq_lock(p, &rf); rq = task_rq_lock(p, &rf);
return __set_cpus_allowed_ptr_locked(p, new_mask, flags, rq, &rf); return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf);
} }
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
{ {
return __set_cpus_allowed_ptr(p, new_mask, 0); struct affinity_context ac = {
.new_mask = new_mask,
.flags = 0,
};
return __set_cpus_allowed_ptr(p, &ac);
} }
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
@ -2973,6 +2990,7 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *subset_mask) const struct cpumask *subset_mask)
{ {
struct cpumask *user_mask = NULL; struct cpumask *user_mask = NULL;
struct affinity_context ac;
struct rq_flags rf; struct rq_flags rf;
struct rq *rq; struct rq *rq;
int err; int err;
@ -3009,7 +3027,11 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p,
p->user_cpus_ptr = user_mask; p->user_cpus_ptr = user_mask;
} }
return __set_cpus_allowed_ptr_locked(p, new_mask, 0, rq, &rf); ac = (struct affinity_context){
.new_mask = new_mask,
};
return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf);
err_unlock: err_unlock:
task_rq_unlock(rq, p, &rf); task_rq_unlock(rq, p, &rf);
@ -3063,7 +3085,7 @@ out_free_mask:
} }
static int static int
__sched_setaffinity(struct task_struct *p, const struct cpumask *mask); __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx);
/* /*
* Restore the affinity of a task @p which was previously restricted by a * Restore the affinity of a task @p which was previously restricted by a
@ -3076,6 +3098,9 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask);
void relax_compatible_cpus_allowed_ptr(struct task_struct *p) void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
{ {
struct cpumask *user_mask = p->user_cpus_ptr; struct cpumask *user_mask = p->user_cpus_ptr;
struct affinity_context ac = {
.new_mask = user_mask,
};
unsigned long flags; unsigned long flags;
/* /*
@ -3083,7 +3108,7 @@ void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
* we free the mask explicitly to avoid it being inherited across * we free the mask explicitly to avoid it being inherited across
* a subsequent fork(). * a subsequent fork().
*/ */
if (!user_mask || !__sched_setaffinity(p, user_mask)) if (!user_mask || !__sched_setaffinity(p, &ac))
return; return;
raw_spin_lock_irqsave(&p->pi_lock, flags); raw_spin_lock_irqsave(&p->pi_lock, flags);
@ -3568,10 +3593,9 @@ void sched_set_stop_task(int cpu, struct task_struct *stop)
#else /* CONFIG_SMP */ #else /* CONFIG_SMP */
static inline int __set_cpus_allowed_ptr(struct task_struct *p, static inline int __set_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *new_mask, struct affinity_context *ctx)
u32 flags)
{ {
return set_cpus_allowed_ptr(p, new_mask); return set_cpus_allowed_ptr(p, ctx->new_mask);
} }
static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { }
@ -7975,7 +7999,7 @@ int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
#endif #endif
static int static int
__sched_setaffinity(struct task_struct *p, const struct cpumask *mask) __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx)
{ {
int retval; int retval;
cpumask_var_t cpus_allowed, new_mask; cpumask_var_t cpus_allowed, new_mask;
@ -7989,13 +8013,16 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask)
} }
cpuset_cpus_allowed(p, cpus_allowed); cpuset_cpus_allowed(p, cpus_allowed);
cpumask_and(new_mask, mask, cpus_allowed); cpumask_and(new_mask, ctx->new_mask, cpus_allowed);
ctx->new_mask = new_mask;
ctx->flags |= SCA_CHECK;
retval = dl_task_check_affinity(p, new_mask); retval = dl_task_check_affinity(p, new_mask);
if (retval) if (retval)
goto out_free_new_mask; goto out_free_new_mask;
again: again:
retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK | SCA_USER); retval = __set_cpus_allowed_ptr(p, ctx);
if (retval) if (retval)
goto out_free_new_mask; goto out_free_new_mask;
@ -8018,6 +8045,9 @@ out_free_cpus_allowed:
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
{ {
struct affinity_context ac = {
.new_mask = in_mask,
};
struct task_struct *p; struct task_struct *p;
int retval; int retval;
@ -8052,7 +8082,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
if (retval) if (retval)
goto out_put_task; goto out_put_task;
retval = __sched_setaffinity(p, in_mask); retval = __sched_setaffinity(p, &ac);
out_put_task: out_put_task:
put_task_struct(p); put_task_struct(p);
return retval; return retval;
@ -8833,6 +8863,12 @@ void show_state_filter(unsigned int state_filter)
*/ */
void __init init_idle(struct task_struct *idle, int cpu) void __init init_idle(struct task_struct *idle, int cpu)
{ {
#ifdef CONFIG_SMP
struct affinity_context ac = (struct affinity_context) {
.new_mask = cpumask_of(cpu),
.flags = 0,
};
#endif
struct rq *rq = cpu_rq(cpu); struct rq *rq = cpu_rq(cpu);
unsigned long flags; unsigned long flags;
@ -8857,7 +8893,7 @@ void __init init_idle(struct task_struct *idle, int cpu)
* *
* And since this is boot we can forgo the serialization. * And since this is boot we can forgo the serialization.
*/ */
set_cpus_allowed_common(idle, cpumask_of(cpu), 0); set_cpus_allowed_common(idle, &ac);
#endif #endif
/* /*
* We're having a chicken and egg problem, even though we are * We're having a chicken and egg problem, even though we are

7
kernel/sched/deadline.c
View File

@ -2452,8 +2452,7 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p)
} }
static void set_cpus_allowed_dl(struct task_struct *p, static void set_cpus_allowed_dl(struct task_struct *p,
const struct cpumask *new_mask, struct affinity_context *ctx)
u32 flags)
{ {
struct root_domain *src_rd; struct root_domain *src_rd;
struct rq *rq; struct rq *rq;
@ -2468,7 +2467,7 @@ static void set_cpus_allowed_dl(struct task_struct *p,
* update. We already made space for us in the destination * update. We already made space for us in the destination
* domain (see cpuset_can_attach()). * domain (see cpuset_can_attach()).
*/ */
if (!cpumask_intersects(src_rd->span, new_mask)) { if (!cpumask_intersects(src_rd->span, ctx->new_mask)) {
struct dl_bw *src_dl_b; struct dl_bw *src_dl_b;
src_dl_b = dl_bw_of(cpu_of(rq)); src_dl_b = dl_bw_of(cpu_of(rq));
@ -2482,7 +2481,7 @@ static void set_cpus_allowed_dl(struct task_struct *p,
raw_spin_unlock(&src_dl_b->lock); raw_spin_unlock(&src_dl_b->lock);
} }
set_cpus_allowed_common(p, new_mask, flags); set_cpus_allowed_common(p, ctx);
} }
/* Assumes rq->lock is held */ /* Assumes rq->lock is held */

11
kernel/sched/sched.h
View File

@ -2137,6 +2137,11 @@ extern const u32 sched_prio_to_wmult[40];
#define RETRY_TASK ((void *)-1UL) #define RETRY_TASK ((void *)-1UL)
struct affinity_context {
const struct cpumask *new_mask;
unsigned int flags;
};
struct sched_class { struct sched_class {
#ifdef CONFIG_UCLAMP_TASK #ifdef CONFIG_UCLAMP_TASK
@ -2165,9 +2170,7 @@ struct sched_class {
void (*task_woken)(struct rq *this_rq, struct task_struct *task); void (*task_woken)(struct rq *this_rq, struct task_struct *task);
void (*set_cpus_allowed)(struct task_struct *p, void (*set_cpus_allowed)(struct task_struct *p, struct affinity_context *ctx);
const struct cpumask *newmask,
u32 flags);
void (*rq_online)(struct rq *rq); void (*rq_online)(struct rq *rq);
void (*rq_offline)(struct rq *rq); void (*rq_offline)(struct rq *rq);
@ -2280,7 +2283,7 @@ extern void update_group_capacity(struct sched_domain *sd, int cpu);
extern void trigger_load_balance(struct rq *rq); extern void trigger_load_balance(struct rq *rq);
extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags); extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx);
static inline struct task_struct *get_push_task(struct rq *rq) static inline struct task_struct *get_push_task(struct rq *rq)
{ {