qiurui/Centos-kernel-stream-9
1
0
Fork 0
mirror of https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9.git synced 2026-09-09 00:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge: block: fix one lockdep warning among switching elevator and removing disk

Browse Source 
MR: https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9/-/merge_requests/8010

block: fix one lockdep warning among switching elevator and removing disk

JIRA: https://issues.redhat.com/browse/RHEL-165529

Signed-off-by: Ming Lei <ming.lei@redhat.com>

Approved-by: Jeff Moyer <jmoyer@redhat.com>
Approved-by: Jay Shin <jaeshin@redhat.com>
Approved-by: CKI KWF Bot <cki-ci-bot+kwf-gitlab-com@redhat.com>

Merged-by: CKI GitLab Kmaint Pipeline Bot <26919896-cki-kmaint-pipeline-bot@users.noreply.gitlab.com>
This commit is contained in:
CKI KWF Bot
2026-07-02 08:10:30 +00:00
parent 400173f195 684ce75861
commit a87317c4a4
10 changed files with 375 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
block/blk-mq-sched.c
+99 -30
View File
@@ -427,11 +427,25 @@ void blk_mq_free_sched_tags(struct elevator_tags *et,
kfree(et);
}
void blk_mq_free_sched_tags_batch(struct xarray *et_table,
void blk_mq_free_sched_res(struct elevator_resources *res,
struct elevator_type *type,
struct blk_mq_tag_set *set)
{
if (res->et) {
blk_mq_free_sched_tags(res->et, set);
res->et = NULL;
}
if (res->data) {
blk_mq_free_sched_data(type, res->data);
res->data = NULL;
}
}
void blk_mq_free_sched_res_batch(struct xarray *elv_tbl,
struct blk_mq_tag_set *set)
{
struct request_queue *q;
struct elevator_tags *et;
struct elv_change_ctx *ctx;
lockdep_assert_held_write(&set->update_nr_hwq_lock);
@@ -444,17 +458,50 @@ void blk_mq_free_sched_tags_batch(struct xarray *et_table,
* concurrently.
*/
if (q->elevator) {
et = xa_load(et_table, q->id);
if (unlikely(!et))
ctx = xa_load(elv_tbl, q->id);
if (!ctx) {
WARN_ON_ONCE(1);
else
blk_mq_free_sched_tags(et, set);
continue;
}
blk_mq_free_sched_res(&ctx->res, ctx->type, set);
}
}
}
void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl)
{
unsigned long i;
struct elv_change_ctx *ctx;
xa_for_each(elv_tbl, i, ctx) {
xa_erase(elv_tbl, i);
kfree(ctx);
}
}
int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl,
struct blk_mq_tag_set *set)
{
struct request_queue *q;
struct elv_change_ctx *ctx;
lockdep_assert_held_write(&set->update_nr_hwq_lock);
list_for_each_entry(q, &set->tag_list, tag_set_list) {
ctx = kzalloc(sizeof(struct elv_change_ctx), GFP_KERNEL);
if (!ctx)
return -ENOMEM;
if (xa_insert(elv_tbl, q->id, ctx, GFP_KERNEL)) {
kfree(ctx);
return -ENOMEM;
}
}
return 0;
}
struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
unsigned int nr_hw_queues)
unsigned int nr_hw_queues, unsigned int nr_requests)
{
unsigned int nr_tags;
int i;
@@ -470,13 +517,8 @@ struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
nr_tags * sizeof(struct blk_mq_tags *), gfp);
if (!et)
return NULL;
/*
* Default to double of smaller one between hw queue_depth and
* 128, since we don't split into sync/async like the old code
* did. Additionally, this is a per-hw queue depth.
*/
et->nr_requests = 2 * min_t(unsigned int, set->queue_depth,
BLKDEV_DEFAULT_RQ);
et->nr_requests = nr_requests;
et->nr_hw_queues = nr_hw_queues;
if (blk_mq_is_shared_tags(set->flags)) {
@@ -503,12 +545,33 @@ out:
return NULL;
}
int blk_mq_alloc_sched_tags_batch(struct xarray *et_table,
int blk_mq_alloc_sched_res(struct request_queue *q,
struct elevator_type *type,
struct elevator_resources *res,
unsigned int nr_hw_queues)
{
struct blk_mq_tag_set *set = q->tag_set;
res->et = blk_mq_alloc_sched_tags(set, nr_hw_queues,
blk_mq_default_nr_requests(set));
if (!res->et)
return -ENOMEM;
res->data = blk_mq_alloc_sched_data(q, type);
if (IS_ERR(res->data)) {
blk_mq_free_sched_tags(res->et, set);
return -ENOMEM;
}
return 0;
}
int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
struct blk_mq_tag_set *set, unsigned int nr_hw_queues)
{
struct elv_change_ctx *ctx;
struct request_queue *q;
struct elevator_tags *et;
gfp_t gfp = GFP_NOIO | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
int ret = -ENOMEM;
lockdep_assert_held_write(&set->update_nr_hwq_lock);
@@ -521,38 +584,44 @@ int blk_mq_alloc_sched_tags_batch(struct xarray *et_table,
* concurrently.
*/
if (q->elevator) {
et = blk_mq_alloc_sched_tags(set, nr_hw_queues);
if (!et)
ctx = xa_load(elv_tbl, q->id);
if (WARN_ON_ONCE(!ctx)) {
ret = -ENOENT;
goto out_unwind;
}
ret = blk_mq_alloc_sched_res(q, q->elevator->type,
&ctx->res, nr_hw_queues);
if (ret)
goto out_unwind;
if (xa_insert(et_table, q->id, et, gfp))
goto out_free_tags;
}
}
return 0;
out_free_tags:
blk_mq_free_sched_tags(et, set);
out_unwind:
list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) {
if (q->elevator) {
et = xa_load(et_table, q->id);
if (et)
blk_mq_free_sched_tags(et, set);
ctx = xa_load(elv_tbl, q->id);
if (ctx)
blk_mq_free_sched_res(&ctx->res,
ctx->type, set);
}
}
return -ENOMEM;
return ret;
}
/* caller must have a reference to @e, will grab another one if successful */
int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e,
struct elevator_tags *et)
struct elevator_resources *res)
{
unsigned int flags = q->tag_set->flags;
struct elevator_tags *et = res->et;
struct blk_mq_hw_ctx *hctx;
struct elevator_queue *eq;
unsigned long i;
int ret;
eq = elevator_alloc(q, e, et);
eq = elevator_alloc(q, e, res);
if (!eq)
return -ENOMEM;
@@ -561,7 +630,7 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e,
if (blk_mq_is_shared_tags(flags)) {
/* Shared tags are stored at index 0 in @et->tags. */
q->sched_shared_tags = et->tags[0];
blk_mq_tag_update_sched_shared_tags(q);
blk_mq_tag_update_sched_shared_tags(q, et->nr_requests);
}
queue_for_each_hw_ctx(q, hctx, i) {
block/blk-mq-sched.h
+37 -3
View File
@@ -19,18 +19,52 @@ void __blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx);
void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e,
struct elevator_tags *et);
struct elevator_resources *res);
void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e);
void blk_mq_sched_free_rqs(struct request_queue *q);
struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
unsigned int nr_hw_queues, unsigned int nr_requests);
int blk_mq_alloc_sched_res(struct request_queue *q,
struct elevator_type *type,
struct elevator_resources *res,
unsigned int nr_hw_queues);
int blk_mq_alloc_sched_tags_batch(struct xarray *et_table,
int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
struct blk_mq_tag_set *set, unsigned int nr_hw_queues);
int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl,
struct blk_mq_tag_set *set);
void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl);
void blk_mq_free_sched_tags(struct elevator_tags *et,
struct blk_mq_tag_set *set);
void blk_mq_free_sched_tags_batch(struct xarray *et_table,
void blk_mq_free_sched_res(struct elevator_resources *res,
struct elevator_type *type,
struct blk_mq_tag_set *set);
void blk_mq_free_sched_res_batch(struct xarray *et_table,
struct blk_mq_tag_set *set);
/*
* blk_mq_alloc_sched_data() - Allocates scheduler specific data
* Returns:
* - Pointer to allocated data on success
* - NULL if no allocation needed
* - ERR_PTR(-ENOMEM) in case of failure
*/
static inline void *blk_mq_alloc_sched_data(struct request_queue *q,
struct elevator_type *e)
{
void *sched_data;
if (!e || !e->ops.alloc_sched_data)
return NULL;
sched_data = e->ops.alloc_sched_data(q);
return (sched_data) ?: ERR_PTR(-ENOMEM);
}
static inline void blk_mq_free_sched_data(struct elevator_type *e, void *data)
{
if (e && e->ops.free_sched_data)
e->ops.free_sched_data(data);
}
static inline void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx)
{
block/blk-mq-tag.c
+3 -55
View File
@@ -618,59 +618,6 @@ void blk_mq_free_tags(struct blk_mq_tag_set *set, struct blk_mq_tags *tags)
call_srcu(&set->tags_srcu, &tags->rcu_head, blk_mq_free_tags_callback);
}
int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
struct blk_mq_tags **tagsptr, unsigned int tdepth,
bool can_grow)
{
struct blk_mq_tags *tags = *tagsptr;
if (tdepth <= tags->nr_reserved_tags)
return -EINVAL;
/*
* If we are allowed to grow beyond the original size, allocate
* a new set of tags before freeing the old one.
*/
if (tdepth > tags->nr_tags) {
struct blk_mq_tag_set *set = hctx->queue->tag_set;
struct blk_mq_tags *new;
if (!can_grow)
return -EINVAL;
/*
* We need some sort of upper limit, set it high enough that
* no valid use cases should require more.
*/
if (tdepth > MAX_SCHED_RQ)
return -EINVAL;
/*
* Only the sbitmap needs resizing since we allocated the max
* initially.
*/
if (blk_mq_is_shared_tags(set->flags))
return 0;
new = blk_mq_alloc_map_and_rqs(set, hctx->queue_num, tdepth);
if (!new)
return -ENOMEM;
blk_mq_free_map_and_rqs(set, *tagsptr, hctx->queue_num);
hctx->queue->elevator->et->tags[hctx->queue_num] = new;
*tagsptr = new;
} else {
/*
* Don't need (or can't) update reserved tags here, they
* remain static and should never need resizing.
*/
sbitmap_queue_resize(&tags->bitmap_tags,
tdepth - tags->nr_reserved_tags);
}
return 0;
}
void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size)
{
struct blk_mq_tags *tags = set->shared_tags;
@@ -678,10 +625,11 @@ void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size
sbitmap_queue_resize(&tags->bitmap_tags, size - set->reserved_tags);
}
void blk_mq_tag_update_sched_shared_tags(struct request_queue *q)
void blk_mq_tag_update_sched_shared_tags(struct request_queue *q,
unsigned int nr)
{
sbitmap_queue_resize(&q->sched_shared_tags->bitmap_tags,
q->nr_requests - q->tag_set->reserved_tags);
nr - q->tag_set->reserved_tags);
}
/**
block/blk-mq.c
+65 -56
View File
@@ -4937,85 +4937,87 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
}
EXPORT_SYMBOL(blk_mq_free_tag_set);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q,
struct elevator_tags *et,
unsigned int nr)
{
struct blk_mq_tag_set *set = q->tag_set;
struct elevator_tags *old_et = NULL;
struct blk_mq_hw_ctx *hctx;
int ret;
unsigned long i;
if (WARN_ON_ONCE(!q->mq_freeze_depth))
return -EINVAL;
if (!set)
return -EINVAL;
if (q->nr_requests == nr)
return 0;
blk_mq_quiesce_queue(q);
ret = 0;
queue_for_each_hw_ctx(q, hctx, i) {
if (!hctx->tags)
continue;
if (blk_mq_is_shared_tags(set->flags)) {
/*
* If we're using an MQ scheduler, just update the scheduler
* queue depth. This is similar to what the old code would do.
* Shared tags, for sched tags, we allocate max initially hence
* tags can't grow, see blk_mq_alloc_sched_tags().
*/
if (hctx->sched_tags) {
ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
nr, true);
} else {
ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
false);
if (q->elevator)
blk_mq_tag_update_sched_shared_tags(q, nr);
else
blk_mq_tag_resize_shared_tags(set, nr);
} else if (!q->elevator) {
/*
* Non-shared hardware tags, nr is already checked from
* queue_requests_store() and tags can't grow.
*/
queue_for_each_hw_ctx(q, hctx, i) {
if (!hctx->tags)
continue;
sbitmap_queue_resize(&hctx->tags->bitmap_tags,
nr - hctx->tags->nr_reserved_tags);
}
if (ret)
goto out;
} else if (nr <= q->elevator->et->nr_requests) {
/* Non-shared sched tags, and tags don't grow. */
queue_for_each_hw_ctx(q, hctx, i) {
if (!hctx->sched_tags)
continue;
sbitmap_queue_resize(&hctx->sched_tags->bitmap_tags,
nr - hctx->sched_tags->nr_reserved_tags);
}
} else {
/* Non-shared sched tags, and tags grow */
queue_for_each_hw_ctx(q, hctx, i)
hctx->sched_tags = et->tags[i];
old_et = q->elevator->et;
q->elevator->et = et;
}
q->nr_requests = nr;
if (q->elevator && q->elevator->type->ops.depth_updated)
q->elevator->type->ops.depth_updated(q);
if (blk_mq_is_shared_tags(set->flags)) {
if (q->elevator)
blk_mq_tag_update_sched_shared_tags(q);
else
blk_mq_tag_resize_shared_tags(set, nr);
}
out:
blk_mq_unquiesce_queue(q);
return ret;
return old_et;
}
/*
* Switch back to the elevator type stored in the xarray.
*/
static void blk_mq_elv_switch_back(struct request_queue *q,
struct xarray *elv_tbl, struct xarray *et_tbl)
struct xarray *elv_tbl)
{
struct elevator_type *e = xa_load(elv_tbl, q->id);
struct elevator_tags *t = xa_load(et_tbl, q->id);
struct elv_change_ctx *ctx = xa_load(elv_tbl, q->id);
if (WARN_ON_ONCE(!ctx))
return;
/* The elv_update_nr_hw_queues unfreezes the queue. */
elv_update_nr_hw_queues(q, e, t);
elv_update_nr_hw_queues(q, ctx);
/* Drop the reference acquired in blk_mq_elv_switch_none. */
if (e)
elevator_put(e);
if (ctx->type)
elevator_put(ctx->type);
}
/*
* Stores elevator type in xarray and set current elevator to none. It uses
* q->id as an index to store the elevator type into the xarray.
* Stores elevator name and type in ctx and set current elevator to none.
*/
static int blk_mq_elv_switch_none(struct request_queue *q,
struct xarray *elv_tbl)
{
int ret = 0;
struct elv_change_ctx *ctx;
lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock);
@@ -5027,10 +5029,11 @@ static int blk_mq_elv_switch_none(struct request_queue *q,
* can't run concurrently.
*/
if (q->elevator) {
ctx = xa_load(elv_tbl, q->id);
if (WARN_ON_ONCE(!ctx))
return -ENOENT;
ret = xa_insert(elv_tbl, q->id, q->elevator->type, GFP_KERNEL);
if (WARN_ON_ONCE(ret))
return ret;
ctx->name = q->elevator->type->elevator_name;
/*
* Before we switch elevator to 'none', take a reference to
@@ -5041,9 +5044,14 @@ static int blk_mq_elv_switch_none(struct request_queue *q,
*/
__elevator_get(q->elevator->type);
/*
* Store elevator type so that we can release the reference
* taken above later.
*/
ctx->type = q->elevator->type;
elevator_set_none(q);
}
return ret;
return 0;
}
static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
@@ -5053,7 +5061,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
int prev_nr_hw_queues = set->nr_hw_queues;
unsigned int memflags;
int i;
struct xarray elv_tbl, et_tbl;
struct xarray elv_tbl;
bool queues_frozen = false;
lockdep_assert_held(&set->tag_list_lock);
@@ -5067,11 +5075,12 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
memflags = memalloc_noio_save();
xa_init(&et_tbl);
if (blk_mq_alloc_sched_tags_batch(&et_tbl, set, nr_hw_queues) < 0)
goto out_memalloc_restore;
xa_init(&elv_tbl);
if (blk_mq_alloc_sched_ctx_batch(&elv_tbl, set) < 0)
goto out_free_ctx;
if (blk_mq_alloc_sched_res_batch(&elv_tbl, set, nr_hw_queues) < 0)
goto out_free_ctx;
list_for_each_entry(q, &set->tag_list, tag_set_list) {
blk_mq_debugfs_unregister_hctxs(q);
@@ -5117,7 +5126,7 @@ switch_back:
/* switch_back expects queue to be frozen */
if (!queues_frozen)
blk_mq_freeze_queue_nomemsave(q);
blk_mq_elv_switch_back(q, &elv_tbl, &et_tbl);
blk_mq_elv_switch_back(q, &elv_tbl);
}
list_for_each_entry(q, &set->tag_list, tag_set_list) {
@@ -5128,9 +5137,9 @@ switch_back:
blk_mq_add_hw_queues_cpuhp(q);
}
out_free_ctx:
blk_mq_free_sched_ctx_batch(&elv_tbl);
xa_destroy(&elv_tbl);
xa_destroy(&et_tbl);
out_memalloc_restore:
memalloc_noio_restore(memflags);
/* Free the excess tags when nr_hw_queues shrink. */
block/blk-mq.h
+17 -4
View File
@@ -6,6 +6,7 @@
#include "blk-stat.h"
struct blk_mq_tag_set;
struct elevator_tags;
struct blk_mq_ctxs {
struct kobject kobj;
@@ -45,7 +46,9 @@ void blk_mq_submit_bio(struct bio *bio);
int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, struct io_comp_batch *iob,
unsigned int flags);
void blk_mq_exit_queue(struct request_queue *q);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q,
struct elevator_tags *tags,
unsigned int nr);
void blk_mq_wake_waiters(struct request_queue *q);
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
bool);
@@ -109,6 +112,17 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue(blk_opf_t opf,
return ctx->hctxs[blk_mq_get_hctx_type(opf)];
}
/*
* Default to double of smaller one between hw queue_depth and
* 128, since we don't split into sync/async like the old code
* did. Additionally, this is a per-hw queue depth.
*/
static inline unsigned int blk_mq_default_nr_requests(
struct blk_mq_tag_set *set)
{
return 2 * min_t(unsigned int, set->queue_depth, BLKDEV_DEFAULT_RQ);
}
/*
* sysfs helpers
*/
@@ -170,11 +184,10 @@ unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags,
void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
unsigned int tag);
void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags);
int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
struct blk_mq_tags **tags, unsigned int depth, bool can_grow);
void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set,
unsigned int size);
void blk_mq_tag_update_sched_shared_tags(struct request_queue *q);
void blk_mq_tag_update_sched_shared_tags(struct request_queue *q,
unsigned int nr);
void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn,
block/blk-sysfs.c
+55 -11
View File
@@ -64,28 +64,72 @@ static ssize_t queue_requests_show(struct gendisk *disk, char *page)
static ssize_t
queue_requests_store(struct gendisk *disk, const char *page, size_t count)
{
unsigned long nr;
int ret, err;
unsigned int memflags;
struct request_queue *q = disk->queue;
if (!queue_is_mq(q))
return -EINVAL;
struct blk_mq_tag_set *set = q->tag_set;
struct elevator_tags *et = NULL;
unsigned int memflags;
unsigned long nr;
int ret;
ret = queue_var_store(&nr, page, count);
if (ret < 0)
return ret;
memflags = blk_mq_freeze_queue(q);
mutex_lock(&q->elevator_lock);
/*
* Serialize updating nr_requests with concurrent queue_requests_store()
* and switching elevator.
*
* Use trylock to avoid circular lock dependency with kernfs active
* reference during concurrent disk deletion:
* update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del)
* kn->active -> update_nr_hwq_lock (via this sysfs write path)
*/
if (!down_write_trylock(&set->update_nr_hwq_lock))
return -EBUSY;
if (nr == q->nr_requests)
goto unlock;
if (nr < BLKDEV_MIN_RQ)
nr = BLKDEV_MIN_RQ;
err = blk_mq_update_nr_requests(disk->queue, nr);
if (err)
ret = err;
/*
* Switching elevator is protected by update_nr_hwq_lock:
* - read lock is held from elevator sysfs attribute;
* - write lock is held from updating nr_hw_queues;
* Hence it's safe to access q->elevator here with write lock held.
*/
if (nr <= set->reserved_tags ||
(q->elevator && nr > MAX_SCHED_RQ) ||
(!q->elevator && nr > set->queue_depth)) {
ret = -EINVAL;
goto unlock;
}
if (!blk_mq_is_shared_tags(set->flags) && q->elevator &&
nr > q->elevator->et->nr_requests) {
/*
* Tags will grow, allocate memory before freezing queue to
* prevent deadlock.
*/
et = blk_mq_alloc_sched_tags(set, q->nr_hw_queues, nr);
if (!et) {
ret = -ENOMEM;
goto unlock;
}
}
memflags = blk_mq_freeze_queue(q);
mutex_lock(&q->elevator_lock);
et = blk_mq_update_nr_requests(q, et, nr);
mutex_unlock(&q->elevator_lock);
blk_mq_unfreeze_queue(q, memflags);
if (et)
blk_mq_free_sched_tags(et, set);
unlock:
up_write(&set->update_nr_hwq_lock);
return ret;
}
block/blk.h
+3 -4
View File
@@ -11,8 +11,7 @@
#include <xen/xen.h>
#include "blk-crypto-internal.h"
struct elevator_type;
struct elevator_tags;
struct elv_change_ctx;
#define BLK_DEV_MAX_SECTORS (LLONG_MAX >> 9)
#define BLK_MIN_SEGMENT_SIZE 4096
@@ -324,8 +323,8 @@ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
bool blk_insert_flush(struct request *rq);
void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e,
struct elevator_tags *t);
void elv_update_nr_hw_queues(struct request_queue *q,
struct elv_change_ctx *ctx);
void elevator_set_default(struct request_queue *q);
void elevator_set_none(struct request_queue *q);
block/elevator.c
+49 -43
View File
@@ -45,19 +45,6 @@
#include "blk-wbt.h"
#include "blk-cgroup.h"
/* Holding context data for changing elevator */
struct elv_change_ctx {
const char *name;
bool no_uevent;
/* for unregistering old elevator */
struct elevator_queue *old;
/* for registering new elevator */
struct elevator_queue *new;
/* holds sched tags data */
struct elevator_tags *et;
};
static DEFINE_SPINLOCK(elv_list_lock);
static LIST_HEAD(elv_list);
@@ -134,7 +121,7 @@ static struct elevator_type *elevator_find_get(const char *name)
static const struct kobj_type elv_ktype;
struct elevator_queue *elevator_alloc(struct request_queue *q,
struct elevator_type *e, struct elevator_tags *et)
struct elevator_type *e, struct elevator_resources *res)
{
struct elevator_queue *eq;
@@ -147,7 +134,8 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
kobject_init(&eq->kobj, &elv_ktype);
mutex_init(&eq->sysfs_lock);
hash_init(eq->hash);
eq->et = et;
eq->et = res->et;
eq->elevator_data = res->data;
return eq;
}
@@ -593,7 +581,7 @@ static int elevator_switch(struct request_queue *q, struct elv_change_ctx *ctx)
}
if (new_e) {
ret = blk_mq_init_sched(q, new_e, ctx->et);
ret = blk_mq_init_sched(q, new_e, &ctx->res);
if (ret)
goto out_unfreeze;
ctx->new = q->elevator;
@@ -617,7 +605,8 @@ out_unfreeze:
return ret;
}
static void elv_exit_and_release(struct request_queue *q)
static void elv_exit_and_release(struct elv_change_ctx *ctx,
struct request_queue *q)
{
struct elevator_queue *e;
unsigned memflags;
@@ -629,7 +618,7 @@ static void elv_exit_and_release(struct request_queue *q)
mutex_unlock(&q->elevator_lock);
blk_mq_unfreeze_queue(q, memflags);
if (e) {
blk_mq_free_sched_tags(e->et, q->tag_set);
blk_mq_free_sched_res(&ctx->res, ctx->type, q->tag_set);
kobject_put(&e->kobj);
}
}
@@ -640,14 +629,19 @@ static int elevator_change_done(struct request_queue *q,
int ret = 0;
if (ctx->old) {
struct elevator_resources res = {
.et = ctx->old->et,
.data = ctx->old->elevator_data
};
elv_unregister_queue(q, ctx->old);
blk_mq_free_sched_tags(ctx->old->et, q->tag_set);
blk_mq_free_sched_res(&res, ctx->old->type, q->tag_set);
kobject_put(&ctx->old->kobj);
}
if (ctx->new) {
ret = elv_register_queue(q, ctx->new, !ctx->no_uevent);
if (ret)
elv_exit_and_release(q);
elv_exit_and_release(ctx, q);
}
return ret;
}
@@ -664,9 +658,10 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
lockdep_assert_held(&set->update_nr_hwq_lock);
if (strncmp(ctx->name, "none", 4)) {
ctx->et = blk_mq_alloc_sched_tags(set, set->nr_hw_queues);
if (!ctx->et)
return -ENOMEM;
ret = blk_mq_alloc_sched_res(q, ctx->type, &ctx->res,
set->nr_hw_queues);
if (ret)
return ret;
}
memflags = blk_mq_freeze_queue(q);
@@ -687,11 +682,12 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
blk_mq_unfreeze_queue(q, memflags);
if (!ret)
ret = elevator_change_done(q, ctx);
/*
* Free sched tags if it's allocated but we couldn't switch elevator.
* Free sched resource if it's allocated but we couldn't switch elevator.
*/
if (ctx->et && !ctx->new)
blk_mq_free_sched_tags(ctx->et, set);
if (!ctx->new)
blk_mq_free_sched_res(&ctx->res, ctx->type, set);
return ret;
}
@@ -700,32 +696,29 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
* The I/O scheduler depends on the number of hardware queues, this forces a
* reattachment when nr_hw_queues changes.
*/
void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e,
struct elevator_tags *t)
void elv_update_nr_hw_queues(struct request_queue *q,
struct elv_change_ctx *ctx)
{
struct blk_mq_tag_set *set = q->tag_set;
struct elv_change_ctx ctx = {};
int ret = -ENODEV;
WARN_ON_ONCE(q->mq_freeze_depth == 0);
if (e && !blk_queue_dying(q) && blk_queue_registered(q)) {
ctx.name = e->elevator_name;
ctx.et = t;
if (ctx->type && !blk_queue_dying(q) && blk_queue_registered(q)) {
mutex_lock(&q->elevator_lock);
/* force to reattach elevator after nr_hw_queue is updated */
ret = elevator_switch(q, &ctx);
ret = elevator_switch(q, ctx);
mutex_unlock(&q->elevator_lock);
}
blk_mq_unfreeze_queue_nomemrestore(q);
if (!ret)
WARN_ON_ONCE(elevator_change_done(q, &ctx));
WARN_ON_ONCE(elevator_change_done(q, ctx));
/*
* Free sched tags if it's allocated but we couldn't switch elevator.
* Free sched resource if it's allocated but we couldn't switch elevator.
*/
if (t && !ctx.new)
blk_mq_free_sched_tags(t, set);
if (!ctx->new)
blk_mq_free_sched_res(&ctx->res, ctx->type, set);
}
/*
@@ -739,7 +732,6 @@ void elevator_set_default(struct request_queue *q)
.no_uevent = true,
};
int err;
struct elevator_type *e;
/* now we allow to switch elevator */
blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q);
@@ -752,8 +744,8 @@ void elevator_set_default(struct request_queue *q)
* have multiple queues or mq-deadline is not available, default
* to "none".
*/
e = elevator_find_get(ctx.name);
if (!e)
ctx.type = elevator_find_get(ctx.name);
if (!ctx.type)
return;
if ((q->nr_hw_queues == 1 ||
@@ -763,7 +755,7 @@ void elevator_set_default(struct request_queue *q)
pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n",
ctx.name, err);
}
elevator_put(e);
elevator_put(ctx.type);
}
void elevator_set_none(struct request_queue *q)
@@ -812,8 +804,18 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
ctx.name = strstrip(elevator_name);
elv_iosched_load_module(ctx.name);
ctx.type = elevator_find_get(ctx.name);
down_read(&set->update_nr_hwq_lock);
/*
* Use trylock to avoid circular lock dependency with kernfs active
* reference during concurrent disk deletion:
* update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del)
* kn->active -> update_nr_hwq_lock (via this sysfs write path)
*/
if (!down_read_trylock(&set->update_nr_hwq_lock)) {
ret = -EBUSY;
goto out;
}
if (!blk_queue_no_elv_switch(q)) {
ret = elevator_change(q, &ctx);
if (!ret)
@@ -822,6 +824,10 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
ret = -ENOENT;
}
up_read(&set->update_nr_hwq_lock);
out:
if (ctx.type)
elevator_put(ctx.type);
return ret;
}
block/elevator.h
+25 -1
View File
@@ -33,12 +33,36 @@ struct elevator_tags {
struct blk_mq_tags *tags[];
};
struct elevator_resources {
/* holds elevator data */
void *data;
/* holds elevator tags */
struct elevator_tags *et;
};
/* Holding context data for changing elevator */
struct elv_change_ctx {
const char *name;
bool no_uevent;
/* for unregistering old elevator */
struct elevator_queue *old;
/* for registering new elevator */
struct elevator_queue *new;
/* store elevator type */
struct elevator_type *type;
/* store elevator resources */
struct elevator_resources res;
};
struct elevator_mq_ops {
int (*init_sched)(struct request_queue *, struct elevator_queue *);
void (*exit_sched)(struct elevator_queue *);
int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int);
void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int);
void (*depth_updated)(struct request_queue *);
void *(*alloc_sched_data)(struct request_queue *);
void (*free_sched_data)(void *);
bool (*allow_merge)(struct request_queue *, struct request *, struct bio *);
bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int);
@@ -173,7 +197,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *page, size_t count);
extern bool elv_bio_merge_ok(struct request *, struct bio *);
struct elevator_queue *elevator_alloc(struct request_queue *,
struct elevator_type *, struct elevator_tags *);
struct elevator_type *, struct elevator_resources *);
/*
* Helper functions.
block/kyber-iosched.c
+22 -8
View File
@@ -409,30 +409,42 @@ static void kyber_depth_updated(struct request_queue *q)
static int kyber_init_sched(struct request_queue *q, struct elevator_queue *eq)
{
struct kyber_queue_data *kqd;
kqd = kyber_queue_data_alloc(q);
if (IS_ERR(kqd))
return PTR_ERR(kqd);
blk_stat_enable_accounting(q);
blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q);
eq->elevator_data = kqd;
q->elevator = eq;
kyber_depth_updated(q);
return 0;
}
static void *kyber_alloc_sched_data(struct request_queue *q)
{
struct kyber_queue_data *kqd;
kqd = kyber_queue_data_alloc(q);
if (IS_ERR(kqd))
return NULL;
return kqd;
}
static void kyber_exit_sched(struct elevator_queue *e)
{
struct kyber_queue_data *kqd = e->elevator_data;
int i;
timer_shutdown_sync(&kqd->timer);
blk_stat_disable_accounting(kqd->q);
}
static void kyber_free_sched_data(void *elv_data)
{
struct kyber_queue_data *kqd = elv_data;
int i;
if (!kqd)
return;
for (i = 0; i < KYBER_NUM_DOMAINS; i++)
sbitmap_queue_free(&kqd->domain_tokens[i]);
@@ -1004,6 +1016,8 @@ static struct elevator_type kyber_sched = {
.exit_sched = kyber_exit_sched,
.init_hctx = kyber_init_hctx,
.exit_hctx = kyber_exit_hctx,
.alloc_sched_data = kyber_alloc_sched_data,
.free_sched_data = kyber_free_sched_data,
.limit_depth = kyber_limit_depth,
.bio_merge = kyber_bio_merge,
.prepare_request = kyber_prepare_request,