diff --git a/fs/attr.c b/fs/attr.c index b9ec6b47bab2..e7d7c6d19fe9 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -169,7 +169,17 @@ int setattr_prepare(struct mnt_idmap *idmap, struct dentry *dentry, * ATTR_FORCE. */ if (ia_valid & ATTR_SIZE) { - int error = inode_newsize_ok(inode, attr->ia_size); + int error; + + /* + * Verity files are immutable, so deny truncates. This isn't + * covered by the open-time check because sys_truncate() takes a + * path, not an open file. + */ + if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) + return -EPERM; + + error = inode_newsize_ok(inode, attr->ia_size); if (error) return error; } diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 73602ee8de3f..55c272fe5d92 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -339,10 +339,6 @@ struct btrfs_inode { struct rw_semaphore i_mmap_lock; -#ifdef CONFIG_FS_VERITY - struct fsverity_info *i_verity_info; -#endif - struct inode vfs_inode; }; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 3df399dc8856..744a1fff6eef 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -475,25 +475,25 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, end, page_ops); } -static bool btrfs_verify_folio(struct folio *folio, u64 start, u32 len) +static bool btrfs_verify_folio(struct fsverity_info *vi, struct folio *folio, + u64 start, u32 len) { struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); - if (!fsverity_active(folio->mapping->host) || - btrfs_folio_test_uptodate(fs_info, folio, start, len) || - start >= i_size_read(folio->mapping->host)) + if (!vi || btrfs_folio_test_uptodate(fs_info, folio, start, len)) return true; - return fsverity_verify_folio(folio); + return fsverity_verify_folio(vi, folio); } -static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 len) +static void end_folio_read(struct fsverity_info *vi, struct folio *folio, + bool uptodate, u64 start, u32 len) { struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); ASSERT(folio_pos(folio) <= start && start + len <= folio_next_pos(folio)); - if (uptodate && btrfs_verify_folio(folio, start, len)) + if (uptodate && btrfs_verify_folio(vi, folio, start, len)) btrfs_folio_set_uptodate(fs_info, folio, start, len); else btrfs_folio_clear_uptodate(fs_info, folio, start, len); @@ -573,14 +573,19 @@ static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio) static void end_bbio_data_read(struct btrfs_bio *bbio) { struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct inode *inode = &bbio->inode->vfs_inode; struct bio *bio = &bbio->bio; + struct fsverity_info *vi = NULL; struct folio_iter fi; ASSERT(!bio_flagged(bio, BIO_CLONED)); + + if (bbio->file_offset < i_size_read(inode)) + vi = fsverity_get_info(inode); + bio_for_each_folio_all(fi, &bbio->bio) { bool uptodate = !bio->bi_status; struct folio *folio = fi.folio; - struct inode *inode = folio->mapping->host; u64 start = folio_pos(folio) + fi.offset; btrfs_debug(fs_info, @@ -615,7 +620,7 @@ static void end_bbio_data_read(struct btrfs_bio *bbio) } /* Update page status and unlock. */ - end_folio_read(folio, uptodate, start, fi.length); + end_folio_read(vi, folio, uptodate, start, fi.length); } bio_put(bio); } @@ -990,7 +995,8 @@ static void btrfs_readahead_expand(struct readahead_control *ractl, * return 0 on success, otherwise return error */ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, - struct btrfs_bio_ctrl *bio_ctrl) + struct btrfs_bio_ctrl *bio_ctrl, + struct fsverity_info *vi) { struct inode *inode = folio->mapping->host; struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); @@ -1034,11 +1040,11 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, ASSERT(IS_ALIGNED(cur, fs_info->sectorsize)); if (cur >= last_byte) { folio_zero_range(folio, pg_offset, end - cur + 1); - end_folio_read(folio, true, cur, end - cur + 1); + end_folio_read(vi, folio, true, cur, end - cur + 1); break; } if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) { - end_folio_read(folio, true, cur, blocksize); + end_folio_read(vi, folio, true, cur, blocksize); continue; } /* @@ -1050,7 +1056,7 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, */ em = get_extent_map(BTRFS_I(inode), folio, cur, locked_end - cur + 1, em_cached); if (IS_ERR(em)) { - end_folio_read(folio, false, cur, end + 1 - cur); + end_folio_read(vi, folio, false, cur, end + 1 - cur); return PTR_ERR(em); } extent_offset = cur - em->start; @@ -1127,12 +1133,12 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, /* we've found a hole, just zero and go on */ if (block_start == EXTENT_MAP_HOLE) { folio_zero_range(folio, pg_offset, blocksize); - end_folio_read(folio, true, cur, blocksize); + end_folio_read(vi, folio, true, cur, blocksize); continue; } /* the get_extent function already copied into the folio */ if (block_start == EXTENT_MAP_INLINE) { - end_folio_read(folio, true, cur, blocksize); + end_folio_read(vi, folio, true, cur, blocksize); continue; } @@ -1329,7 +1335,8 @@ again: int btrfs_read_folio(struct file *file, struct folio *folio) { - struct btrfs_inode *inode = folio_to_inode(folio); + struct inode *vfs_inode = folio->mapping->host; + struct btrfs_inode *inode = BTRFS_I(vfs_inode); const u64 start = folio_pos(folio); const u64 end = start + folio_size(folio) - 1; struct extent_state *cached_state = NULL; @@ -1338,10 +1345,13 @@ int btrfs_read_folio(struct file *file, struct folio *folio) .last_em_start = U64_MAX, }; struct extent_map *em_cached = NULL; + struct fsverity_info *vi = NULL; int ret; lock_extents_for_read(inode, start, end, &cached_state); - ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl); + if (folio_pos(folio) < i_size_read(vfs_inode)) + vi = fsverity_get_info(vfs_inode); + ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi); btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state); btrfs_free_extent_map(em_cached); @@ -2714,16 +2724,19 @@ void btrfs_readahead(struct readahead_control *rac) .last_em_start = U64_MAX, }; struct folio *folio; - struct btrfs_inode *inode = BTRFS_I(rac->mapping->host); + struct inode *vfs_inode = rac->mapping->host; + struct btrfs_inode *inode = BTRFS_I(vfs_inode); const u64 start = readahead_pos(rac); const u64 end = start + readahead_length(rac) - 1; struct extent_state *cached_state = NULL; struct extent_map *em_cached = NULL; + struct fsverity_info *vi = NULL; lock_extents_for_read(inode, start, end, &cached_state); - + if (start < i_size_read(vfs_inode)) + vi = fsverity_get_info(vfs_inode); while ((folio = readahead_folio(rac)) != NULL) - btrfs_do_readpage(folio, &em_cached, &bio_ctrl); + btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi); btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 82df115bd0c5..845164485a00 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -34,7 +34,6 @@ #include #include #include -#include #include "misc.h" #include "ctree.h" #include "disk-io.h" @@ -5616,11 +5615,8 @@ void btrfs_evict_inode(struct inode *inode) trace_btrfs_inode_evict(inode); - if (!root) { - fsverity_cleanup_inode(inode); - clear_inode(inode); - return; - } + if (!root) + goto clear_inode; fs_info = inode_to_fs_info(inode); evict_inode_truncate_pages(inode); @@ -5720,7 +5716,7 @@ out: * to retry these periodically in the future. */ btrfs_remove_delayed_node(BTRFS_I(inode)); - fsverity_cleanup_inode(inode); +clear_inode: clear_inode(inode); } @@ -8151,9 +8147,6 @@ static void init_once(void *foo) struct btrfs_inode *ei = foo; inode_init_once(&ei->vfs_inode); -#ifdef CONFIG_FS_VERITY - ei->i_verity_info = NULL; -#endif } void __cold btrfs_destroy_cachep(void) diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 06cbd6f00a78..d12537a212e9 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -694,7 +694,6 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size) * * @inode: inode to read a merkle tree page for * @index: page index relative to the start of the merkle tree - * @num_ra_pages: number of pages to readahead. Optional, we ignore it * * The Merkle tree is stored in the filesystem btree, but its pages are cached * with a logical position past EOF in the inode's mapping. @@ -702,8 +701,7 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size) * Returns the page we read, or an ERR_PTR on error. */ static struct page *btrfs_read_merkle_tree_page(struct inode *inode, - pgoff_t index, - unsigned long num_ra_pages) + pgoff_t index) { struct folio *folio; u64 off = (u64)index << PAGE_SHIFT; @@ -771,16 +769,17 @@ out: /* * fsverity op that writes a Merkle tree block into the btree. * - * @inode: inode to write a Merkle tree block for + * @file: file to write a Merkle tree block for * @buf: Merkle tree block to write * @pos: the position of the block in the Merkle tree (in bytes) * @size: the Merkle tree block size (in bytes) * * Returns 0 on success or negative error code on failure */ -static int btrfs_write_merkle_tree_block(struct inode *inode, const void *buf, +static int btrfs_write_merkle_tree_block(struct file *file, const void *buf, u64 pos, unsigned int size) { + struct inode *inode = file_inode(file); loff_t merkle_pos = merkle_file_pos(inode); if (merkle_pos < 0) @@ -793,8 +792,6 @@ static int btrfs_write_merkle_tree_block(struct inode *inode, const void *buf, } const struct fsverity_operations btrfs_verityops = { - .inode_info_offs = (int)offsetof(struct btrfs_inode, i_verity_info) - - (int)offsetof(struct btrfs_inode, vfs_inode), .begin_enable_verity = btrfs_begin_enable_verity, .end_enable_verity = btrfs_end_enable_verity, .get_verity_descriptor = btrfs_get_verity_descriptor, diff --git a/fs/buffer.c b/fs/buffer.c index b67791690ed3..ed724a902657 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -303,6 +303,7 @@ still_busy: struct postprocess_bh_ctx { struct work_struct work; struct buffer_head *bh; + struct fsverity_info *vi; }; static void verify_bh(struct work_struct *work) @@ -312,21 +313,12 @@ static void verify_bh(struct work_struct *work) struct buffer_head *bh = ctx->bh; bool valid; - valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh)); + valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size, + bh_offset(bh)); end_buffer_async_read(bh, valid); kfree(ctx); } -static bool need_fsverity(struct buffer_head *bh) -{ - struct folio *folio = bh->b_folio; - struct inode *inode = folio->mapping->host; - - return fsverity_active(inode) && - /* needed by ext4 */ - folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE); -} - static void decrypt_bh(struct work_struct *work) { struct postprocess_bh_ctx *ctx = @@ -336,7 +328,7 @@ static void decrypt_bh(struct work_struct *work) err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size, bh_offset(bh)); - if (err == 0 && need_fsverity(bh)) { + if (err == 0 && ctx->vi) { /* * We use different work queues for decryption and for verity * because verity may require reading metadata pages that need @@ -358,15 +350,20 @@ static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) { struct inode *inode = bh->b_folio->mapping->host; bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode); - bool verify = need_fsverity(bh); + struct fsverity_info *vi = NULL; + + /* needed by ext4 */ + if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + vi = fsverity_get_info(inode); /* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */ - if (uptodate && (decrypt || verify)) { + if (uptodate && (decrypt || vi)) { struct postprocess_bh_ctx *ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC); if (ctx) { ctx->bh = bh; + ctx->vi = vi; if (decrypt) { INIT_WORK(&ctx->work, decrypt_bh); fscrypt_enqueue_decrypt_work(&ctx->work); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index e0ed273e2e8a..293f698b7042 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1196,10 +1196,6 @@ struct ext4_inode_info { #ifdef CONFIG_FS_ENCRYPTION struct fscrypt_inode_info *i_crypt_info; #endif - -#ifdef CONFIG_FS_VERITY - struct fsverity_info *i_verity_info; -#endif }; /* @@ -3744,8 +3740,8 @@ static inline void ext4_set_de_type(struct super_block *sb, } /* readpages.c */ -extern int ext4_mpage_readpages(struct inode *inode, - struct readahead_control *rac, struct folio *folio); +int ext4_read_folio(struct file *file, struct folio *folio); +void ext4_readahead(struct readahead_control *rac); extern int __init ext4_init_post_read_processing(void); extern void ext4_exit_post_read_processing(void); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 15ba4d42982f..8a544f79dcb4 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3373,33 +3373,6 @@ out: return ret; } -static int ext4_read_folio(struct file *file, struct folio *folio) -{ - int ret = -EAGAIN; - struct inode *inode = folio->mapping->host; - - trace_ext4_read_folio(inode, folio); - - if (ext4_has_inline_data(inode)) - ret = ext4_readpage_inline(inode, folio); - - if (ret == -EAGAIN) - return ext4_mpage_readpages(inode, NULL, folio); - - return ret; -} - -static void ext4_readahead(struct readahead_control *rac) -{ - struct inode *inode = rac->mapping->host; - - /* If the file has inline data, no need to do readahead. */ - if (ext4_has_inline_data(inode)) - return; - - ext4_mpage_readpages(inode, rac, NULL); -} - static void ext4_invalidate_folio(struct folio *folio, size_t offset, size_t length) { @@ -5815,10 +5788,6 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (error) return error; - error = fsverity_prepare_setattr(dentry, attr); - if (error) - return error; - if (is_quota_modification(idmap, inode, attr)) { error = dquot_initialize(inode); if (error) diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index 49a6d36a8dba..830f3b8a321f 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -46,6 +46,7 @@ #include #include "ext4.h" +#include #define NUM_PREALLOC_POST_READ_CTXS 128 @@ -62,6 +63,7 @@ enum bio_post_read_step { struct bio_post_read_ctx { struct bio *bio; + struct fsverity_info *vi; struct work_struct work; unsigned int cur_step; unsigned int enabled_steps; @@ -97,6 +99,7 @@ static void verity_work(struct work_struct *work) struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; + struct fsverity_info *vi = ctx->vi; /* * fsverity_verify_bio() may call readahead() again, and although verity @@ -109,7 +112,7 @@ static void verity_work(struct work_struct *work) mempool_free(ctx, bio_post_read_ctx_pool); bio->bi_private = NULL; - fsverity_verify_bio(bio); + fsverity_verify_bio(vi, bio); __read_end_io(bio); } @@ -131,7 +134,8 @@ static void bio_post_read_processing(struct bio_post_read_ctx *ctx) ctx->cur_step++; fallthrough; case STEP_VERITY: - if (ctx->enabled_steps & (1 << STEP_VERITY)) { + if (IS_ENABLED(CONFIG_FS_VERITY) && + ctx->enabled_steps & (1 << STEP_VERITY)) { INIT_WORK(&ctx->work, verity_work); fsverity_enqueue_verify_work(&ctx->work); return; @@ -172,22 +176,16 @@ static void mpage_end_io(struct bio *bio) __read_end_io(bio); } -static inline bool ext4_need_verity(const struct inode *inode, pgoff_t idx) -{ - return fsverity_active(inode) && - idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE); -} - static void ext4_set_bio_post_read_ctx(struct bio *bio, const struct inode *inode, - pgoff_t first_idx) + struct fsverity_info *vi) { unsigned int post_read_steps = 0; if (fscrypt_inode_uses_fs_layer_crypto(inode)) post_read_steps |= 1 << STEP_DECRYPT; - if (ext4_need_verity(inode, first_idx)) + if (vi) post_read_steps |= 1 << STEP_VERITY; if (post_read_steps) { @@ -196,6 +194,7 @@ static void ext4_set_bio_post_read_ctx(struct bio *bio, mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); ctx->bio = bio; + ctx->vi = vi; ctx->enabled_steps = post_read_steps; bio->bi_private = ctx; } @@ -209,7 +208,7 @@ static inline loff_t ext4_readpage_limit(struct inode *inode) return i_size_read(inode); } -int ext4_mpage_readpages(struct inode *inode, +static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi, struct readahead_control *rac, struct folio *folio) { struct bio *bio = NULL; @@ -329,8 +328,7 @@ int ext4_mpage_readpages(struct inode *inode, folio_zero_segment(folio, first_hole << blkbits, folio_size(folio)); if (first_hole == 0) { - if (ext4_need_verity(inode, folio->index) && - !fsverity_verify_folio(folio)) + if (vi && !fsverity_verify_folio(vi, folio)) goto set_error_page; folio_end_read(folio, true); continue; @@ -358,7 +356,7 @@ int ext4_mpage_readpages(struct inode *inode, REQ_OP_READ, GFP_KERNEL); fscrypt_set_bio_crypt_ctx(bio, inode, next_block, GFP_KERNEL); - ext4_set_bio_post_read_ctx(bio, inode, folio->index); + ext4_set_bio_post_read_ctx(bio, inode, vi); bio->bi_iter.bi_sector = first_block << (blkbits - 9); bio->bi_end_io = mpage_end_io; if (rac) @@ -394,6 +392,44 @@ next_page: return 0; } +int ext4_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct fsverity_info *vi = NULL; + int ret; + + trace_ext4_read_folio(inode, folio); + + if (ext4_has_inline_data(inode)) { + ret = ext4_readpage_inline(inode, folio); + if (ret != -EAGAIN) + return ret; + } + + if (folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + vi = fsverity_get_info(inode); + if (vi) + fsverity_readahead(vi, folio->index, folio_nr_pages(folio)); + return ext4_mpage_readpages(inode, vi, NULL, folio); +} + +void ext4_readahead(struct readahead_control *rac) +{ + struct inode *inode = rac->mapping->host; + struct fsverity_info *vi = NULL; + + /* If the file has inline data, no need to do readahead. */ + if (ext4_has_inline_data(inode)) + return; + + if (readahead_index(rac) < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + vi = fsverity_get_info(inode); + if (vi) + fsverity_readahead(vi, readahead_index(rac), + readahead_count(rac)); + ext4_mpage_readpages(inode, vi, rac, NULL); +} + int __init ext4_init_post_read_processing(void) { bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, SLAB_RECLAIM_ACCOUNT); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 69eb63dde983..504148b2142b 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1489,9 +1489,6 @@ static void init_once(void *foo) #ifdef CONFIG_FS_ENCRYPTION ei->i_crypt_info = NULL; #endif -#ifdef CONFIG_FS_VERITY - ei->i_verity_info = NULL; -#endif } static int __init init_inodecache(void) @@ -1539,7 +1536,6 @@ void ext4_clear_inode(struct inode *inode) EXT4_I(inode)->jinode = NULL; } fscrypt_put_encryption_info(inode); - fsverity_cleanup_inode(inode); } static struct inode *ext4_nfs_get_inode(struct super_block *sb, diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c index 667f9e8d4da9..ca61da53f313 100644 --- a/fs/ext4/verity.c +++ b/fs/ext4/verity.c @@ -360,42 +360,32 @@ static int ext4_get_verity_descriptor(struct inode *inode, void *buf, } static struct page *ext4_read_merkle_tree_page(struct inode *inode, - pgoff_t index, - unsigned long num_ra_pages) + pgoff_t index) { - struct folio *folio; - index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT; - - folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { - DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); - - if (!IS_ERR(folio)) - folio_put(folio); - else if (num_ra_pages > 1) - page_cache_ra_unbounded(&ractl, num_ra_pages, 0); - folio = read_mapping_folio(inode->i_mapping, index, NULL); - if (IS_ERR(folio)) - return ERR_CAST(folio); - } - return folio_file_page(folio, index); + return generic_read_merkle_tree_page(inode, index); } -static int ext4_write_merkle_tree_block(struct inode *inode, const void *buf, +static void ext4_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages) +{ + index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT; + generic_readahead_merkle_tree(inode, index, nr_pages); +} + +static int ext4_write_merkle_tree_block(struct file *file, const void *buf, u64 pos, unsigned int size) { - pos += ext4_verity_metadata_pos(inode); + pos += ext4_verity_metadata_pos(file_inode(file)); - return pagecache_write(inode, buf, size, pos); + return pagecache_write(file_inode(file), buf, size, pos); } const struct fsverity_operations ext4_verityops = { - .inode_info_offs = (int)offsetof(struct ext4_inode_info, i_verity_info) - - (int)offsetof(struct ext4_inode_info, vfs_inode), .begin_enable_verity = ext4_begin_enable_verity, .end_enable_verity = ext4_end_enable_verity, .get_verity_descriptor = ext4_get_verity_descriptor, .read_merkle_tree_page = ext4_read_merkle_tree_page, + .readahead_merkle_tree = ext4_readahead_merkle_tree, .write_merkle_tree_block = ext4_write_merkle_tree_block, }; diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 7b68bf22989d..ef1225af2acf 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1181,6 +1181,7 @@ int f2fs_prepare_compress_overwrite(struct inode *inode, .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, .rpages = NULL, .nr_rpages = 0, + .vi = NULL, /* can't write to fsverity files */ }; return prepare_compress_overwrite(&cc, pagep, index, fsdata); @@ -1716,7 +1717,7 @@ struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) dic->nr_cpages = cc->nr_cpages; refcount_set(&dic->refcnt, 1); dic->failed = false; - dic->need_verity = f2fs_need_verity(cc->inode, start_idx); + dic->vi = cc->vi; for (i = 0; i < dic->cluster_size; i++) dic->rpages[i] = cc->rpages[i]; @@ -1814,7 +1815,7 @@ static void f2fs_verify_cluster(struct work_struct *work) if (!rpage) continue; - if (fsverity_verify_page(rpage)) + if (fsverity_verify_page(dic->vi, rpage)) SetPageUptodate(rpage); else ClearPageUptodate(rpage); @@ -1833,7 +1834,7 @@ void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed, { int i; - if (!failed && dic->need_verity) { + if (IS_ENABLED(CONFIG_FS_VERITY) && !failed && dic->vi) { /* * Note that to avoid deadlocks, the verity work can't be done * on the decompression workqueue. This is because verifying diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index c3dd8a5c8589..491f66511201 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -109,6 +109,7 @@ enum bio_post_read_step { struct bio_post_read_ctx { struct bio *bio; struct f2fs_sb_info *sbi; + struct fsverity_info *vi; struct work_struct work; unsigned int enabled_steps; /* @@ -165,6 +166,7 @@ static void f2fs_verify_bio(struct work_struct *work) container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); + struct fsverity_info *vi = ctx->vi; /* * fsverity_verify_bio() may call readahead() again, and while verity @@ -187,13 +189,13 @@ static void f2fs_verify_bio(struct work_struct *work) struct folio *folio = fi.folio; if (!f2fs_is_compressed_page(folio) && - !fsverity_verify_page(&folio->page)) { + !fsverity_verify_page(vi, &folio->page)) { bio->bi_status = BLK_STS_IOERR; break; } } } else { - fsverity_verify_bio(bio); + fsverity_verify_bio(vi, bio); } f2fs_finish_read_bio(bio, true); @@ -1036,7 +1038,8 @@ out: f2fs_up_write(&io->io_rwsem); } -static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, +static struct bio *f2fs_grab_read_bio(struct inode *inode, + struct fsverity_info *vi, block_t blkaddr, unsigned nr_pages, blk_opf_t op_flag, pgoff_t first_idx, bool for_write) { @@ -1057,7 +1060,7 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, if (fscrypt_inode_uses_fs_layer_crypto(inode)) post_read_steps |= STEP_DECRYPT; - if (f2fs_need_verity(inode, first_idx)) + if (vi) post_read_steps |= STEP_VERITY; /* @@ -1072,6 +1075,7 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); ctx->bio = bio; ctx->sbi = sbi; + ctx->vi = vi; ctx->enabled_steps = post_read_steps; ctx->fs_blkaddr = blkaddr; ctx->decompression_attempted = false; @@ -1083,15 +1087,15 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, } /* This can handle encryption stuffs */ -static void f2fs_submit_page_read(struct inode *inode, struct folio *folio, - block_t blkaddr, blk_opf_t op_flags, - bool for_write) +static void f2fs_submit_page_read(struct inode *inode, struct fsverity_info *vi, + struct folio *folio, block_t blkaddr, + blk_opf_t op_flags, bool for_write) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct bio *bio; - bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags, - folio->index, for_write); + bio = f2fs_grab_read_bio(inode, vi, blkaddr, 1, op_flags, folio->index, + for_write); /* wait for GCed page writeback via META_MAPPING */ f2fs_wait_on_block_writeback(inode, blkaddr); @@ -1193,6 +1197,14 @@ int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) return err; } +static inline struct fsverity_info *f2fs_need_verity(const struct inode *inode, + pgoff_t idx) +{ + if (idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + return fsverity_get_info(inode); + return NULL; +} + struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index, blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs) { @@ -1258,8 +1270,8 @@ got_it: return folio; } - f2fs_submit_page_read(inode, folio, dn.data_blkaddr, - op_flags, for_write); + f2fs_submit_page_read(inode, f2fs_need_verity(inode, folio->index), + folio, dn.data_blkaddr, op_flags, for_write); return folio; put_err: @@ -2063,12 +2075,12 @@ static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac) return rac ? REQ_RAHEAD : 0; } -static int f2fs_read_single_page(struct inode *inode, struct folio *folio, - unsigned nr_pages, - struct f2fs_map_blocks *map, - struct bio **bio_ret, - sector_t *last_block_in_bio, - struct readahead_control *rac) +static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi, + struct folio *folio, unsigned int nr_pages, + struct f2fs_map_blocks *map, + struct bio **bio_ret, + sector_t *last_block_in_bio, + struct readahead_control *rac) { struct bio *bio = *bio_ret; const unsigned int blocksize = F2FS_BLKSIZE; @@ -2120,8 +2132,7 @@ got_it: } else { zero_out: folio_zero_segment(folio, 0, folio_size(folio)); - if (f2fs_need_verity(inode, index) && - !fsverity_verify_folio(folio)) { + if (vi && !fsverity_verify_folio(vi, folio)) { ret = -EIO; goto out; } @@ -2143,9 +2154,8 @@ submit_and_realloc: bio = NULL; } if (bio == NULL) - bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, - f2fs_ra_op_flags(rac), index, - false); + bio = f2fs_grab_read_bio(inode, vi, block_nr, nr_pages, + f2fs_ra_op_flags(rac), index, false); /* * If the page is under writeback, we need to wait for @@ -2295,9 +2305,10 @@ submit_and_realloc: } if (!bio) - bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages - i, - f2fs_ra_op_flags(rac), - folio->index, for_write); + bio = f2fs_grab_read_bio(inode, cc->vi, blkaddr, + nr_pages - i, + f2fs_ra_op_flags(rac), + folio->index, for_write); if (!bio_add_folio(bio, folio, blocksize, 0)) goto submit_and_realloc; @@ -2336,7 +2347,7 @@ out: * This function was originally taken from fs/mpage.c, and customized for f2fs. * Major change was from block_size == page_size in f2fs by default. */ -static int f2fs_mpage_readpages(struct inode *inode, +static int f2fs_mpage_readpages(struct inode *inode, struct fsverity_info *vi, struct readahead_control *rac, struct folio *folio) { struct bio *bio = NULL; @@ -2391,6 +2402,7 @@ static int f2fs_mpage_readpages(struct inode *inode, /* there are remained compressed pages, submit them */ if (!f2fs_cluster_can_merge_page(&cc, index)) { + cc.vi = vi; ret = f2fs_read_multi_pages(&cc, &bio, max_nr_pages, &last_block_in_bio, @@ -2424,8 +2436,9 @@ static int f2fs_mpage_readpages(struct inode *inode, read_single_page: #endif - ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map, - &bio, &last_block_in_bio, rac); + ret = f2fs_read_single_page(inode, vi, folio, max_nr_pages, + &map, &bio, &last_block_in_bio, + rac); if (ret) { #ifdef CONFIG_F2FS_FS_COMPRESSION set_error_page: @@ -2441,6 +2454,7 @@ next_page: if (f2fs_compressed_file(inode)) { /* last page */ if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) { + cc.vi = vi; ret = f2fs_read_multi_pages(&cc, &bio, max_nr_pages, &last_block_in_bio, @@ -2458,7 +2472,8 @@ next_page: static int f2fs_read_data_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; - int ret = -EAGAIN; + struct fsverity_info *vi = NULL; + int ret; trace_f2fs_readpage(folio, DATA); @@ -2468,16 +2483,22 @@ static int f2fs_read_data_folio(struct file *file, struct folio *folio) } /* If the file has inline data, try to read it directly */ - if (f2fs_has_inline_data(inode)) + if (f2fs_has_inline_data(inode)) { ret = f2fs_read_inline_data(inode, folio); - if (ret == -EAGAIN) - ret = f2fs_mpage_readpages(inode, NULL, folio); - return ret; + if (ret != -EAGAIN) + return ret; + } + + vi = f2fs_need_verity(inode, folio->index); + if (vi) + fsverity_readahead(vi, folio->index, folio_nr_pages(folio)); + return f2fs_mpage_readpages(inode, vi, NULL, folio); } static void f2fs_readahead(struct readahead_control *rac) { struct inode *inode = rac->mapping->host; + struct fsverity_info *vi = NULL; trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac)); @@ -2488,7 +2509,11 @@ static void f2fs_readahead(struct readahead_control *rac) if (f2fs_has_inline_data(inode)) return; - f2fs_mpage_readpages(inode, rac, NULL); + vi = f2fs_need_verity(inode, readahead_index(rac)); + if (vi) + fsverity_readahead(vi, readahead_index(rac), + readahead_count(rac)); + f2fs_mpage_readpages(inode, vi, rac, NULL); } int f2fs_encrypt_one_page(struct f2fs_io_info *fio) @@ -3637,9 +3662,10 @@ repeat: err = -EFSCORRUPTED; goto put_folio; } - f2fs_submit_page_read(use_cow ? - F2FS_I(inode)->cow_inode : inode, - folio, blkaddr, 0, true); + f2fs_submit_page_read(use_cow ? F2FS_I(inode)->cow_inode : + inode, + NULL, /* can't write to fsverity files */ + folio, blkaddr, 0, true); folio_lock(folio); if (unlikely(folio->mapping != mapping)) { diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 9f3aa3c7f126..a90a62cfe617 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -974,9 +974,6 @@ struct f2fs_inode_info { #ifdef CONFIG_FS_ENCRYPTION struct fscrypt_inode_info *i_crypt_info; /* filesystem encryption info */ #endif -#ifdef CONFIG_FS_VERITY - struct fsverity_info *i_verity_info; /* filesystem verity info */ -#endif }; static inline void get_read_extent_info(struct extent_info *ext, @@ -1603,6 +1600,7 @@ struct compress_ctx { size_t clen; /* valid data length in cbuf */ void *private; /* payload buffer for specified compression algorithm */ void *private2; /* extra payload buffer */ + struct fsverity_info *vi; /* verity info if needed */ }; /* compress context for write IO path */ @@ -1658,7 +1656,7 @@ struct decompress_io_ctx { refcount_t refcnt; bool failed; /* IO error occurred before decompression? */ - bool need_verity; /* need fs-verity verification after decompression? */ + struct fsverity_info *vi; /* fs-verity context if needed */ unsigned char compress_algorithm; /* backup algorithm type */ void *private; /* payload buffer for specified decompression algorithm */ void *private2; /* extra payload buffer */ @@ -4886,12 +4884,6 @@ static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi, return sbi->aligned_blksize; } -static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx) -{ - return fsverity_active(inode) && - idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE); -} - #ifdef CONFIG_F2FS_FAULT_INJECTION extern int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, unsigned long type, enum fault_option fo); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 31a0c1bb41f6..1fdbe18692be 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1076,10 +1076,6 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (err) return err; - err = fsverity_prepare_setattr(dentry, attr); - if (err) - return err; - if (unlikely(IS_IMMUTABLE(inode))) return -EPERM; @@ -4424,7 +4420,9 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) pgoff_t redirty_idx = page_idx; int page_len = 0, ret = 0; + filemap_invalidate_lock_shared(mapping); page_cache_ra_unbounded(&ractl, len, 0); + filemap_invalidate_unlock_shared(mapping); do { folio = read_cache_folio(mapping, page_idx, NULL, NULL); diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 38b8994bc1b2..ee332b994348 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -1000,7 +1000,6 @@ no_delete: } out_clear: fscrypt_put_encryption_info(inode); - fsverity_cleanup_inode(inode); clear_inode(inode); } diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index c4c225e09dc4..cd00d030edda 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -504,9 +504,6 @@ static void init_once(void *foo) #ifdef CONFIG_FS_ENCRYPTION fi->i_crypt_info = NULL; #endif -#ifdef CONFIG_FS_VERITY - fi->i_verity_info = NULL; -#endif } #ifdef CONFIG_QUOTA diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c index 05b935b55216..92ebcc19cab0 100644 --- a/fs/f2fs/verity.c +++ b/fs/f2fs/verity.c @@ -256,42 +256,32 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, } static struct page *f2fs_read_merkle_tree_page(struct inode *inode, - pgoff_t index, - unsigned long num_ra_pages) + pgoff_t index) { - struct folio *folio; - index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT; - - folio = f2fs_filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { - DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); - - if (!IS_ERR(folio)) - folio_put(folio); - else if (num_ra_pages > 1) - page_cache_ra_unbounded(&ractl, num_ra_pages, 0); - folio = read_mapping_folio(inode->i_mapping, index, NULL); - if (IS_ERR(folio)) - return ERR_CAST(folio); - } - return folio_file_page(folio, index); + return generic_read_merkle_tree_page(inode, index); } -static int f2fs_write_merkle_tree_block(struct inode *inode, const void *buf, +static void f2fs_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages) +{ + index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT; + generic_readahead_merkle_tree(inode, index, nr_pages); +} + +static int f2fs_write_merkle_tree_block(struct file *file, const void *buf, u64 pos, unsigned int size) { - pos += f2fs_verity_metadata_pos(inode); + pos += f2fs_verity_metadata_pos(file_inode(file)); - return pagecache_write(inode, buf, size, pos); + return pagecache_write(file_inode(file), buf, size, pos); } const struct fsverity_operations f2fs_verityops = { - .inode_info_offs = (int)offsetof(struct f2fs_inode_info, i_verity_info) - - (int)offsetof(struct f2fs_inode_info, vfs_inode), .begin_enable_verity = f2fs_begin_enable_verity, .end_enable_verity = f2fs_end_enable_verity, .get_verity_descriptor = f2fs_get_verity_descriptor, .read_merkle_tree_page = f2fs_read_merkle_tree_page, + .readahead_merkle_tree = f2fs_readahead_merkle_tree, .write_merkle_tree_block = f2fs_write_merkle_tree_block, }; diff --git a/fs/inode.c b/fs/inode.c index dae43a8de7e0..cc12b68e021b 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include /* for inode_has_buffers */ @@ -773,6 +774,14 @@ void dump_mapping(const struct address_space *mapping) void clear_inode(struct inode *inode) { + /* + * Only IS_VERITY() inodes can have verity info, so start by checking + * for IS_VERITY() (which is faster than retrieving the pointer to the + * verity info). This minimizes overhead for non-verity inodes. + */ + if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) + fsverity_cleanup_inode(inode); + /* * We have to cycle the i_pages lock here because reclaim can be in the * process of removing the last page (in __filemap_remove_folio()) diff --git a/fs/verity/Makefile b/fs/verity/Makefile index 435559a4fa9e..ddb4a88a0d60 100644 --- a/fs/verity/Makefile +++ b/fs/verity/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_FS_VERITY) += enable.o \ init.o \ measure.o \ open.o \ + pagecache.o \ read_metadata.o \ verify.o diff --git a/fs/verity/enable.c b/fs/verity/enable.c index 95ec42b84797..c9448074cce1 100644 --- a/fs/verity/enable.c +++ b/fs/verity/enable.c @@ -41,14 +41,15 @@ static int hash_one_block(const struct merkle_tree_params *params, return 0; } -static int write_merkle_tree_block(struct inode *inode, const u8 *buf, +static int write_merkle_tree_block(struct file *file, const u8 *buf, unsigned long index, const struct merkle_tree_params *params) { + struct inode *inode = file_inode(file); u64 pos = (u64)index << params->log_blocksize; int err; - err = inode->i_sb->s_vop->write_merkle_tree_block(inode, buf, pos, + err = inode->i_sb->s_vop->write_merkle_tree_block(file, buf, pos, params->block_size); if (err) fsverity_err(inode, "Error %d writing Merkle tree block %lu", @@ -135,7 +136,7 @@ static int build_merkle_tree(struct file *filp, err = hash_one_block(params, &buffers[level]); if (err) goto out; - err = write_merkle_tree_block(inode, + err = write_merkle_tree_block(filp, buffers[level].data, level_offset[level], params); @@ -155,7 +156,7 @@ static int build_merkle_tree(struct file *filp, err = hash_one_block(params, &buffers[level]); if (err) goto out; - err = write_merkle_tree_block(inode, + err = write_merkle_tree_block(filp, buffers[level].data, level_offset[level], params); @@ -264,9 +265,26 @@ static int enable_verity(struct file *filp, goto rollback; } + /* + * Add the fsverity_info into the hash table before finishing the + * initialization so that we don't have to undo the enabling when memory + * allocation for the hash table fails. This is safe because looking up + * the fsverity_info always first checks the S_VERITY flag on the inode, + * which will only be set at the very end of the ->end_enable_verity + * method. + */ + err = fsverity_set_info(vi); + if (err) { + fsverity_free_info(vi); + goto rollback; + } + /* * Tell the filesystem to finish enabling verity on the file. - * Serialized with ->begin_enable_verity() by the inode lock. + * Serialized with ->begin_enable_verity() by the inode lock. The file + * system needs to set the S_VERITY flag on the inode at the very end of + * the method, at which point the fsverity information can be accessed + * by other threads. */ inode_lock(inode); err = vops->end_enable_verity(filp, desc, desc_size, params.tree_size); @@ -274,19 +292,10 @@ static int enable_verity(struct file *filp, if (err) { fsverity_err(inode, "%ps() failed with err %d", vops->end_enable_verity, err); - fsverity_free_info(vi); + fsverity_remove_info(vi); } else if (WARN_ON_ONCE(!IS_VERITY(inode))) { + fsverity_remove_info(vi); err = -EINVAL; - fsverity_free_info(vi); - } else { - /* Successfully enabled verity */ - - /* - * Readers can start using the inode's verity info immediately, - * so it can't be rolled back once set. So don't set it until - * just after the filesystem has successfully enabled verity. - */ - fsverity_set_info(inode, vi); } out: kfree(params.hashstate); diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index dd20b138d452..2887cb849cec 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -11,6 +11,7 @@ #define pr_fmt(fmt) "fs-verity: " fmt #include +#include /* * Implementation limit: maximum depth of the Merkle tree. For now 8 is plenty; @@ -63,17 +64,18 @@ struct merkle_tree_params { * fsverity_info - cached verity metadata for an inode * * When a verity file is first opened, an instance of this struct is allocated - * and a pointer to it is stored in the file's in-memory inode. It remains - * until the inode is evicted. It caches information about the Merkle tree - * that's needed to efficiently verify data read from the file. It also caches - * the file digest. The Merkle tree pages themselves are not cached here, but - * the filesystem may cache them. + * and a pointer to it is stored in the global hash table, indexed by the inode + * pointer value. It remains alive until the inode is evicted. It caches + * information about the Merkle tree that's needed to efficiently verify data + * read from the file. It also caches the file digest. The Merkle tree pages + * themselves are not cached here, but the filesystem may cache them. */ struct fsverity_info { + struct rhash_head rhash_head; struct merkle_tree_params tree_params; u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE]; - const struct inode *inode; + struct inode *inode; unsigned long *hash_block_verified; }; @@ -124,12 +126,12 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, unsigned int log_blocksize, const u8 *salt, size_t salt_size); -struct fsverity_info *fsverity_create_info(const struct inode *inode, +struct fsverity_info *fsverity_create_info(struct inode *inode, struct fsverity_descriptor *desc); -void fsverity_set_info(struct inode *inode, struct fsverity_info *vi); - +int fsverity_set_info(struct fsverity_info *vi); void fsverity_free_info(struct fsverity_info *vi); +void fsverity_remove_info(struct fsverity_info *vi); int fsverity_get_descriptor(struct inode *inode, struct fsverity_descriptor **desc_ret); diff --git a/fs/verity/open.c b/fs/verity/open.c index 77b1c977af02..dfa0d1afe0fe 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -12,6 +12,14 @@ #include static struct kmem_cache *fsverity_info_cachep; +static struct rhashtable fsverity_info_hash; + +static const struct rhashtable_params fsverity_info_hash_params = { + .key_len = sizeof_field(struct fsverity_info, inode), + .key_offset = offsetof(struct fsverity_info, inode), + .head_offset = offsetof(struct fsverity_info, rhash_head), + .automatic_shrinking = true, +}; /** * fsverity_init_merkle_tree_params() - initialize Merkle tree parameters @@ -175,7 +183,7 @@ static void compute_file_digest(const struct fsverity_hash_alg *hash_alg, * appended builtin signature), and check the signature if present. The * fsverity_descriptor must have already undergone basic validation. */ -struct fsverity_info *fsverity_create_info(const struct inode *inode, +struct fsverity_info *fsverity_create_info(struct inode *inode, struct fsverity_descriptor *desc) { struct fsverity_info *vi; @@ -241,33 +249,19 @@ fail: return ERR_PTR(err); } -void fsverity_set_info(struct inode *inode, struct fsverity_info *vi) +int fsverity_set_info(struct fsverity_info *vi) { - /* - * Multiple tasks may race to set the inode's verity info pointer, so - * use cmpxchg_release(). This pairs with the smp_load_acquire() in - * fsverity_get_info(). I.e., publish the pointer with a RELEASE - * barrier so that other tasks can ACQUIRE it. - */ - if (cmpxchg_release(fsverity_info_addr(inode), NULL, vi) != NULL) { - /* Lost the race, so free the verity info we allocated. */ - fsverity_free_info(vi); - /* - * Afterwards, the caller may access the inode's verity info - * directly, so make sure to ACQUIRE the winning verity info. - */ - (void)fsverity_get_info(inode); - } + return rhashtable_lookup_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); } -void fsverity_free_info(struct fsverity_info *vi) +struct fsverity_info *__fsverity_get_info(const struct inode *inode) { - if (!vi) - return; - kfree(vi->tree_params.hashstate); - kvfree(vi->hash_block_verified); - kmem_cache_free(fsverity_info_cachep, vi); + return rhashtable_lookup_fast(&fsverity_info_hash, &inode, + fsverity_info_hash_params); } +EXPORT_SYMBOL_GPL(__fsverity_get_info); static bool validate_fsverity_descriptor(struct inode *inode, const struct fsverity_descriptor *desc, @@ -352,7 +346,7 @@ int fsverity_get_descriptor(struct inode *inode, static int ensure_verity_info(struct inode *inode) { - struct fsverity_info *vi = fsverity_get_info(inode); + struct fsverity_info *vi = fsverity_get_info(inode), *found; struct fsverity_descriptor *desc; int err; @@ -369,8 +363,19 @@ static int ensure_verity_info(struct inode *inode) goto out_free_desc; } - fsverity_set_info(inode, vi); - err = 0; + /* + * Multiple tasks may race to set the inode's verity info, in which case + * we might find an existing fsverity_info in the hash table. + */ + found = rhashtable_lookup_get_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); + if (found) { + fsverity_free_info(vi); + if (IS_ERR(found)) + err = PTR_ERR(found); + } + out_free_desc: kfree(desc); return err; @@ -384,25 +389,32 @@ int __fsverity_file_open(struct inode *inode, struct file *filp) } EXPORT_SYMBOL_GPL(__fsverity_file_open); -int __fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr) +void fsverity_free_info(struct fsverity_info *vi) { - if (attr->ia_valid & ATTR_SIZE) - return -EPERM; - return 0; + kfree(vi->tree_params.hashstate); + kvfree(vi->hash_block_verified); + kmem_cache_free(fsverity_info_cachep, vi); } -EXPORT_SYMBOL_GPL(__fsverity_prepare_setattr); -void __fsverity_cleanup_inode(struct inode *inode) +void fsverity_remove_info(struct fsverity_info *vi) { - struct fsverity_info **vi_addr = fsverity_info_addr(inode); - - fsverity_free_info(*vi_addr); - *vi_addr = NULL; + rhashtable_remove_fast(&fsverity_info_hash, &vi->rhash_head, + fsverity_info_hash_params); + fsverity_free_info(vi); +} + +void fsverity_cleanup_inode(struct inode *inode) +{ + struct fsverity_info *vi = fsverity_get_info(inode); + + if (vi) + fsverity_remove_info(vi); } -EXPORT_SYMBOL_GPL(__fsverity_cleanup_inode); void __init fsverity_init_info_cache(void) { + if (rhashtable_init(&fsverity_info_hash, &fsverity_info_hash_params)) + panic("failed to initialize fsverity hash\n"); fsverity_info_cachep = KMEM_CACHE_USERCOPY( fsverity_info, SLAB_RECLAIM_ACCOUNT | SLAB_PANIC, diff --git a/fs/verity/pagecache.c b/fs/verity/pagecache.c new file mode 100644 index 000000000000..1819314ecaa3 --- /dev/null +++ b/fs/verity/pagecache.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ + +#include +#include +#include + +/** + * generic_read_merkle_tree_page - generic ->read_merkle_tree_page helper + * @inode: inode containing the Merkle tree + * @index: 0-based index of the Merkle tree page in the inode + * + * The caller needs to adjust @index from the Merkle-tree relative index passed + * to ->read_merkle_tree_page to the actual index where the Merkle tree is + * stored in the page cache for @inode. + */ +struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index) +{ + struct folio *folio; + + folio = read_mapping_folio(inode->i_mapping, index, NULL); + if (IS_ERR(folio)) + return ERR_CAST(folio); + return folio_file_page(folio, index); +} +EXPORT_SYMBOL_GPL(generic_read_merkle_tree_page); + +/** + * generic_readahead_merkle_tree() - generic ->readahead_merkle_tree helper + * @inode: inode containing the Merkle tree + * @index: 0-based index of the first Merkle tree page to read ahead in the + * inode + * @nr_pages: the number of Merkle tree pages that should be read ahead + * + * The caller needs to adjust @index from the Merkle-tree relative index passed + * to ->read_merkle_tree_page to the actual index where the Merkle tree is + * stored in the page cache for @inode. + */ +void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages) +{ + struct folio *folio; + + lockdep_assert_held(&inode->i_mapping->invalidate_lock); + + folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0); + if (folio == ERR_PTR(-ENOENT) || + (!IS_ERR(folio) && !folio_test_uptodate(folio))) { + DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); + + page_cache_ra_unbounded(&ractl, nr_pages, 0); + } + if (!IS_ERR(folio)) + folio_put(folio); +} +EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree); diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c index cba5d6af4e04..b4c0892430cd 100644 --- a/fs/verity/read_metadata.c +++ b/fs/verity/read_metadata.c @@ -28,24 +28,33 @@ static int fsverity_read_merkle_tree(struct inode *inode, if (offset >= end_offset) return 0; offs_in_page = offset_in_page(offset); + index = offset >> PAGE_SHIFT; last_index = (end_offset - 1) >> PAGE_SHIFT; + /* + * Kick off readahead for the range we are going to read to ensure a + * single large sequential read instead of lots of small ones. + */ + if (inode->i_sb->s_vop->readahead_merkle_tree) { + filemap_invalidate_lock_shared(inode->i_mapping); + inode->i_sb->s_vop->readahead_merkle_tree( + inode, index, last_index - index + 1); + filemap_invalidate_unlock_shared(inode->i_mapping); + } + /* * Iterate through each Merkle tree page in the requested range and copy * the requested portion to userspace. Note that the Merkle tree block * size isn't important here, as we are returning a byte stream; i.e., * we can just work with pages even if the tree block size != PAGE_SIZE. */ - for (index = offset >> PAGE_SHIFT; index <= last_index; index++) { - unsigned long num_ra_pages = - min_t(unsigned long, last_index - index + 1, - inode->i_sb->s_bdi->io_pages); + for (; index <= last_index; index++) { unsigned int bytes_to_copy = min_t(u64, end_offset - offset, PAGE_SIZE - offs_in_page); struct page *page; const void *virt; - page = vops->read_merkle_tree_page(inode, index, num_ra_pages); + page = vops->read_merkle_tree_page(inode, index); if (IS_ERR(page)) { err = PTR_ERR(page); fsverity_err(inode, diff --git a/fs/verity/verify.c b/fs/verity/verify.c index 86067c8b40cf..31797f9b24d0 100644 --- a/fs/verity/verify.c +++ b/fs/verity/verify.c @@ -19,9 +19,7 @@ struct fsverity_pending_block { }; struct fsverity_verification_context { - struct inode *inode; struct fsverity_info *vi; - unsigned long max_ra_pages; /* * This is the queue of data blocks that are pending verification. When @@ -37,6 +35,50 @@ struct fsverity_verification_context { static struct workqueue_struct *fsverity_read_workqueue; +/** + * fsverity_readahead() - kick off readahead on fsverity hashes + * @vi: fsverity_info for the inode to be read + * @index: first file data page index that is being read + * @nr_pages: number of file data pages to be read + * + * Start readahead on the fsverity hashes that are needed to verify the file + * data in the range from @index to @index + @nr_pages (exclusive upper bound). + * + * To be called from the file systems' ->read_folio and ->readahead methods to + * ensure that the hashes are already cached on completion of the file data + * read if possible. + */ +void fsverity_readahead(struct fsverity_info *vi, pgoff_t index, + unsigned long nr_pages) +{ + struct inode *inode = vi->inode; + const struct merkle_tree_params *params = &vi->tree_params; + u64 start_hidx = (u64)index << params->log_blocks_per_page; + u64 end_hidx = + (((u64)index + nr_pages) << params->log_blocks_per_page) - 1; + int level; + + if (!inode->i_sb->s_vop->readahead_merkle_tree) + return; + + for (level = 0; level < params->num_levels; level++) { + unsigned long level_start = params->level_start[level]; + unsigned long next_start_hidx = start_hidx >> params->log_arity; + unsigned long next_end_hidx = end_hidx >> params->log_arity; + pgoff_t start_idx = (level_start + next_start_hidx) >> + params->log_blocks_per_page; + pgoff_t end_idx = (level_start + next_end_hidx) >> + params->log_blocks_per_page; + + inode->i_sb->s_vop->readahead_merkle_tree( + inode, start_idx, end_idx - start_idx + 1); + + start_hidx = next_start_hidx; + end_hidx = next_end_hidx; + } +} +EXPORT_SYMBOL_GPL(fsverity_readahead); + /* * Returns true if the hash block with index @hblock_idx in the tree, located in * @hpage, has already been verified. @@ -113,10 +155,10 @@ static bool is_hash_block_verified(struct fsverity_info *vi, struct page *hpage, * * Return: %true if the data block is valid, else %false. */ -static bool verify_data_block(struct inode *inode, struct fsverity_info *vi, - const struct fsverity_pending_block *dblock, - unsigned long max_ra_pages) +static bool verify_data_block(struct fsverity_info *vi, + const struct fsverity_pending_block *dblock) { + struct inode *inode = vi->inode; const u64 data_pos = dblock->pos; const struct merkle_tree_params *params = &vi->tree_params; const unsigned int hsize = params->digest_size; @@ -200,8 +242,7 @@ static bool verify_data_block(struct inode *inode, struct fsverity_info *vi, (params->block_size - 1); hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode, - hpage_idx, level == 0 ? min(max_ra_pages, - params->tree_pages - hpage_idx) : 0); + hpage_idx); if (IS_ERR(hpage)) { fsverity_err(inode, "Error %ld reading Merkle tree page %lu", @@ -272,14 +313,9 @@ error: static void fsverity_init_verification_context(struct fsverity_verification_context *ctx, - struct inode *inode, - unsigned long max_ra_pages) + struct fsverity_info *vi) { - struct fsverity_info *vi = *fsverity_info_addr(inode); - - ctx->inode = inode; ctx->vi = vi; - ctx->max_ra_pages = max_ra_pages; ctx->num_pending = 0; if (vi->tree_params.hash_alg->algo_id == HASH_ALGO_SHA256 && sha256_finup_2x_is_optimized()) @@ -322,8 +358,7 @@ fsverity_verify_pending_blocks(struct fsverity_verification_context *ctx) } for (i = 0; i < ctx->num_pending; i++) { - if (!verify_data_block(ctx->inode, vi, &ctx->pending_blocks[i], - ctx->max_ra_pages)) + if (!verify_data_block(vi, &ctx->pending_blocks[i])) return false; } fsverity_clear_pending_blocks(ctx); @@ -359,6 +394,7 @@ static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx, /** * fsverity_verify_blocks() - verify data in a folio + * @vi: fsverity_info for the inode to be read * @folio: the folio containing the data to verify * @len: the length of the data to verify in the folio * @offset: the offset of the data to verify in the folio @@ -369,11 +405,12 @@ static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx, * * Return: %true if the data is valid, else %false. */ -bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset) +bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio, + size_t len, size_t offset) { struct fsverity_verification_context ctx; - fsverity_init_verification_context(&ctx, folio->mapping->host, 0); + fsverity_init_verification_context(&ctx, vi); if (fsverity_add_data_blocks(&ctx, folio, len, offset) && fsverity_verify_pending_blocks(&ctx)) @@ -386,6 +423,7 @@ EXPORT_SYMBOL_GPL(fsverity_verify_blocks); #ifdef CONFIG_BLOCK /** * fsverity_verify_bio() - verify a 'read' bio that has just completed + * @vi: fsverity_info for the inode to be read * @bio: the bio to verify * * Verify the bio's data against the file's Merkle tree. All bio data segments @@ -398,27 +436,12 @@ EXPORT_SYMBOL_GPL(fsverity_verify_blocks); * filesystems) must instead call fsverity_verify_page() directly on each page. * All filesystems must also call fsverity_verify_page() on holes. */ -void fsverity_verify_bio(struct bio *bio) +void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio) { - struct inode *inode = bio_first_folio_all(bio)->mapping->host; struct fsverity_verification_context ctx; struct folio_iter fi; - unsigned long max_ra_pages = 0; - if (bio->bi_opf & REQ_RAHEAD) { - /* - * If this bio is for data readahead, then we also do readahead - * of the first (largest) level of the Merkle tree. Namely, - * when a Merkle tree page is read, we also try to piggy-back on - * some additional pages -- up to 1/4 the number of data pages. - * - * This improves sequential read performance, as it greatly - * reduces the number of I/O requests made to the Merkle tree. - */ - max_ra_pages = bio->bi_iter.bi_size >> (PAGE_SHIFT + 2); - } - - fsverity_init_verification_context(&ctx, inode, max_ra_pages); + fsverity_init_verification_context(&ctx, vi); bio_for_each_folio_all(fi, bio) { if (!fsverity_add_data_blocks(&ctx, fi.folio, fi.length, diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index 5bc7280425a7..fed91023bea9 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -30,13 +30,6 @@ struct fsverity_info; /* Verity operations for filesystems */ struct fsverity_operations { - /** - * The offset of the pointer to struct fsverity_info in the - * filesystem-specific part of the inode, relative to the beginning of - * the common part of the inode (the 'struct inode'). - */ - ptrdiff_t inode_info_offs; - /** * Begin enabling verity on the given file. * @@ -97,10 +90,6 @@ struct fsverity_operations { * * @inode: the inode * @index: 0-based index of the page within the Merkle tree - * @num_ra_pages: The number of Merkle tree pages that should be - * prefetched starting at @index if the page at @index - * isn't already cached. Implementations may ignore this - * argument; it's only a performance optimization. * * This can be called at any time on an open verity file. It may be * called by multiple processes concurrently, even with the same page. @@ -110,13 +99,28 @@ struct fsverity_operations { * Return: the page on success, ERR_PTR() on failure */ struct page *(*read_merkle_tree_page)(struct inode *inode, - pgoff_t index, - unsigned long num_ra_pages); + pgoff_t index); /** - * Write a Merkle tree block to the given inode. + * Perform readahead of a Merkle tree for the given inode. * - * @inode: the inode for which the Merkle tree is being built + * @inode: the inode + * @index: 0-based index of the first page within the Merkle tree + * @nr_pages: number of pages to be read ahead. + * + * This can be called at any time on an open verity file. It may be + * called by multiple processes concurrently, even with the same range. + * + * Optional method so that ->read_merkle_tree_page preferably finds + * cached data instead of issuing dependent I/O. + */ + void (*readahead_merkle_tree)(struct inode *inode, pgoff_t index, + unsigned long nr_pages); + + /** + * Write a Merkle tree block to the given file. + * + * @file: the file for which the Merkle tree is being built * @buf: the Merkle tree block to write * @pos: the position of the block in the Merkle tree (in bytes) * @size: the Merkle tree block size (in bytes) @@ -126,43 +130,48 @@ struct fsverity_operations { * * Return: 0 on success, -errno on failure */ - int (*write_merkle_tree_block)(struct inode *inode, const void *buf, + int (*write_merkle_tree_block)(struct file *file, const void *buf, u64 pos, unsigned int size); }; #ifdef CONFIG_FS_VERITY - -/* - * Returns the address of the verity info pointer within the filesystem-specific - * part of the inode. (To save memory on filesystems that don't support - * fsverity, a field in 'struct inode' itself is no longer used.) +/** + * fsverity_active() - do reads from the inode need to go through fs-verity? + * @inode: inode to check + * + * This checks whether the inode's verity info has been set, and reads need + * to verify the file data. + * + * Return: true if reads need to go through fs-verity, otherwise false */ -static inline struct fsverity_info ** -fsverity_info_addr(const struct inode *inode) +static inline bool fsverity_active(const struct inode *inode) { - VFS_WARN_ON_ONCE(inode->i_sb->s_vop->inode_info_offs == 0); - return (void *)inode + inode->i_sb->s_vop->inode_info_offs; + if (IS_VERITY(inode)) { + /* + * This pairs with the try_cmpxchg in set_mask_bits() + * used to set the S_VERITY bit in i_flags. + */ + smp_mb(); + return true; + } + + return false; } +struct fsverity_info *__fsverity_get_info(const struct inode *inode); +/** + * fsverity_get_info - get fsverity information for an inode + * @inode: inode to operate on. + * + * This gets the fsverity_info for @inode if it exists. Safe to call without + * knowin that a fsverity_info exist for @inode, including on file systems that + * do not support fsverity. + */ static inline struct fsverity_info *fsverity_get_info(const struct inode *inode) { - /* - * Since this function can be called on inodes belonging to filesystems - * that don't support fsverity at all, and fsverity_info_addr() doesn't - * work on such filesystems, we have to start with an IS_VERITY() check. - * Checking IS_VERITY() here is also useful to minimize the overhead of - * fsverity_active() on non-verity files. - */ - if (!IS_VERITY(inode)) + if (!fsverity_active(inode)) return NULL; - - /* - * Pairs with the cmpxchg_release() in fsverity_set_info(). I.e., - * another task may publish the inode's verity info concurrently, - * executing a RELEASE barrier. Use smp_load_acquire() here to safely - * ACQUIRE the memory the other task published. - */ - return smp_load_acquire(fsverity_info_addr(inode)); + return __fsverity_get_info(inode); } /* enable.c */ @@ -179,27 +188,6 @@ int fsverity_get_digest(struct inode *inode, /* open.c */ int __fsverity_file_open(struct inode *inode, struct file *filp); -int __fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr); -void __fsverity_cleanup_inode(struct inode *inode); - -/** - * fsverity_cleanup_inode() - free the inode's verity info, if present - * @inode: an inode being evicted - * - * Filesystems must call this on inode eviction to free the inode's verity info. - */ -static inline void fsverity_cleanup_inode(struct inode *inode) -{ - /* - * Only IS_VERITY() inodes can have verity info, so start by checking - * for IS_VERITY() (which is faster than retrieving the pointer to the - * verity info). This minimizes overhead for non-verity inodes. - */ - if (IS_VERITY(inode)) - __fsverity_cleanup_inode(inode); - else - VFS_WARN_ON_ONCE(*fsverity_info_addr(inode) != NULL); -} /* read_metadata.c */ @@ -207,12 +195,18 @@ int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg); /* verify.c */ -bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset); -void fsverity_verify_bio(struct bio *bio); +bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio, + size_t len, size_t offset); +void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio); void fsverity_enqueue_verify_work(struct work_struct *work); #else /* !CONFIG_FS_VERITY */ +static inline bool fsverity_active(const struct inode *inode) +{ + return false; +} + static inline struct fsverity_info *fsverity_get_info(const struct inode *inode) { return NULL; @@ -251,16 +245,6 @@ static inline int __fsverity_file_open(struct inode *inode, struct file *filp) return -EOPNOTSUPP; } -static inline int __fsverity_prepare_setattr(struct dentry *dentry, - struct iattr *attr) -{ - return -EOPNOTSUPP; -} - -static inline void fsverity_cleanup_inode(struct inode *inode) -{ -} - /* read_metadata.c */ static inline int fsverity_ioctl_read_metadata(struct file *filp, @@ -271,14 +255,16 @@ static inline int fsverity_ioctl_read_metadata(struct file *filp, /* verify.c */ -static inline bool fsverity_verify_blocks(struct folio *folio, size_t len, +static inline bool fsverity_verify_blocks(struct fsverity_info *vi, + struct folio *folio, size_t len, size_t offset) { WARN_ON_ONCE(1); return false; } -static inline void fsverity_verify_bio(struct bio *bio) +static inline void fsverity_verify_bio(struct fsverity_info *vi, + struct bio *bio) { WARN_ON_ONCE(1); } @@ -290,32 +276,16 @@ static inline void fsverity_enqueue_verify_work(struct work_struct *work) #endif /* !CONFIG_FS_VERITY */ -static inline bool fsverity_verify_folio(struct folio *folio) +static inline bool fsverity_verify_folio(struct fsverity_info *vi, + struct folio *folio) { - return fsverity_verify_blocks(folio, folio_size(folio), 0); + return fsverity_verify_blocks(vi, folio, folio_size(folio), 0); } -static inline bool fsverity_verify_page(struct page *page) +static inline bool fsverity_verify_page(struct fsverity_info *vi, + struct page *page) { - return fsverity_verify_blocks(page_folio(page), PAGE_SIZE, 0); -} - -/** - * fsverity_active() - do reads from the inode need to go through fs-verity? - * @inode: inode to check - * - * This checks whether the inode's verity info has been set. - * - * Filesystems call this from ->readahead() to check whether the pages need to - * be verified or not. Don't use IS_VERITY() for this purpose; it's subject to - * a race condition where the file is being read concurrently with - * FS_IOC_ENABLE_VERITY completing. (S_VERITY is set before the verity info.) - * - * Return: true if reads need to go through fs-verity, otherwise false - */ -static inline bool fsverity_active(const struct inode *inode) -{ - return fsverity_get_info(inode) != NULL; + return fsverity_verify_blocks(vi, page_folio(page), PAGE_SIZE, 0); } /** @@ -338,22 +308,12 @@ static inline int fsverity_file_open(struct inode *inode, struct file *filp) return 0; } -/** - * fsverity_prepare_setattr() - prepare to change a verity inode's attributes - * @dentry: dentry through which the inode is being changed - * @attr: attributes to change - * - * Verity files are immutable, so deny truncates. This isn't covered by the - * open-time check because sys_truncate() takes a path, not a file descriptor. - * - * Return: 0 on success, -errno on failure - */ -static inline int fsverity_prepare_setattr(struct dentry *dentry, - struct iattr *attr) -{ - if (IS_VERITY(d_inode(dentry))) - return __fsverity_prepare_setattr(dentry, attr); - return 0; -} +void fsverity_cleanup_inode(struct inode *inode); +void fsverity_readahead(struct fsverity_info *vi, pgoff_t index, + unsigned long nr_pages); + +struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index); +void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages); #endif /* _LINUX_FSVERITY_H */ diff --git a/mm/readahead.c b/mm/readahead.c index b415c9969176..f43d03558e62 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -204,8 +204,9 @@ static struct folio *ractl_alloc_folio(struct readahead_control *ractl, * not the function you want to call. Use page_cache_async_readahead() * or page_cache_sync_readahead() instead. * - * Context: File is referenced by caller. Mutexes may be held by caller. - * May sleep, but will not reenter filesystem to reclaim memory. + * Context: File is referenced by caller, and ractl->mapping->invalidate_lock + * must be held by the caller at least in shared mode. Mutexes may be held by + * caller. May sleep, but will not reenter filesystem to reclaim memory. */ void page_cache_ra_unbounded(struct readahead_control *ractl, unsigned long nr_to_read, unsigned long lookahead_size) @@ -228,9 +229,10 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, */ unsigned int nofs = memalloc_nofs_save(); + lockdep_assert_held(&mapping->invalidate_lock); + trace_page_cache_ra_unbounded(mapping->host, index, nr_to_read, lookahead_size); - filemap_invalidate_lock_shared(mapping); index = mapping_align_index(mapping, index); /* @@ -300,7 +302,6 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, * will then handle the error. */ read_pages(ractl); - filemap_invalidate_unlock_shared(mapping); memalloc_nofs_restore(nofs); } EXPORT_SYMBOL_GPL(page_cache_ra_unbounded); @@ -314,9 +315,9 @@ EXPORT_SYMBOL_GPL(page_cache_ra_unbounded); static void do_page_cache_ra(struct readahead_control *ractl, unsigned long nr_to_read, unsigned long lookahead_size) { - struct inode *inode = ractl->mapping->host; + struct address_space *mapping = ractl->mapping; unsigned long index = readahead_index(ractl); - loff_t isize = i_size_read(inode); + loff_t isize = i_size_read(mapping->host); pgoff_t end_index; /* The last page we want to read */ if (isize == 0) @@ -329,7 +330,9 @@ static void do_page_cache_ra(struct readahead_control *ractl, if (nr_to_read > end_index - index) nr_to_read = end_index - index + 1; + filemap_invalidate_lock_shared(mapping); page_cache_ra_unbounded(ractl, nr_to_read, lookahead_size); + filemap_invalidate_unlock_shared(mapping); } /*