1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/fs/ext4/readpage.c 4 * 5 * Copyright (C) 2002, Linus Torvalds. 6 * Copyright (C) 2015, Google, Inc. 7 * 8 * This was originally taken from fs/mpage.c 9 * 10 * The ext4_mpage_readpages() function here is intended to 11 * replace mpage_readahead() in the general case, not just for 12 * encrypted files. It has some limitations (see below), where it 13 * will fall back to read_block_full_page(), but these limitations 14 * should only be hit when page_size != block_size. 15 * 16 * This will allow us to attach a callback function to support ext4 17 * encryption. 18 * 19 * If anything unusual happens, such as: 20 * 21 * - encountering a page which has buffers 22 * - encountering a page which has a non-hole after a hole 23 * - encountering a page with non-contiguous blocks 24 * 25 * then this code just gives up and calls the buffer_head-based read function. 26 * It does handle a page which has holes at the end - that is a common case: 27 * the end-of-file on blocksize < PAGE_SIZE setups. 28 * 29 */ 30 31 #include <linux/kernel.h> 32 #include <linux/export.h> 33 #include <linux/mm.h> 34 #include <linux/kdev_t.h> 35 #include <linux/gfp.h> 36 #include <linux/bio.h> 37 #include <linux/fs.h> 38 #include <linux/buffer_head.h> 39 #include <linux/blk-crypto.h> 40 #include <linux/blkdev.h> 41 #include <linux/highmem.h> 42 #include <linux/prefetch.h> 43 #include <linux/mpage.h> 44 #include <linux/writeback.h> 45 #include <linux/backing-dev.h> 46 47 #include "ext4.h" 48 #include <trace/events/ext4.h> 49 50 #define NUM_VERITY_WORKS 128 51 52 static struct kmem_cache *ext4_verity_work_cache; 53 static mempool_t *ext4_verity_work_pool; 54 55 struct ext4_verity_work { 56 struct bio *bio; 57 struct fsverity_info *vi; 58 struct work_struct work; 59 }; 60 61 static void __read_end_io(struct bio *bio) 62 { 63 struct folio_iter fi; 64 65 bio_for_each_folio_all(fi, bio) 66 folio_end_read(fi.folio, bio->bi_status == 0); 67 if (bio->bi_private) 68 mempool_free(bio->bi_private, ext4_verity_work_pool); 69 bio_put(bio); 70 } 71 72 static void verity_work(struct work_struct *work) 73 { 74 struct ext4_verity_work *ctx = 75 container_of(work, struct ext4_verity_work, work); 76 struct bio *bio = ctx->bio; 77 struct fsverity_info *vi = ctx->vi; 78 79 /* 80 * Free the ext4_verity_work right away, since it's no longer needed. 81 * This relieves the pressure on the mempool as much as possible. 82 */ 83 mempool_free(ctx, ext4_verity_work_pool); 84 bio->bi_private = NULL; 85 86 fsverity_verify_bio(vi, bio); 87 88 __read_end_io(bio); 89 } 90 91 /* 92 * I/O completion handler for multipage BIOs. 93 * 94 * The mpage code never puts partial pages into a BIO (except for end-of-file). 95 * If a page does not map to a contiguous run of blocks then it simply falls 96 * back to block_read_full_folio(). 97 * 98 * Why is this? If a page's completion depends on a number of different BIOs 99 * which can complete in any order (or at the same time) then determining the 100 * status of that page is hard. See end_buffer_async_read() for the details. 101 * There is no point in duplicating all that complexity. 102 */ 103 static void mpage_end_io(struct bio *bio) 104 { 105 if (IS_ENABLED(CONFIG_FS_VERITY) && bio->bi_private && 106 !bio->bi_status) { 107 struct ext4_verity_work *ctx = bio->bi_private; 108 109 INIT_WORK(&ctx->work, verity_work); 110 fsverity_enqueue_verify_work(&ctx->work); 111 return; 112 } 113 __read_end_io(bio); 114 } 115 116 static void ext4_set_verity_work(struct bio *bio, struct fsverity_info *vi) 117 { 118 if (vi) { 119 /* Due to the mempool, this never fails. */ 120 struct ext4_verity_work *ctx = 121 mempool_alloc(ext4_verity_work_pool, GFP_NOFS); 122 123 ctx->bio = bio; 124 ctx->vi = vi; 125 bio->bi_private = ctx; 126 } 127 } 128 129 static inline loff_t ext4_readpage_limit(struct inode *inode) 130 { 131 if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) 132 return inode->i_sb->s_maxbytes; 133 134 return i_size_read(inode); 135 } 136 137 static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi, 138 struct readahead_control *rac, struct folio *folio) 139 { 140 struct bio *bio = NULL; 141 sector_t last_block_in_bio = 0; 142 const unsigned blkbits = inode->i_blkbits; 143 const unsigned blocksize = 1 << blkbits; 144 sector_t block_in_file; 145 sector_t last_block; 146 sector_t last_block_in_file; 147 sector_t first_block; 148 loff_t pos; 149 unsigned page_block; 150 struct block_device *bdev = inode->i_sb->s_bdev; 151 int length; 152 unsigned relative_block = 0; 153 struct ext4_map_blocks map; 154 unsigned int nr_pages, folio_pages; 155 156 map.m_pblk = 0; 157 map.m_lblk = 0; 158 map.m_len = 0; 159 map.m_flags = 0; 160 161 nr_pages = rac ? readahead_count(rac) : folio_nr_pages(folio); 162 for (; nr_pages; nr_pages -= folio_pages) { 163 int fully_mapped = 1; 164 unsigned int first_hole; 165 unsigned int blocks_per_folio; 166 167 if (rac) 168 folio = readahead_folio(rac); 169 170 folio_pages = folio_nr_pages(folio); 171 prefetchw(&folio->flags); 172 173 if (folio_buffers(folio)) 174 goto confused; 175 176 blocks_per_folio = folio_size(folio) >> blkbits; 177 first_hole = blocks_per_folio; 178 pos = folio_pos(folio); 179 block_in_file = pos >> blkbits; 180 last_block = EXT4_PG_TO_LBLK(inode, folio->index + nr_pages); 181 last_block_in_file = (ext4_readpage_limit(inode) + 182 blocksize - 1) >> blkbits; 183 if (last_block > last_block_in_file) 184 last_block = last_block_in_file; 185 page_block = 0; 186 187 /* 188 * Map blocks using the previous result first. 189 */ 190 if ((map.m_flags & EXT4_MAP_MAPPED) && 191 block_in_file > map.m_lblk && 192 block_in_file < (map.m_lblk + map.m_len)) { 193 unsigned map_offset = block_in_file - map.m_lblk; 194 unsigned last = map.m_len - map_offset; 195 196 first_block = map.m_pblk + map_offset; 197 for (relative_block = 0; ; relative_block++) { 198 if (relative_block == last) { 199 /* needed? */ 200 map.m_flags &= ~EXT4_MAP_MAPPED; 201 break; 202 } 203 if (page_block == blocks_per_folio) 204 break; 205 page_block++; 206 block_in_file++; 207 } 208 } 209 210 /* 211 * Then do more ext4_map_blocks() calls until we are 212 * done with this folio. 213 */ 214 while (page_block < blocks_per_folio) { 215 if (block_in_file < last_block) { 216 map.m_lblk = block_in_file; 217 map.m_len = last_block - block_in_file; 218 219 if (ext4_map_blocks(NULL, inode, &map, 0) < 0) { 220 set_error_page: 221 folio_zero_segment(folio, 0, 222 folio_size(folio)); 223 folio_unlock(folio); 224 goto next_page; 225 } 226 } 227 if ((map.m_flags & EXT4_MAP_MAPPED) == 0) { 228 fully_mapped = 0; 229 if (first_hole == blocks_per_folio) 230 first_hole = page_block; 231 page_block++; 232 block_in_file++; 233 continue; 234 } 235 if (first_hole != blocks_per_folio) 236 goto confused; /* hole -> non-hole */ 237 238 /* Contiguous blocks? */ 239 if (!page_block) 240 first_block = map.m_pblk; 241 else if (first_block + page_block != map.m_pblk) 242 goto confused; 243 for (relative_block = 0; ; relative_block++) { 244 if (relative_block == map.m_len) { 245 /* needed? */ 246 map.m_flags &= ~EXT4_MAP_MAPPED; 247 break; 248 } else if (page_block == blocks_per_folio) 249 break; 250 page_block++; 251 block_in_file++; 252 } 253 } 254 if (first_hole != blocks_per_folio) { 255 folio_zero_segment(folio, first_hole << blkbits, 256 folio_size(folio)); 257 if (first_hole == 0) { 258 if (vi && !fsverity_verify_folio(vi, folio)) 259 goto set_error_page; 260 folio_end_read(folio, true); 261 continue; 262 } 263 } else if (fully_mapped) { 264 folio_set_mappedtodisk(folio); 265 } 266 267 /* 268 * This folio will go to BIO. Do we need to send this 269 * BIO off first? 270 */ 271 if (bio && (last_block_in_bio != first_block - 1 || 272 !fscrypt_mergeable_bio(bio, inode, pos))) { 273 submit_and_realloc: 274 blk_crypto_submit_bio(bio); 275 bio = NULL; 276 } 277 if (bio == NULL) { 278 /* 279 * bio_alloc will _always_ be able to allocate a bio if 280 * __GFP_DIRECT_RECLAIM is set, see bio_alloc_bioset(). 281 */ 282 bio = bio_alloc(bdev, bio_max_segs(nr_pages), 283 REQ_OP_READ, GFP_KERNEL); 284 fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_KERNEL); 285 ext4_set_verity_work(bio, vi); 286 bio->bi_iter.bi_sector = first_block << (blkbits - 9); 287 bio->bi_end_io = mpage_end_io; 288 if (rac) 289 bio->bi_opf |= REQ_RAHEAD; 290 } 291 292 length = first_hole << blkbits; 293 if (!bio_add_folio(bio, folio, length, 0)) 294 goto submit_and_realloc; 295 296 if (((map.m_flags & EXT4_MAP_BOUNDARY) && 297 (relative_block == map.m_len)) || 298 (first_hole != blocks_per_folio)) { 299 blk_crypto_submit_bio(bio); 300 bio = NULL; 301 } else 302 last_block_in_bio = first_block + blocks_per_folio - 1; 303 continue; 304 confused: 305 if (bio) { 306 blk_crypto_submit_bio(bio); 307 bio = NULL; 308 } 309 if (!folio_test_uptodate(folio)) 310 block_read_full_folio(folio, ext4_get_block); 311 else 312 folio_unlock(folio); 313 next_page: 314 ; /* A label shall be followed by a statement until C23 */ 315 } 316 if (bio) 317 blk_crypto_submit_bio(bio); 318 return 0; 319 } 320 321 int ext4_read_folio(struct file *file, struct folio *folio) 322 { 323 struct inode *inode = folio->mapping->host; 324 struct fsverity_info *vi = NULL; 325 int ret; 326 327 trace_ext4_read_folio(inode, folio); 328 329 if (ext4_has_inline_data(inode)) { 330 ret = ext4_readpage_inline(inode, folio); 331 if (ret != -EAGAIN) 332 return ret; 333 } 334 335 if (folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) 336 vi = fsverity_get_info(inode); 337 if (vi) 338 fsverity_readahead(vi, folio->index, folio_nr_pages(folio)); 339 return ext4_mpage_readpages(inode, vi, NULL, folio); 340 } 341 342 void ext4_readahead(struct readahead_control *rac) 343 { 344 struct inode *inode = rac->mapping->host; 345 struct fsverity_info *vi = NULL; 346 347 /* If the file has inline data, no need to do readahead. */ 348 if (ext4_has_inline_data(inode)) 349 return; 350 351 if (readahead_index(rac) < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) 352 vi = fsverity_get_info(inode); 353 if (vi) 354 fsverity_readahead(vi, readahead_index(rac), 355 readahead_count(rac)); 356 ext4_mpage_readpages(inode, vi, rac, NULL); 357 } 358 359 int __init ext4_init_verity_caches(void) 360 { 361 if (!IS_ENABLED(CONFIG_FS_VERITY)) 362 return 0; 363 ext4_verity_work_cache = 364 KMEM_CACHE(ext4_verity_work, SLAB_RECLAIM_ACCOUNT); 365 366 if (!ext4_verity_work_cache) 367 goto fail; 368 ext4_verity_work_pool = mempool_create_slab_pool( 369 NUM_VERITY_WORKS, ext4_verity_work_cache); 370 if (!ext4_verity_work_pool) 371 goto fail_free_cache; 372 return 0; 373 374 fail_free_cache: 375 kmem_cache_destroy(ext4_verity_work_cache); 376 fail: 377 return -ENOMEM; 378 } 379 380 void ext4_exit_verity_caches(void) 381 { 382 if (!IS_ENABLED(CONFIG_FS_VERITY)) 383 return; 384 mempool_destroy(ext4_verity_work_pool); 385 kmem_cache_destroy(ext4_verity_work_cache); 386 } 387