1 /* 2 * linux/fs/ext4/page-io.c 3 * 4 * This contains the new page_io functions for ext4 5 * 6 * Written by Theodore Ts'o, 2010. 7 */ 8 9 #include <linux/fs.h> 10 #include <linux/time.h> 11 #include <linux/highuid.h> 12 #include <linux/pagemap.h> 13 #include <linux/quotaops.h> 14 #include <linux/string.h> 15 #include <linux/buffer_head.h> 16 #include <linux/writeback.h> 17 #include <linux/pagevec.h> 18 #include <linux/mpage.h> 19 #include <linux/namei.h> 20 #include <linux/uio.h> 21 #include <linux/bio.h> 22 #include <linux/workqueue.h> 23 #include <linux/kernel.h> 24 #include <linux/slab.h> 25 #include <linux/mm.h> 26 #include <linux/backing-dev.h> 27 28 #include "ext4_jbd2.h" 29 #include "xattr.h" 30 #include "acl.h" 31 32 static struct kmem_cache *io_end_cachep; 33 34 int __init ext4_init_pageio(void) 35 { 36 io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT); 37 if (io_end_cachep == NULL) 38 return -ENOMEM; 39 return 0; 40 } 41 42 void ext4_exit_pageio(void) 43 { 44 kmem_cache_destroy(io_end_cachep); 45 } 46 47 /* 48 * Print an buffer I/O error compatible with the fs/buffer.c. This 49 * provides compatibility with dmesg scrapers that look for a specific 50 * buffer I/O error message. We really need a unified error reporting 51 * structure to userspace ala Digital Unix's uerf system, but it's 52 * probably not going to happen in my lifetime, due to LKML politics... 53 */ 54 static void buffer_io_error(struct buffer_head *bh) 55 { 56 printk_ratelimited(KERN_ERR "Buffer I/O error on device %pg, logical block %llu\n", 57 bh->b_bdev, 58 (unsigned long long)bh->b_blocknr); 59 } 60 61 static void ext4_finish_bio(struct bio *bio) 62 { 63 int i; 64 struct bio_vec *bvec; 65 66 bio_for_each_segment_all(bvec, bio, i) { 67 struct page *page = bvec->bv_page; 68 #ifdef CONFIG_EXT4_FS_ENCRYPTION 69 struct page *data_page = NULL; 70 #endif 71 struct buffer_head *bh, *head; 72 unsigned bio_start = bvec->bv_offset; 73 unsigned bio_end = bio_start + bvec->bv_len; 74 unsigned under_io = 0; 75 unsigned long flags; 76 77 if (!page) 78 continue; 79 80 #ifdef CONFIG_EXT4_FS_ENCRYPTION 81 if (!page->mapping) { 82 /* The bounce data pages are unmapped. */ 83 data_page = page; 84 fscrypt_pullback_bio_page(&page, false); 85 } 86 #endif 87 88 if (bio->bi_status) { 89 SetPageError(page); 90 mapping_set_error(page->mapping, -EIO); 91 } 92 bh = head = page_buffers(page); 93 /* 94 * We check all buffers in the page under BH_Uptodate_Lock 95 * to avoid races with other end io clearing async_write flags 96 */ 97 local_irq_save(flags); 98 bit_spin_lock(BH_Uptodate_Lock, &head->b_state); 99 do { 100 if (bh_offset(bh) < bio_start || 101 bh_offset(bh) + bh->b_size > bio_end) { 102 if (buffer_async_write(bh)) 103 under_io++; 104 continue; 105 } 106 clear_buffer_async_write(bh); 107 if (bio->bi_status) 108 buffer_io_error(bh); 109 } while ((bh = bh->b_this_page) != head); 110 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); 111 local_irq_restore(flags); 112 if (!under_io) { 113 #ifdef CONFIG_EXT4_FS_ENCRYPTION 114 if (data_page) 115 fscrypt_restore_control_page(data_page); 116 #endif 117 end_page_writeback(page); 118 } 119 } 120 } 121 122 static void ext4_release_io_end(ext4_io_end_t *io_end) 123 { 124 struct bio *bio, *next_bio; 125 126 BUG_ON(!list_empty(&io_end->list)); 127 BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN); 128 WARN_ON(io_end->handle); 129 130 for (bio = io_end->bio; bio; bio = next_bio) { 131 next_bio = bio->bi_private; 132 ext4_finish_bio(bio); 133 bio_put(bio); 134 } 135 kmem_cache_free(io_end_cachep, io_end); 136 } 137 138 /* 139 * Check a range of space and convert unwritten extents to written. Note that 140 * we are protected from truncate touching same part of extent tree by the 141 * fact that truncate code waits for all DIO to finish (thus exclusion from 142 * direct IO is achieved) and also waits for PageWriteback bits. Thus we 143 * cannot get to ext4_ext_truncate() before all IOs overlapping that range are 144 * completed (happens from ext4_free_ioend()). 145 */ 146 static int ext4_end_io(ext4_io_end_t *io) 147 { 148 struct inode *inode = io->inode; 149 loff_t offset = io->offset; 150 ssize_t size = io->size; 151 handle_t *handle = io->handle; 152 int ret = 0; 153 154 ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," 155 "list->prev 0x%p\n", 156 io, inode->i_ino, io->list.next, io->list.prev); 157 158 io->handle = NULL; /* Following call will use up the handle */ 159 ret = ext4_convert_unwritten_extents(handle, inode, offset, size); 160 if (ret < 0 && !ext4_forced_shutdown(EXT4_SB(inode->i_sb))) { 161 ext4_msg(inode->i_sb, KERN_EMERG, 162 "failed to convert unwritten extents to written " 163 "extents -- potential data loss! " 164 "(inode %lu, offset %llu, size %zd, error %d)", 165 inode->i_ino, offset, size, ret); 166 } 167 ext4_clear_io_unwritten_flag(io); 168 ext4_release_io_end(io); 169 return ret; 170 } 171 172 static void dump_completed_IO(struct inode *inode, struct list_head *head) 173 { 174 #ifdef EXT4FS_DEBUG 175 struct list_head *cur, *before, *after; 176 ext4_io_end_t *io, *io0, *io1; 177 178 if (list_empty(head)) 179 return; 180 181 ext4_debug("Dump inode %lu completed io list\n", inode->i_ino); 182 list_for_each_entry(io, head, list) { 183 cur = &io->list; 184 before = cur->prev; 185 io0 = container_of(before, ext4_io_end_t, list); 186 after = cur->next; 187 io1 = container_of(after, ext4_io_end_t, list); 188 189 ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n", 190 io, inode->i_ino, io0, io1); 191 } 192 #endif 193 } 194 195 /* Add the io_end to per-inode completed end_io list. */ 196 static void ext4_add_complete_io(ext4_io_end_t *io_end) 197 { 198 struct ext4_inode_info *ei = EXT4_I(io_end->inode); 199 struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb); 200 struct workqueue_struct *wq; 201 unsigned long flags; 202 203 /* Only reserved conversions from writeback should enter here */ 204 WARN_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN)); 205 WARN_ON(!io_end->handle && sbi->s_journal); 206 spin_lock_irqsave(&ei->i_completed_io_lock, flags); 207 wq = sbi->rsv_conversion_wq; 208 if (list_empty(&ei->i_rsv_conversion_list)) 209 queue_work(wq, &ei->i_rsv_conversion_work); 210 list_add_tail(&io_end->list, &ei->i_rsv_conversion_list); 211 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 212 } 213 214 static int ext4_do_flush_completed_IO(struct inode *inode, 215 struct list_head *head) 216 { 217 ext4_io_end_t *io; 218 struct list_head unwritten; 219 unsigned long flags; 220 struct ext4_inode_info *ei = EXT4_I(inode); 221 int err, ret = 0; 222 223 spin_lock_irqsave(&ei->i_completed_io_lock, flags); 224 dump_completed_IO(inode, head); 225 list_replace_init(head, &unwritten); 226 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 227 228 while (!list_empty(&unwritten)) { 229 io = list_entry(unwritten.next, ext4_io_end_t, list); 230 BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN)); 231 list_del_init(&io->list); 232 233 err = ext4_end_io(io); 234 if (unlikely(!ret && err)) 235 ret = err; 236 } 237 return ret; 238 } 239 240 /* 241 * work on completed IO, to convert unwritten extents to extents 242 */ 243 void ext4_end_io_rsv_work(struct work_struct *work) 244 { 245 struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info, 246 i_rsv_conversion_work); 247 ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list); 248 } 249 250 ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags) 251 { 252 ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags); 253 if (io) { 254 io->inode = inode; 255 INIT_LIST_HEAD(&io->list); 256 atomic_set(&io->count, 1); 257 } 258 return io; 259 } 260 261 void ext4_put_io_end_defer(ext4_io_end_t *io_end) 262 { 263 if (atomic_dec_and_test(&io_end->count)) { 264 if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) || !io_end->size) { 265 ext4_release_io_end(io_end); 266 return; 267 } 268 ext4_add_complete_io(io_end); 269 } 270 } 271 272 int ext4_put_io_end(ext4_io_end_t *io_end) 273 { 274 int err = 0; 275 276 if (atomic_dec_and_test(&io_end->count)) { 277 if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 278 err = ext4_convert_unwritten_extents(io_end->handle, 279 io_end->inode, io_end->offset, 280 io_end->size); 281 io_end->handle = NULL; 282 ext4_clear_io_unwritten_flag(io_end); 283 } 284 ext4_release_io_end(io_end); 285 } 286 return err; 287 } 288 289 ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end) 290 { 291 atomic_inc(&io_end->count); 292 return io_end; 293 } 294 295 /* BIO completion function for page writeback */ 296 static void ext4_end_bio(struct bio *bio) 297 { 298 ext4_io_end_t *io_end = bio->bi_private; 299 sector_t bi_sector = bio->bi_iter.bi_sector; 300 char b[BDEVNAME_SIZE]; 301 302 if (WARN_ONCE(!io_end, "io_end is NULL: %s: sector %Lu len %u err %d\n", 303 bdevname(bio->bi_bdev, b), 304 (long long) bio->bi_iter.bi_sector, 305 (unsigned) bio_sectors(bio), 306 bio->bi_status)) { 307 ext4_finish_bio(bio); 308 bio_put(bio); 309 return; 310 } 311 bio->bi_end_io = NULL; 312 313 if (bio->bi_status) { 314 struct inode *inode = io_end->inode; 315 316 ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu " 317 "(offset %llu size %ld starting block %llu)", 318 bio->bi_status, inode->i_ino, 319 (unsigned long long) io_end->offset, 320 (long) io_end->size, 321 (unsigned long long) 322 bi_sector >> (inode->i_blkbits - 9)); 323 mapping_set_error(inode->i_mapping, 324 blk_status_to_errno(bio->bi_status)); 325 } 326 327 if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 328 /* 329 * Link bio into list hanging from io_end. We have to do it 330 * atomically as bio completions can be racing against each 331 * other. 332 */ 333 bio->bi_private = xchg(&io_end->bio, bio); 334 ext4_put_io_end_defer(io_end); 335 } else { 336 /* 337 * Drop io_end reference early. Inode can get freed once 338 * we finish the bio. 339 */ 340 ext4_put_io_end_defer(io_end); 341 ext4_finish_bio(bio); 342 bio_put(bio); 343 } 344 } 345 346 void ext4_io_submit(struct ext4_io_submit *io) 347 { 348 struct bio *bio = io->io_bio; 349 350 if (bio) { 351 int io_op_flags = io->io_wbc->sync_mode == WB_SYNC_ALL ? 352 REQ_SYNC : 0; 353 io->io_bio->bi_write_hint = io->io_end->inode->i_write_hint; 354 bio_set_op_attrs(io->io_bio, REQ_OP_WRITE, io_op_flags); 355 submit_bio(io->io_bio); 356 } 357 io->io_bio = NULL; 358 } 359 360 void ext4_io_submit_init(struct ext4_io_submit *io, 361 struct writeback_control *wbc) 362 { 363 io->io_wbc = wbc; 364 io->io_bio = NULL; 365 io->io_end = NULL; 366 } 367 368 static int io_submit_init_bio(struct ext4_io_submit *io, 369 struct buffer_head *bh) 370 { 371 struct bio *bio; 372 373 bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); 374 if (!bio) 375 return -ENOMEM; 376 wbc_init_bio(io->io_wbc, bio); 377 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 378 bio->bi_bdev = bh->b_bdev; 379 bio->bi_end_io = ext4_end_bio; 380 bio->bi_private = ext4_get_io_end(io->io_end); 381 io->io_bio = bio; 382 io->io_next_block = bh->b_blocknr; 383 return 0; 384 } 385 386 static int io_submit_add_bh(struct ext4_io_submit *io, 387 struct inode *inode, 388 struct page *page, 389 struct buffer_head *bh) 390 { 391 int ret; 392 393 if (io->io_bio && bh->b_blocknr != io->io_next_block) { 394 submit_and_retry: 395 ext4_io_submit(io); 396 } 397 if (io->io_bio == NULL) { 398 ret = io_submit_init_bio(io, bh); 399 if (ret) 400 return ret; 401 io->io_bio->bi_write_hint = inode->i_write_hint; 402 } 403 ret = bio_add_page(io->io_bio, page, bh->b_size, bh_offset(bh)); 404 if (ret != bh->b_size) 405 goto submit_and_retry; 406 wbc_account_io(io->io_wbc, page, bh->b_size); 407 io->io_next_block++; 408 return 0; 409 } 410 411 int ext4_bio_write_page(struct ext4_io_submit *io, 412 struct page *page, 413 int len, 414 struct writeback_control *wbc, 415 bool keep_towrite) 416 { 417 struct page *data_page = NULL; 418 struct inode *inode = page->mapping->host; 419 unsigned block_start; 420 struct buffer_head *bh, *head; 421 int ret = 0; 422 int nr_submitted = 0; 423 int nr_to_submit = 0; 424 425 BUG_ON(!PageLocked(page)); 426 BUG_ON(PageWriteback(page)); 427 428 if (keep_towrite) 429 set_page_writeback_keepwrite(page); 430 else 431 set_page_writeback(page); 432 ClearPageError(page); 433 434 /* 435 * Comments copied from block_write_full_page: 436 * 437 * The page straddles i_size. It must be zeroed out on each and every 438 * writepage invocation because it may be mmapped. "A file is mapped 439 * in multiples of the page size. For a file that is not a multiple of 440 * the page size, the remaining memory is zeroed when mapped, and 441 * writes to that region are not written out to the file." 442 */ 443 if (len < PAGE_SIZE) 444 zero_user_segment(page, len, PAGE_SIZE); 445 /* 446 * In the first loop we prepare and mark buffers to submit. We have to 447 * mark all buffers in the page before submitting so that 448 * end_page_writeback() cannot be called from ext4_bio_end_io() when IO 449 * on the first buffer finishes and we are still working on submitting 450 * the second buffer. 451 */ 452 bh = head = page_buffers(page); 453 do { 454 block_start = bh_offset(bh); 455 if (block_start >= len) { 456 clear_buffer_dirty(bh); 457 set_buffer_uptodate(bh); 458 continue; 459 } 460 if (!buffer_dirty(bh) || buffer_delay(bh) || 461 !buffer_mapped(bh) || buffer_unwritten(bh)) { 462 /* A hole? We can safely clear the dirty bit */ 463 if (!buffer_mapped(bh)) 464 clear_buffer_dirty(bh); 465 if (io->io_bio) 466 ext4_io_submit(io); 467 continue; 468 } 469 if (buffer_new(bh)) { 470 clear_buffer_new(bh); 471 clean_bdev_bh_alias(bh); 472 } 473 set_buffer_async_write(bh); 474 nr_to_submit++; 475 } while ((bh = bh->b_this_page) != head); 476 477 bh = head = page_buffers(page); 478 479 if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode) && 480 nr_to_submit) { 481 gfp_t gfp_flags = GFP_NOFS; 482 483 retry_encrypt: 484 data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0, 485 page->index, gfp_flags); 486 if (IS_ERR(data_page)) { 487 ret = PTR_ERR(data_page); 488 if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) { 489 if (io->io_bio) { 490 ext4_io_submit(io); 491 congestion_wait(BLK_RW_ASYNC, HZ/50); 492 } 493 gfp_flags |= __GFP_NOFAIL; 494 goto retry_encrypt; 495 } 496 data_page = NULL; 497 goto out; 498 } 499 } 500 501 /* Now submit buffers to write */ 502 do { 503 if (!buffer_async_write(bh)) 504 continue; 505 ret = io_submit_add_bh(io, inode, 506 data_page ? data_page : page, bh); 507 if (ret) { 508 /* 509 * We only get here on ENOMEM. Not much else 510 * we can do but mark the page as dirty, and 511 * better luck next time. 512 */ 513 break; 514 } 515 nr_submitted++; 516 clear_buffer_dirty(bh); 517 } while ((bh = bh->b_this_page) != head); 518 519 /* Error stopped previous loop? Clean up buffers... */ 520 if (ret) { 521 out: 522 if (data_page) 523 fscrypt_restore_control_page(data_page); 524 printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret); 525 redirty_page_for_writepage(wbc, page); 526 do { 527 clear_buffer_async_write(bh); 528 bh = bh->b_this_page; 529 } while (bh != head); 530 } 531 unlock_page(page); 532 /* Nothing submitted - we have to end page writeback */ 533 if (!nr_submitted) 534 end_page_writeback(page); 535 return ret; 536 } 537