1bd2d0210STheodore Ts'o /* 2bd2d0210STheodore Ts'o * linux/fs/ext4/page-io.c 3bd2d0210STheodore Ts'o * 4bd2d0210STheodore Ts'o * This contains the new page_io functions for ext4 5bd2d0210STheodore Ts'o * 6bd2d0210STheodore Ts'o * Written by Theodore Ts'o, 2010. 7bd2d0210STheodore Ts'o */ 8bd2d0210STheodore Ts'o 9bd2d0210STheodore Ts'o #include <linux/fs.h> 10bd2d0210STheodore Ts'o #include <linux/time.h> 11bd2d0210STheodore Ts'o #include <linux/highuid.h> 12bd2d0210STheodore Ts'o #include <linux/pagemap.h> 13bd2d0210STheodore Ts'o #include <linux/quotaops.h> 14bd2d0210STheodore Ts'o #include <linux/string.h> 15bd2d0210STheodore Ts'o #include <linux/buffer_head.h> 16bd2d0210STheodore Ts'o #include <linux/writeback.h> 17bd2d0210STheodore Ts'o #include <linux/pagevec.h> 18bd2d0210STheodore Ts'o #include <linux/mpage.h> 19bd2d0210STheodore Ts'o #include <linux/namei.h> 20bd2d0210STheodore Ts'o #include <linux/uio.h> 21bd2d0210STheodore Ts'o #include <linux/bio.h> 22bd2d0210STheodore Ts'o #include <linux/workqueue.h> 23bd2d0210STheodore Ts'o #include <linux/kernel.h> 24bd2d0210STheodore Ts'o #include <linux/slab.h> 251ae48a63SJan Kara #include <linux/mm.h> 26c9af28fdSTheodore Ts'o #include <linux/backing-dev.h> 27bd2d0210STheodore Ts'o 28bd2d0210STheodore Ts'o #include "ext4_jbd2.h" 29bd2d0210STheodore Ts'o #include "xattr.h" 30bd2d0210STheodore Ts'o #include "acl.h" 31bd2d0210STheodore Ts'o 320058f965SJan Kara static struct kmem_cache *io_end_cachep; 33bd2d0210STheodore Ts'o 345dabfc78STheodore Ts'o int __init ext4_init_pageio(void) 35bd2d0210STheodore Ts'o { 36bd2d0210STheodore Ts'o io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT); 370058f965SJan Kara if (io_end_cachep == NULL) 38bd2d0210STheodore Ts'o return -ENOMEM; 39bd2d0210STheodore Ts'o return 0; 40bd2d0210STheodore Ts'o } 41bd2d0210STheodore Ts'o 425dabfc78STheodore Ts'o void ext4_exit_pageio(void) 43bd2d0210STheodore Ts'o { 44bd2d0210STheodore Ts'o kmem_cache_destroy(io_end_cachep); 45bd2d0210STheodore Ts'o } 46bd2d0210STheodore Ts'o 471ada47d9STheodore Ts'o /* 48b0857d30SJan Kara * Print an buffer I/O error compatible with the fs/buffer.c. This 49b0857d30SJan Kara * provides compatibility with dmesg scrapers that look for a specific 50b0857d30SJan Kara * buffer I/O error message. We really need a unified error reporting 51b0857d30SJan Kara * structure to userspace ala Digital Unix's uerf system, but it's 52b0857d30SJan Kara * probably not going to happen in my lifetime, due to LKML politics... 53b0857d30SJan Kara */ 54b0857d30SJan Kara static void buffer_io_error(struct buffer_head *bh) 55b0857d30SJan Kara { 56a1c6f057SDmitry Monakhov printk_ratelimited(KERN_ERR "Buffer I/O error on device %pg, logical block %llu\n", 57a1c6f057SDmitry Monakhov bh->b_bdev, 58b0857d30SJan Kara (unsigned long long)bh->b_blocknr); 59b0857d30SJan Kara } 60b0857d30SJan Kara 61b0857d30SJan Kara static void ext4_finish_bio(struct bio *bio) 62b0857d30SJan Kara { 63b0857d30SJan Kara int i; 642c30c71bSKent Overstreet struct bio_vec *bvec; 65b0857d30SJan Kara 662c30c71bSKent Overstreet bio_for_each_segment_all(bvec, bio, i) { 67b0857d30SJan Kara struct page *page = bvec->bv_page; 682058f83aSMichael Halcrow #ifdef CONFIG_EXT4_FS_ENCRYPTION 692058f83aSMichael Halcrow struct page *data_page = NULL; 702058f83aSMichael Halcrow #endif 71b0857d30SJan Kara struct buffer_head *bh, *head; 72b0857d30SJan Kara unsigned bio_start = bvec->bv_offset; 73b0857d30SJan Kara unsigned bio_end = bio_start + bvec->bv_len; 74b0857d30SJan Kara unsigned under_io = 0; 75b0857d30SJan Kara unsigned long flags; 76b0857d30SJan Kara 77b0857d30SJan Kara if (!page) 78b0857d30SJan Kara continue; 79b0857d30SJan Kara 802058f83aSMichael Halcrow #ifdef CONFIG_EXT4_FS_ENCRYPTION 812058f83aSMichael Halcrow if (!page->mapping) { 822058f83aSMichael Halcrow /* The bounce data pages are unmapped. */ 832058f83aSMichael Halcrow data_page = page; 84a7550b30SJaegeuk Kim fscrypt_pullback_bio_page(&page, false); 852058f83aSMichael Halcrow } 862058f83aSMichael Halcrow #endif 872058f83aSMichael Halcrow 88*4e4cbee9SChristoph Hellwig if (bio->bi_status) { 89b0857d30SJan Kara SetPageError(page); 905114a97aSMichal Hocko mapping_set_error(page->mapping, -EIO); 91b0857d30SJan Kara } 92b0857d30SJan Kara bh = head = page_buffers(page); 93b0857d30SJan Kara /* 94b0857d30SJan Kara * We check all buffers in the page under BH_Uptodate_Lock 95b0857d30SJan Kara * to avoid races with other end io clearing async_write flags 96b0857d30SJan Kara */ 97b0857d30SJan Kara local_irq_save(flags); 98b0857d30SJan Kara bit_spin_lock(BH_Uptodate_Lock, &head->b_state); 99b0857d30SJan Kara do { 100b0857d30SJan Kara if (bh_offset(bh) < bio_start || 101b0857d30SJan Kara bh_offset(bh) + bh->b_size > bio_end) { 102b0857d30SJan Kara if (buffer_async_write(bh)) 103b0857d30SJan Kara under_io++; 104b0857d30SJan Kara continue; 105b0857d30SJan Kara } 106b0857d30SJan Kara clear_buffer_async_write(bh); 107*4e4cbee9SChristoph Hellwig if (bio->bi_status) 108b0857d30SJan Kara buffer_io_error(bh); 109b0857d30SJan Kara } while ((bh = bh->b_this_page) != head); 110b0857d30SJan Kara bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); 111b0857d30SJan Kara local_irq_restore(flags); 1122058f83aSMichael Halcrow if (!under_io) { 1132058f83aSMichael Halcrow #ifdef CONFIG_EXT4_FS_ENCRYPTION 114a7550b30SJaegeuk Kim if (data_page) 115a7550b30SJaegeuk Kim fscrypt_restore_control_page(data_page); 1162058f83aSMichael Halcrow #endif 117b0857d30SJan Kara end_page_writeback(page); 118b0857d30SJan Kara } 119b0857d30SJan Kara } 1202058f83aSMichael Halcrow } 121b0857d30SJan Kara 12297a851edSJan Kara static void ext4_release_io_end(ext4_io_end_t *io_end) 123bd2d0210STheodore Ts'o { 124b0857d30SJan Kara struct bio *bio, *next_bio; 125b0857d30SJan Kara 12697a851edSJan Kara BUG_ON(!list_empty(&io_end->list)); 12797a851edSJan Kara BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN); 1286b523df4SJan Kara WARN_ON(io_end->handle); 129bd2d0210STheodore Ts'o 130b0857d30SJan Kara for (bio = io_end->bio; bio; bio = next_bio) { 131b0857d30SJan Kara next_bio = bio->bi_private; 132b0857d30SJan Kara ext4_finish_bio(bio); 133b0857d30SJan Kara bio_put(bio); 134b0857d30SJan Kara } 13597a851edSJan Kara kmem_cache_free(io_end_cachep, io_end); 13697a851edSJan Kara } 13797a851edSJan Kara 138a115f749SJan Kara /* 139a115f749SJan Kara * Check a range of space and convert unwritten extents to written. Note that 140a115f749SJan Kara * we are protected from truncate touching same part of extent tree by the 141a115f749SJan Kara * fact that truncate code waits for all DIO to finish (thus exclusion from 142a115f749SJan Kara * direct IO is achieved) and also waits for PageWriteback bits. Thus we 143a115f749SJan Kara * cannot get to ext4_ext_truncate() before all IOs overlapping that range are 144a115f749SJan Kara * completed (happens from ext4_free_ioend()). 145a115f749SJan Kara */ 14628a535f9SDmitry Monakhov static int ext4_end_io(ext4_io_end_t *io) 147bd2d0210STheodore Ts'o { 148bd2d0210STheodore Ts'o struct inode *inode = io->inode; 149bd2d0210STheodore Ts'o loff_t offset = io->offset; 150bd2d0210STheodore Ts'o ssize_t size = io->size; 1516b523df4SJan Kara handle_t *handle = io->handle; 152bd2d0210STheodore Ts'o int ret = 0; 153bd2d0210STheodore Ts'o 154bd2d0210STheodore Ts'o ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," 155bd2d0210STheodore Ts'o "list->prev 0x%p\n", 156bd2d0210STheodore Ts'o io, inode->i_ino, io->list.next, io->list.prev); 157bd2d0210STheodore Ts'o 1586b523df4SJan Kara io->handle = NULL; /* Following call will use up the handle */ 1596b523df4SJan Kara ret = ext4_convert_unwritten_extents(handle, inode, offset, size); 1600db1ff22STheodore Ts'o if (ret < 0 && !ext4_forced_shutdown(EXT4_SB(inode->i_sb))) { 161b82e384cSTheodore Ts'o ext4_msg(inode->i_sb, KERN_EMERG, 162b82e384cSTheodore Ts'o "failed to convert unwritten extents to written " 163b82e384cSTheodore Ts'o "extents -- potential data loss! " 164b82e384cSTheodore Ts'o "(inode %lu, offset %llu, size %zd, error %d)", 165b82e384cSTheodore Ts'o inode->i_ino, offset, size, ret); 166bd2d0210STheodore Ts'o } 16797a851edSJan Kara ext4_clear_io_unwritten_flag(io); 16897a851edSJan Kara ext4_release_io_end(io); 169bd2d0210STheodore Ts'o return ret; 170bd2d0210STheodore Ts'o } 171bd2d0210STheodore Ts'o 1722e8fa54eSJan Kara static void dump_completed_IO(struct inode *inode, struct list_head *head) 17328a535f9SDmitry Monakhov { 17428a535f9SDmitry Monakhov #ifdef EXT4FS_DEBUG 17528a535f9SDmitry Monakhov struct list_head *cur, *before, *after; 17628a535f9SDmitry Monakhov ext4_io_end_t *io, *io0, *io1; 17728a535f9SDmitry Monakhov 1782e8fa54eSJan Kara if (list_empty(head)) 17928a535f9SDmitry Monakhov return; 18028a535f9SDmitry Monakhov 1812e8fa54eSJan Kara ext4_debug("Dump inode %lu completed io list\n", inode->i_ino); 1822e8fa54eSJan Kara list_for_each_entry(io, head, list) { 18328a535f9SDmitry Monakhov cur = &io->list; 18428a535f9SDmitry Monakhov before = cur->prev; 18528a535f9SDmitry Monakhov io0 = container_of(before, ext4_io_end_t, list); 18628a535f9SDmitry Monakhov after = cur->next; 18728a535f9SDmitry Monakhov io1 = container_of(after, ext4_io_end_t, list); 18828a535f9SDmitry Monakhov 18928a535f9SDmitry Monakhov ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n", 19028a535f9SDmitry Monakhov io, inode->i_ino, io0, io1); 19128a535f9SDmitry Monakhov } 19228a535f9SDmitry Monakhov #endif 19328a535f9SDmitry Monakhov } 19428a535f9SDmitry Monakhov 19528a535f9SDmitry Monakhov /* Add the io_end to per-inode completed end_io list. */ 19697a851edSJan Kara static void ext4_add_complete_io(ext4_io_end_t *io_end) 19728a535f9SDmitry Monakhov { 19828a535f9SDmitry Monakhov struct ext4_inode_info *ei = EXT4_I(io_end->inode); 19978371a45SJan Kara struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb); 20028a535f9SDmitry Monakhov struct workqueue_struct *wq; 20128a535f9SDmitry Monakhov unsigned long flags; 20228a535f9SDmitry Monakhov 2037b7a8665SChristoph Hellwig /* Only reserved conversions from writeback should enter here */ 2047b7a8665SChristoph Hellwig WARN_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN)); 20578371a45SJan Kara WARN_ON(!io_end->handle && sbi->s_journal); 20628a535f9SDmitry Monakhov spin_lock_irqsave(&ei->i_completed_io_lock, flags); 20778371a45SJan Kara wq = sbi->rsv_conversion_wq; 2082e8fa54eSJan Kara if (list_empty(&ei->i_rsv_conversion_list)) 2092e8fa54eSJan Kara queue_work(wq, &ei->i_rsv_conversion_work); 2102e8fa54eSJan Kara list_add_tail(&io_end->list, &ei->i_rsv_conversion_list); 21128a535f9SDmitry Monakhov spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 21228a535f9SDmitry Monakhov } 21328a535f9SDmitry Monakhov 2142e8fa54eSJan Kara static int ext4_do_flush_completed_IO(struct inode *inode, 2152e8fa54eSJan Kara struct list_head *head) 21628a535f9SDmitry Monakhov { 21728a535f9SDmitry Monakhov ext4_io_end_t *io; 218002bd7faSJan Kara struct list_head unwritten; 21928a535f9SDmitry Monakhov unsigned long flags; 22028a535f9SDmitry Monakhov struct ext4_inode_info *ei = EXT4_I(inode); 22128a535f9SDmitry Monakhov int err, ret = 0; 22228a535f9SDmitry Monakhov 22328a535f9SDmitry Monakhov spin_lock_irqsave(&ei->i_completed_io_lock, flags); 2242e8fa54eSJan Kara dump_completed_IO(inode, head); 2252e8fa54eSJan Kara list_replace_init(head, &unwritten); 22628a535f9SDmitry Monakhov spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 22728a535f9SDmitry Monakhov 22828a535f9SDmitry Monakhov while (!list_empty(&unwritten)) { 22928a535f9SDmitry Monakhov io = list_entry(unwritten.next, ext4_io_end_t, list); 23028a535f9SDmitry Monakhov BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN)); 23128a535f9SDmitry Monakhov list_del_init(&io->list); 23228a535f9SDmitry Monakhov 23328a535f9SDmitry Monakhov err = ext4_end_io(io); 23428a535f9SDmitry Monakhov if (unlikely(!ret && err)) 23528a535f9SDmitry Monakhov ret = err; 23628a535f9SDmitry Monakhov } 23728a535f9SDmitry Monakhov return ret; 23828a535f9SDmitry Monakhov } 23928a535f9SDmitry Monakhov 240bd2d0210STheodore Ts'o /* 2412e8fa54eSJan Kara * work on completed IO, to convert unwritten extents to extents 242bd2d0210STheodore Ts'o */ 2432e8fa54eSJan Kara void ext4_end_io_rsv_work(struct work_struct *work) 244bd2d0210STheodore Ts'o { 24584c17543SJan Kara struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info, 2462e8fa54eSJan Kara i_rsv_conversion_work); 2472e8fa54eSJan Kara ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list); 2482e8fa54eSJan Kara } 2492e8fa54eSJan Kara 250bd2d0210STheodore Ts'o ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags) 251bd2d0210STheodore Ts'o { 252b17b35ecSJesper Juhl ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags); 253bd2d0210STheodore Ts'o if (io) { 254f7ad6d2eSTheodore Ts'o io->inode = inode; 255bd2d0210STheodore Ts'o INIT_LIST_HEAD(&io->list); 25697a851edSJan Kara atomic_set(&io->count, 1); 257bd2d0210STheodore Ts'o } 258bd2d0210STheodore Ts'o return io; 259bd2d0210STheodore Ts'o } 260bd2d0210STheodore Ts'o 26197a851edSJan Kara void ext4_put_io_end_defer(ext4_io_end_t *io_end) 26297a851edSJan Kara { 26397a851edSJan Kara if (atomic_dec_and_test(&io_end->count)) { 26497a851edSJan Kara if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) || !io_end->size) { 26597a851edSJan Kara ext4_release_io_end(io_end); 26697a851edSJan Kara return; 26797a851edSJan Kara } 26897a851edSJan Kara ext4_add_complete_io(io_end); 26997a851edSJan Kara } 27097a851edSJan Kara } 27197a851edSJan Kara 27297a851edSJan Kara int ext4_put_io_end(ext4_io_end_t *io_end) 27397a851edSJan Kara { 27497a851edSJan Kara int err = 0; 27597a851edSJan Kara 27697a851edSJan Kara if (atomic_dec_and_test(&io_end->count)) { 27797a851edSJan Kara if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 2786b523df4SJan Kara err = ext4_convert_unwritten_extents(io_end->handle, 2796b523df4SJan Kara io_end->inode, io_end->offset, 2806b523df4SJan Kara io_end->size); 2816b523df4SJan Kara io_end->handle = NULL; 28297a851edSJan Kara ext4_clear_io_unwritten_flag(io_end); 28397a851edSJan Kara } 28497a851edSJan Kara ext4_release_io_end(io_end); 28597a851edSJan Kara } 28697a851edSJan Kara return err; 28797a851edSJan Kara } 28897a851edSJan Kara 28997a851edSJan Kara ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end) 29097a851edSJan Kara { 29197a851edSJan Kara atomic_inc(&io_end->count); 29297a851edSJan Kara return io_end; 29397a851edSJan Kara } 29497a851edSJan Kara 295822dbba3SJan Kara /* BIO completion function for page writeback */ 2964246a0b6SChristoph Hellwig static void ext4_end_bio(struct bio *bio) 297bd2d0210STheodore Ts'o { 298bd2d0210STheodore Ts'o ext4_io_end_t *io_end = bio->bi_private; 2994f024f37SKent Overstreet sector_t bi_sector = bio->bi_iter.bi_sector; 30072d622b4STheodore Ts'o char b[BDEVNAME_SIZE]; 301bd2d0210STheodore Ts'o 30272d622b4STheodore Ts'o if (WARN_ONCE(!io_end, "io_end is NULL: %s: sector %Lu len %u err %d\n", 30372d622b4STheodore Ts'o bdevname(bio->bi_bdev, b), 30472d622b4STheodore Ts'o (long long) bio->bi_iter.bi_sector, 30572d622b4STheodore Ts'o (unsigned) bio_sectors(bio), 306*4e4cbee9SChristoph Hellwig bio->bi_status)) { 30772d622b4STheodore Ts'o ext4_finish_bio(bio); 30872d622b4STheodore Ts'o bio_put(bio); 30972d622b4STheodore Ts'o return; 31072d622b4STheodore Ts'o } 311bd2d0210STheodore Ts'o bio->bi_end_io = NULL; 3120058f965SJan Kara 313*4e4cbee9SChristoph Hellwig if (bio->bi_status) { 314b0857d30SJan Kara struct inode *inode = io_end->inode; 315b0857d30SJan Kara 3169503c67cSMatthew Wilcox ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu " 317f7ad6d2eSTheodore Ts'o "(offset %llu size %ld starting block %llu)", 318*4e4cbee9SChristoph Hellwig bio->bi_status, inode->i_ino, 319f7ad6d2eSTheodore Ts'o (unsigned long long) io_end->offset, 320f7ad6d2eSTheodore Ts'o (long) io_end->size, 321f7ad6d2eSTheodore Ts'o (unsigned long long) 322d50bdd5aSCurt Wohlgemuth bi_sector >> (inode->i_blkbits - 9)); 323*4e4cbee9SChristoph Hellwig mapping_set_error(inode->i_mapping, 324*4e4cbee9SChristoph Hellwig blk_status_to_errno(bio->bi_status)); 325f7ad6d2eSTheodore Ts'o } 326822dbba3SJan Kara 327822dbba3SJan Kara if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 328822dbba3SJan Kara /* 329822dbba3SJan Kara * Link bio into list hanging from io_end. We have to do it 330822dbba3SJan Kara * atomically as bio completions can be racing against each 331822dbba3SJan Kara * other. 332822dbba3SJan Kara */ 333822dbba3SJan Kara bio->bi_private = xchg(&io_end->bio, bio); 33497a851edSJan Kara ext4_put_io_end_defer(io_end); 335822dbba3SJan Kara } else { 336822dbba3SJan Kara /* 337822dbba3SJan Kara * Drop io_end reference early. Inode can get freed once 338822dbba3SJan Kara * we finish the bio. 339822dbba3SJan Kara */ 340822dbba3SJan Kara ext4_put_io_end_defer(io_end); 341822dbba3SJan Kara ext4_finish_bio(bio); 342822dbba3SJan Kara bio_put(bio); 343822dbba3SJan Kara } 344bd2d0210STheodore Ts'o } 345bd2d0210STheodore Ts'o 346bd2d0210STheodore Ts'o void ext4_io_submit(struct ext4_io_submit *io) 347bd2d0210STheodore Ts'o { 348bd2d0210STheodore Ts'o struct bio *bio = io->io_bio; 349bd2d0210STheodore Ts'o 350bd2d0210STheodore Ts'o if (bio) { 35195fe6c1aSMike Christie int io_op_flags = io->io_wbc->sync_mode == WB_SYNC_ALL ? 35270fd7614SChristoph Hellwig REQ_SYNC : 0; 35395fe6c1aSMike Christie bio_set_op_attrs(io->io_bio, REQ_OP_WRITE, io_op_flags); 3544e49ea4aSMike Christie submit_bio(io->io_bio); 355bd2d0210STheodore Ts'o } 3567dc57615SPeter Huewe io->io_bio = NULL; 35797a851edSJan Kara } 35897a851edSJan Kara 35997a851edSJan Kara void ext4_io_submit_init(struct ext4_io_submit *io, 36097a851edSJan Kara struct writeback_control *wbc) 36197a851edSJan Kara { 3625a33911fSTejun Heo io->io_wbc = wbc; 36397a851edSJan Kara io->io_bio = NULL; 3647dc57615SPeter Huewe io->io_end = NULL; 365bd2d0210STheodore Ts'o } 366bd2d0210STheodore Ts'o 36797a851edSJan Kara static int io_submit_init_bio(struct ext4_io_submit *io, 368bd2d0210STheodore Ts'o struct buffer_head *bh) 369bd2d0210STheodore Ts'o { 370bd2d0210STheodore Ts'o struct bio *bio; 371bd2d0210STheodore Ts'o 372b54ffb73SKent Overstreet bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); 373a1d8d9a7STheodore Ts'o if (!bio) 374a1d8d9a7STheodore Ts'o return -ENOMEM; 375001e4a87STejun Heo wbc_init_bio(io->io_wbc, bio); 3764f024f37SKent Overstreet bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 377bd2d0210STheodore Ts'o bio->bi_bdev = bh->b_bdev; 378bd2d0210STheodore Ts'o bio->bi_end_io = ext4_end_bio; 37997a851edSJan Kara bio->bi_private = ext4_get_io_end(io->io_end); 380bd2d0210STheodore Ts'o io->io_bio = bio; 381bd2d0210STheodore Ts'o io->io_next_block = bh->b_blocknr; 382bd2d0210STheodore Ts'o return 0; 383bd2d0210STheodore Ts'o } 384bd2d0210STheodore Ts'o 385bd2d0210STheodore Ts'o static int io_submit_add_bh(struct ext4_io_submit *io, 386bd2d0210STheodore Ts'o struct inode *inode, 3872058f83aSMichael Halcrow struct page *page, 388bd2d0210STheodore Ts'o struct buffer_head *bh) 389bd2d0210STheodore Ts'o { 390bd2d0210STheodore Ts'o int ret; 391bd2d0210STheodore Ts'o 392bd2d0210STheodore Ts'o if (io->io_bio && bh->b_blocknr != io->io_next_block) { 393bd2d0210STheodore Ts'o submit_and_retry: 394bd2d0210STheodore Ts'o ext4_io_submit(io); 395bd2d0210STheodore Ts'o } 396bd2d0210STheodore Ts'o if (io->io_bio == NULL) { 39797a851edSJan Kara ret = io_submit_init_bio(io, bh); 398bd2d0210STheodore Ts'o if (ret) 399bd2d0210STheodore Ts'o return ret; 400bd2d0210STheodore Ts'o } 4012058f83aSMichael Halcrow ret = bio_add_page(io->io_bio, page, bh->b_size, bh_offset(bh)); 402a549984bSTheodore Ts'o if (ret != bh->b_size) 403a549984bSTheodore Ts'o goto submit_and_retry; 404001e4a87STejun Heo wbc_account_io(io->io_wbc, page, bh->b_size); 40597a851edSJan Kara io->io_next_block++; 406bd2d0210STheodore Ts'o return 0; 407bd2d0210STheodore Ts'o } 408bd2d0210STheodore Ts'o 409bd2d0210STheodore Ts'o int ext4_bio_write_page(struct ext4_io_submit *io, 410bd2d0210STheodore Ts'o struct page *page, 411bd2d0210STheodore Ts'o int len, 4121c8349a1SNamjae Jeon struct writeback_control *wbc, 4131c8349a1SNamjae Jeon bool keep_towrite) 414bd2d0210STheodore Ts'o { 4152058f83aSMichael Halcrow struct page *data_page = NULL; 416bd2d0210STheodore Ts'o struct inode *inode = page->mapping->host; 41718017479SEric Engestrom unsigned block_start; 418bd2d0210STheodore Ts'o struct buffer_head *bh, *head; 419bd2d0210STheodore Ts'o int ret = 0; 4200058f965SJan Kara int nr_submitted = 0; 421937d7b84STheodore Ts'o int nr_to_submit = 0; 422bd2d0210STheodore Ts'o 423d50bdd5aSCurt Wohlgemuth BUG_ON(!PageLocked(page)); 424bd2d0210STheodore Ts'o BUG_ON(PageWriteback(page)); 425bd2d0210STheodore Ts'o 4261c8349a1SNamjae Jeon if (keep_towrite) 4271c8349a1SNamjae Jeon set_page_writeback_keepwrite(page); 4281c8349a1SNamjae Jeon else 429a54aa761STheodore Ts'o set_page_writeback(page); 430a54aa761STheodore Ts'o ClearPageError(page); 431bd2d0210STheodore Ts'o 4320058f965SJan Kara /* 433f8409abdSLinus Torvalds * Comments copied from block_write_full_page: 434eeece469SJan Kara * 435eeece469SJan Kara * The page straddles i_size. It must be zeroed out on each and every 436eeece469SJan Kara * writepage invocation because it may be mmapped. "A file is mapped 437eeece469SJan Kara * in multiples of the page size. For a file that is not a multiple of 438eeece469SJan Kara * the page size, the remaining memory is zeroed when mapped, and 439eeece469SJan Kara * writes to that region are not written out to the file." 440eeece469SJan Kara */ 44109cbfeafSKirill A. Shutemov if (len < PAGE_SIZE) 44209cbfeafSKirill A. Shutemov zero_user_segment(page, len, PAGE_SIZE); 443eeece469SJan Kara /* 4440058f965SJan Kara * In the first loop we prepare and mark buffers to submit. We have to 4450058f965SJan Kara * mark all buffers in the page before submitting so that 4460058f965SJan Kara * end_page_writeback() cannot be called from ext4_bio_end_io() when IO 4470058f965SJan Kara * on the first buffer finishes and we are still working on submitting 4480058f965SJan Kara * the second buffer. 4490058f965SJan Kara */ 4500058f965SJan Kara bh = head = page_buffers(page); 4510058f965SJan Kara do { 4520058f965SJan Kara block_start = bh_offset(bh); 453bd2d0210STheodore Ts'o if (block_start >= len) { 454bd2d0210STheodore Ts'o clear_buffer_dirty(bh); 455bd2d0210STheodore Ts'o set_buffer_uptodate(bh); 456bd2d0210STheodore Ts'o continue; 457bd2d0210STheodore Ts'o } 4588a850c3fSJan Kara if (!buffer_dirty(bh) || buffer_delay(bh) || 4598a850c3fSJan Kara !buffer_mapped(bh) || buffer_unwritten(bh)) { 4608a850c3fSJan Kara /* A hole? We can safely clear the dirty bit */ 4618a850c3fSJan Kara if (!buffer_mapped(bh)) 4628a850c3fSJan Kara clear_buffer_dirty(bh); 4638a850c3fSJan Kara if (io->io_bio) 4648a850c3fSJan Kara ext4_io_submit(io); 4658a850c3fSJan Kara continue; 4668a850c3fSJan Kara } 4670058f965SJan Kara if (buffer_new(bh)) { 4680058f965SJan Kara clear_buffer_new(bh); 469e64855c6SJan Kara clean_bdev_bh_alias(bh); 4700058f965SJan Kara } 4710058f965SJan Kara set_buffer_async_write(bh); 472937d7b84STheodore Ts'o nr_to_submit++; 4730058f965SJan Kara } while ((bh = bh->b_this_page) != head); 4740058f965SJan Kara 4750058f965SJan Kara bh = head = page_buffers(page); 4762058f83aSMichael Halcrow 477937d7b84STheodore Ts'o if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode) && 478937d7b84STheodore Ts'o nr_to_submit) { 479c9af28fdSTheodore Ts'o gfp_t gfp_flags = GFP_NOFS; 480c9af28fdSTheodore Ts'o 481c9af28fdSTheodore Ts'o retry_encrypt: 4829c4bb8a3SDavid Gstir data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0, 4839c4bb8a3SDavid Gstir page->index, gfp_flags); 4842058f83aSMichael Halcrow if (IS_ERR(data_page)) { 4852058f83aSMichael Halcrow ret = PTR_ERR(data_page); 4864762cc3fSDan Carpenter if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) { 487c9af28fdSTheodore Ts'o if (io->io_bio) { 488c9af28fdSTheodore Ts'o ext4_io_submit(io); 489c9af28fdSTheodore Ts'o congestion_wait(BLK_RW_ASYNC, HZ/50); 490c9af28fdSTheodore Ts'o } 491c9af28fdSTheodore Ts'o gfp_flags |= __GFP_NOFAIL; 492c9af28fdSTheodore Ts'o goto retry_encrypt; 493c9af28fdSTheodore Ts'o } 4942058f83aSMichael Halcrow data_page = NULL; 4952058f83aSMichael Halcrow goto out; 4962058f83aSMichael Halcrow } 4972058f83aSMichael Halcrow } 4982058f83aSMichael Halcrow 4992058f83aSMichael Halcrow /* Now submit buffers to write */ 5000058f965SJan Kara do { 5010058f965SJan Kara if (!buffer_async_write(bh)) 5020058f965SJan Kara continue; 5032058f83aSMichael Halcrow ret = io_submit_add_bh(io, inode, 5042058f83aSMichael Halcrow data_page ? data_page : page, bh); 505bd2d0210STheodore Ts'o if (ret) { 506bd2d0210STheodore Ts'o /* 507bd2d0210STheodore Ts'o * We only get here on ENOMEM. Not much else 508bd2d0210STheodore Ts'o * we can do but mark the page as dirty, and 509bd2d0210STheodore Ts'o * better luck next time. 510bd2d0210STheodore Ts'o */ 511bd2d0210STheodore Ts'o break; 512bd2d0210STheodore Ts'o } 5130058f965SJan Kara nr_submitted++; 5141ae48a63SJan Kara clear_buffer_dirty(bh); 5150058f965SJan Kara } while ((bh = bh->b_this_page) != head); 5160058f965SJan Kara 5170058f965SJan Kara /* Error stopped previous loop? Clean up buffers... */ 5180058f965SJan Kara if (ret) { 5192058f83aSMichael Halcrow out: 5202058f83aSMichael Halcrow if (data_page) 521a7550b30SJaegeuk Kim fscrypt_restore_control_page(data_page); 5222058f83aSMichael Halcrow printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret); 5232058f83aSMichael Halcrow redirty_page_for_writepage(wbc, page); 5240058f965SJan Kara do { 5250058f965SJan Kara clear_buffer_async_write(bh); 5260058f965SJan Kara bh = bh->b_this_page; 5270058f965SJan Kara } while (bh != head); 528bd2d0210STheodore Ts'o } 529bd2d0210STheodore Ts'o unlock_page(page); 5300058f965SJan Kara /* Nothing submitted - we have to end page writeback */ 5310058f965SJan Kara if (!nr_submitted) 5320058f965SJan Kara end_page_writeback(page); 533bd2d0210STheodore Ts'o return ret; 534bd2d0210STheodore Ts'o } 535