1bd2d0210STheodore Ts'o /* 2bd2d0210STheodore Ts'o * linux/fs/ext4/page-io.c 3bd2d0210STheodore Ts'o * 4bd2d0210STheodore Ts'o * This contains the new page_io functions for ext4 5bd2d0210STheodore Ts'o * 6bd2d0210STheodore Ts'o * Written by Theodore Ts'o, 2010. 7bd2d0210STheodore Ts'o */ 8bd2d0210STheodore Ts'o 9bd2d0210STheodore Ts'o #include <linux/fs.h> 10bd2d0210STheodore Ts'o #include <linux/time.h> 11bd2d0210STheodore Ts'o #include <linux/jbd2.h> 12bd2d0210STheodore Ts'o #include <linux/highuid.h> 13bd2d0210STheodore Ts'o #include <linux/pagemap.h> 14bd2d0210STheodore Ts'o #include <linux/quotaops.h> 15bd2d0210STheodore Ts'o #include <linux/string.h> 16bd2d0210STheodore Ts'o #include <linux/buffer_head.h> 17bd2d0210STheodore Ts'o #include <linux/writeback.h> 18bd2d0210STheodore Ts'o #include <linux/pagevec.h> 19bd2d0210STheodore Ts'o #include <linux/mpage.h> 20bd2d0210STheodore Ts'o #include <linux/namei.h> 21a27bb332SKent Overstreet #include <linux/aio.h> 22bd2d0210STheodore Ts'o #include <linux/uio.h> 23bd2d0210STheodore Ts'o #include <linux/bio.h> 24bd2d0210STheodore Ts'o #include <linux/workqueue.h> 25bd2d0210STheodore Ts'o #include <linux/kernel.h> 26bd2d0210STheodore Ts'o #include <linux/slab.h> 271ae48a63SJan Kara #include <linux/mm.h> 28*e8974c39SAnatol Pomozov #include <linux/ratelimit.h> 29bd2d0210STheodore Ts'o 30bd2d0210STheodore Ts'o #include "ext4_jbd2.h" 31bd2d0210STheodore Ts'o #include "xattr.h" 32bd2d0210STheodore Ts'o #include "acl.h" 33bd2d0210STheodore Ts'o 340058f965SJan Kara static struct kmem_cache *io_end_cachep; 35bd2d0210STheodore Ts'o 365dabfc78STheodore Ts'o int __init ext4_init_pageio(void) 37bd2d0210STheodore Ts'o { 38bd2d0210STheodore Ts'o io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT); 390058f965SJan Kara if (io_end_cachep == NULL) 40bd2d0210STheodore Ts'o return -ENOMEM; 41bd2d0210STheodore Ts'o return 0; 42bd2d0210STheodore Ts'o } 43bd2d0210STheodore Ts'o 445dabfc78STheodore Ts'o void ext4_exit_pageio(void) 45bd2d0210STheodore Ts'o { 46bd2d0210STheodore Ts'o kmem_cache_destroy(io_end_cachep); 47bd2d0210STheodore Ts'o } 48bd2d0210STheodore Ts'o 491ada47d9STheodore Ts'o /* 50b0857d30SJan Kara * Print an buffer I/O error compatible with the fs/buffer.c. This 51b0857d30SJan Kara * provides compatibility with dmesg scrapers that look for a specific 52b0857d30SJan Kara * buffer I/O error message. We really need a unified error reporting 53b0857d30SJan Kara * structure to userspace ala Digital Unix's uerf system, but it's 54b0857d30SJan Kara * probably not going to happen in my lifetime, due to LKML politics... 55b0857d30SJan Kara */ 56b0857d30SJan Kara static void buffer_io_error(struct buffer_head *bh) 57b0857d30SJan Kara { 58b0857d30SJan Kara char b[BDEVNAME_SIZE]; 59*e8974c39SAnatol Pomozov printk_ratelimited(KERN_ERR "Buffer I/O error on device %s, logical block %llu\n", 60b0857d30SJan Kara bdevname(bh->b_bdev, b), 61b0857d30SJan Kara (unsigned long long)bh->b_blocknr); 62b0857d30SJan Kara } 63b0857d30SJan Kara 64b0857d30SJan Kara static void ext4_finish_bio(struct bio *bio) 65b0857d30SJan Kara { 66b0857d30SJan Kara int i; 67b0857d30SJan Kara int error = !test_bit(BIO_UPTODATE, &bio->bi_flags); 68b0857d30SJan Kara 69b0857d30SJan Kara for (i = 0; i < bio->bi_vcnt; i++) { 70b0857d30SJan Kara struct bio_vec *bvec = &bio->bi_io_vec[i]; 71b0857d30SJan Kara struct page *page = bvec->bv_page; 72b0857d30SJan Kara struct buffer_head *bh, *head; 73b0857d30SJan Kara unsigned bio_start = bvec->bv_offset; 74b0857d30SJan Kara unsigned bio_end = bio_start + bvec->bv_len; 75b0857d30SJan Kara unsigned under_io = 0; 76b0857d30SJan Kara unsigned long flags; 77b0857d30SJan Kara 78b0857d30SJan Kara if (!page) 79b0857d30SJan Kara continue; 80b0857d30SJan Kara 81b0857d30SJan Kara if (error) { 82b0857d30SJan Kara SetPageError(page); 83b0857d30SJan Kara set_bit(AS_EIO, &page->mapping->flags); 84b0857d30SJan Kara } 85b0857d30SJan Kara bh = head = page_buffers(page); 86b0857d30SJan Kara /* 87b0857d30SJan Kara * We check all buffers in the page under BH_Uptodate_Lock 88b0857d30SJan Kara * to avoid races with other end io clearing async_write flags 89b0857d30SJan Kara */ 90b0857d30SJan Kara local_irq_save(flags); 91b0857d30SJan Kara bit_spin_lock(BH_Uptodate_Lock, &head->b_state); 92b0857d30SJan Kara do { 93b0857d30SJan Kara if (bh_offset(bh) < bio_start || 94b0857d30SJan Kara bh_offset(bh) + bh->b_size > bio_end) { 95b0857d30SJan Kara if (buffer_async_write(bh)) 96b0857d30SJan Kara under_io++; 97b0857d30SJan Kara continue; 98b0857d30SJan Kara } 99b0857d30SJan Kara clear_buffer_async_write(bh); 100b0857d30SJan Kara if (error) 101b0857d30SJan Kara buffer_io_error(bh); 102b0857d30SJan Kara } while ((bh = bh->b_this_page) != head); 103b0857d30SJan Kara bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); 104b0857d30SJan Kara local_irq_restore(flags); 105b0857d30SJan Kara if (!under_io) 106b0857d30SJan Kara end_page_writeback(page); 107b0857d30SJan Kara } 108b0857d30SJan Kara } 109b0857d30SJan Kara 11097a851edSJan Kara static void ext4_release_io_end(ext4_io_end_t *io_end) 111bd2d0210STheodore Ts'o { 112b0857d30SJan Kara struct bio *bio, *next_bio; 113b0857d30SJan Kara 11497a851edSJan Kara BUG_ON(!list_empty(&io_end->list)); 11597a851edSJan Kara BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN); 1166b523df4SJan Kara WARN_ON(io_end->handle); 117bd2d0210STheodore Ts'o 11897a851edSJan Kara if (atomic_dec_and_test(&EXT4_I(io_end->inode)->i_ioend_count)) 11997a851edSJan Kara wake_up_all(ext4_ioend_wq(io_end->inode)); 120b0857d30SJan Kara 121b0857d30SJan Kara for (bio = io_end->bio; bio; bio = next_bio) { 122b0857d30SJan Kara next_bio = bio->bi_private; 123b0857d30SJan Kara ext4_finish_bio(bio); 124b0857d30SJan Kara bio_put(bio); 125b0857d30SJan Kara } 12697a851edSJan Kara if (io_end->flag & EXT4_IO_END_DIRECT) 12797a851edSJan Kara inode_dio_done(io_end->inode); 12897a851edSJan Kara if (io_end->iocb) 12997a851edSJan Kara aio_complete(io_end->iocb, io_end->result, 0); 13097a851edSJan Kara kmem_cache_free(io_end_cachep, io_end); 13197a851edSJan Kara } 13297a851edSJan Kara 13397a851edSJan Kara static void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) 13497a851edSJan Kara { 13597a851edSJan Kara struct inode *inode = io_end->inode; 13697a851edSJan Kara 13797a851edSJan Kara io_end->flag &= ~EXT4_IO_END_UNWRITTEN; 13897a851edSJan Kara /* Wake up anyone waiting on unwritten extent conversion */ 13997a851edSJan Kara if (atomic_dec_and_test(&EXT4_I(inode)->i_unwritten)) 14097a851edSJan Kara wake_up_all(ext4_ioend_wq(inode)); 141bd2d0210STheodore Ts'o } 142bd2d0210STheodore Ts'o 143a115f749SJan Kara /* 144a115f749SJan Kara * Check a range of space and convert unwritten extents to written. Note that 145a115f749SJan Kara * we are protected from truncate touching same part of extent tree by the 146a115f749SJan Kara * fact that truncate code waits for all DIO to finish (thus exclusion from 147a115f749SJan Kara * direct IO is achieved) and also waits for PageWriteback bits. Thus we 148a115f749SJan Kara * cannot get to ext4_ext_truncate() before all IOs overlapping that range are 149a115f749SJan Kara * completed (happens from ext4_free_ioend()). 150a115f749SJan Kara */ 15128a535f9SDmitry Monakhov static int ext4_end_io(ext4_io_end_t *io) 152bd2d0210STheodore Ts'o { 153bd2d0210STheodore Ts'o struct inode *inode = io->inode; 154bd2d0210STheodore Ts'o loff_t offset = io->offset; 155bd2d0210STheodore Ts'o ssize_t size = io->size; 1566b523df4SJan Kara handle_t *handle = io->handle; 157bd2d0210STheodore Ts'o int ret = 0; 158bd2d0210STheodore Ts'o 159bd2d0210STheodore Ts'o ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," 160bd2d0210STheodore Ts'o "list->prev 0x%p\n", 161bd2d0210STheodore Ts'o io, inode->i_ino, io->list.next, io->list.prev); 162bd2d0210STheodore Ts'o 1636b523df4SJan Kara io->handle = NULL; /* Following call will use up the handle */ 1646b523df4SJan Kara ret = ext4_convert_unwritten_extents(handle, inode, offset, size); 165bd2d0210STheodore Ts'o if (ret < 0) { 166b82e384cSTheodore Ts'o ext4_msg(inode->i_sb, KERN_EMERG, 167b82e384cSTheodore Ts'o "failed to convert unwritten extents to written " 168b82e384cSTheodore Ts'o "extents -- potential data loss! " 169b82e384cSTheodore Ts'o "(inode %lu, offset %llu, size %zd, error %d)", 170b82e384cSTheodore Ts'o inode->i_ino, offset, size, ret); 171bd2d0210STheodore Ts'o } 17297a851edSJan Kara ext4_clear_io_unwritten_flag(io); 17397a851edSJan Kara ext4_release_io_end(io); 174bd2d0210STheodore Ts'o return ret; 175bd2d0210STheodore Ts'o } 176bd2d0210STheodore Ts'o 1772e8fa54eSJan Kara static void dump_completed_IO(struct inode *inode, struct list_head *head) 17828a535f9SDmitry Monakhov { 17928a535f9SDmitry Monakhov #ifdef EXT4FS_DEBUG 18028a535f9SDmitry Monakhov struct list_head *cur, *before, *after; 18128a535f9SDmitry Monakhov ext4_io_end_t *io, *io0, *io1; 18228a535f9SDmitry Monakhov 1832e8fa54eSJan Kara if (list_empty(head)) 18428a535f9SDmitry Monakhov return; 18528a535f9SDmitry Monakhov 1862e8fa54eSJan Kara ext4_debug("Dump inode %lu completed io list\n", inode->i_ino); 1872e8fa54eSJan Kara list_for_each_entry(io, head, list) { 18828a535f9SDmitry Monakhov cur = &io->list; 18928a535f9SDmitry Monakhov before = cur->prev; 19028a535f9SDmitry Monakhov io0 = container_of(before, ext4_io_end_t, list); 19128a535f9SDmitry Monakhov after = cur->next; 19228a535f9SDmitry Monakhov io1 = container_of(after, ext4_io_end_t, list); 19328a535f9SDmitry Monakhov 19428a535f9SDmitry Monakhov ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n", 19528a535f9SDmitry Monakhov io, inode->i_ino, io0, io1); 19628a535f9SDmitry Monakhov } 19728a535f9SDmitry Monakhov #endif 19828a535f9SDmitry Monakhov } 19928a535f9SDmitry Monakhov 20028a535f9SDmitry Monakhov /* Add the io_end to per-inode completed end_io list. */ 20197a851edSJan Kara static void ext4_add_complete_io(ext4_io_end_t *io_end) 20228a535f9SDmitry Monakhov { 20328a535f9SDmitry Monakhov struct ext4_inode_info *ei = EXT4_I(io_end->inode); 20428a535f9SDmitry Monakhov struct workqueue_struct *wq; 20528a535f9SDmitry Monakhov unsigned long flags; 20628a535f9SDmitry Monakhov 20728a535f9SDmitry Monakhov BUG_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN)); 20828a535f9SDmitry Monakhov spin_lock_irqsave(&ei->i_completed_io_lock, flags); 2092e8fa54eSJan Kara if (io_end->handle) { 2102e8fa54eSJan Kara wq = EXT4_SB(io_end->inode->i_sb)->rsv_conversion_wq; 2112e8fa54eSJan Kara if (list_empty(&ei->i_rsv_conversion_list)) 2122e8fa54eSJan Kara queue_work(wq, &ei->i_rsv_conversion_work); 2132e8fa54eSJan Kara list_add_tail(&io_end->list, &ei->i_rsv_conversion_list); 2142e8fa54eSJan Kara } else { 2152e8fa54eSJan Kara wq = EXT4_SB(io_end->inode->i_sb)->unrsv_conversion_wq; 2162e8fa54eSJan Kara if (list_empty(&ei->i_unrsv_conversion_list)) 2172e8fa54eSJan Kara queue_work(wq, &ei->i_unrsv_conversion_work); 2182e8fa54eSJan Kara list_add_tail(&io_end->list, &ei->i_unrsv_conversion_list); 2192e8fa54eSJan Kara } 22028a535f9SDmitry Monakhov spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 22128a535f9SDmitry Monakhov } 22228a535f9SDmitry Monakhov 2232e8fa54eSJan Kara static int ext4_do_flush_completed_IO(struct inode *inode, 2242e8fa54eSJan Kara struct list_head *head) 22528a535f9SDmitry Monakhov { 22628a535f9SDmitry Monakhov ext4_io_end_t *io; 227002bd7faSJan Kara struct list_head unwritten; 22828a535f9SDmitry Monakhov unsigned long flags; 22928a535f9SDmitry Monakhov struct ext4_inode_info *ei = EXT4_I(inode); 23028a535f9SDmitry Monakhov int err, ret = 0; 23128a535f9SDmitry Monakhov 23228a535f9SDmitry Monakhov spin_lock_irqsave(&ei->i_completed_io_lock, flags); 2332e8fa54eSJan Kara dump_completed_IO(inode, head); 2342e8fa54eSJan Kara list_replace_init(head, &unwritten); 23528a535f9SDmitry Monakhov spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 23628a535f9SDmitry Monakhov 23728a535f9SDmitry Monakhov while (!list_empty(&unwritten)) { 23828a535f9SDmitry Monakhov io = list_entry(unwritten.next, ext4_io_end_t, list); 23928a535f9SDmitry Monakhov BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN)); 24028a535f9SDmitry Monakhov list_del_init(&io->list); 24128a535f9SDmitry Monakhov 24228a535f9SDmitry Monakhov err = ext4_end_io(io); 24328a535f9SDmitry Monakhov if (unlikely(!ret && err)) 24428a535f9SDmitry Monakhov ret = err; 24528a535f9SDmitry Monakhov } 24628a535f9SDmitry Monakhov return ret; 24728a535f9SDmitry Monakhov } 24828a535f9SDmitry Monakhov 249bd2d0210STheodore Ts'o /* 2502e8fa54eSJan Kara * work on completed IO, to convert unwritten extents to extents 251bd2d0210STheodore Ts'o */ 2522e8fa54eSJan Kara void ext4_end_io_rsv_work(struct work_struct *work) 253bd2d0210STheodore Ts'o { 25484c17543SJan Kara struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info, 2552e8fa54eSJan Kara i_rsv_conversion_work); 2562e8fa54eSJan Kara ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list); 2572e8fa54eSJan Kara } 2582e8fa54eSJan Kara 2592e8fa54eSJan Kara void ext4_end_io_unrsv_work(struct work_struct *work) 2602e8fa54eSJan Kara { 2612e8fa54eSJan Kara struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info, 2622e8fa54eSJan Kara i_unrsv_conversion_work); 2632e8fa54eSJan Kara ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_unrsv_conversion_list); 264d73d5046STao Ma } 265d73d5046STao Ma 266bd2d0210STheodore Ts'o ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags) 267bd2d0210STheodore Ts'o { 268b17b35ecSJesper Juhl ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags); 269bd2d0210STheodore Ts'o if (io) { 270f7ad6d2eSTheodore Ts'o atomic_inc(&EXT4_I(inode)->i_ioend_count); 271f7ad6d2eSTheodore Ts'o io->inode = inode; 272bd2d0210STheodore Ts'o INIT_LIST_HEAD(&io->list); 27397a851edSJan Kara atomic_set(&io->count, 1); 274bd2d0210STheodore Ts'o } 275bd2d0210STheodore Ts'o return io; 276bd2d0210STheodore Ts'o } 277bd2d0210STheodore Ts'o 27897a851edSJan Kara void ext4_put_io_end_defer(ext4_io_end_t *io_end) 27997a851edSJan Kara { 28097a851edSJan Kara if (atomic_dec_and_test(&io_end->count)) { 28197a851edSJan Kara if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) || !io_end->size) { 28297a851edSJan Kara ext4_release_io_end(io_end); 28397a851edSJan Kara return; 28497a851edSJan Kara } 28597a851edSJan Kara ext4_add_complete_io(io_end); 28697a851edSJan Kara } 28797a851edSJan Kara } 28897a851edSJan Kara 28997a851edSJan Kara int ext4_put_io_end(ext4_io_end_t *io_end) 29097a851edSJan Kara { 29197a851edSJan Kara int err = 0; 29297a851edSJan Kara 29397a851edSJan Kara if (atomic_dec_and_test(&io_end->count)) { 29497a851edSJan Kara if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 2956b523df4SJan Kara err = ext4_convert_unwritten_extents(io_end->handle, 2966b523df4SJan Kara io_end->inode, io_end->offset, 2976b523df4SJan Kara io_end->size); 2986b523df4SJan Kara io_end->handle = NULL; 29997a851edSJan Kara ext4_clear_io_unwritten_flag(io_end); 30097a851edSJan Kara } 30197a851edSJan Kara ext4_release_io_end(io_end); 30297a851edSJan Kara } 30397a851edSJan Kara return err; 30497a851edSJan Kara } 30597a851edSJan Kara 30697a851edSJan Kara ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end) 30797a851edSJan Kara { 30897a851edSJan Kara atomic_inc(&io_end->count); 30997a851edSJan Kara return io_end; 31097a851edSJan Kara } 31197a851edSJan Kara 312822dbba3SJan Kara /* BIO completion function for page writeback */ 313bd2d0210STheodore Ts'o static void ext4_end_bio(struct bio *bio, int error) 314bd2d0210STheodore Ts'o { 315bd2d0210STheodore Ts'o ext4_io_end_t *io_end = bio->bi_private; 316d50bdd5aSCurt Wohlgemuth sector_t bi_sector = bio->bi_sector; 317bd2d0210STheodore Ts'o 318bd2d0210STheodore Ts'o BUG_ON(!io_end); 319bd2d0210STheodore Ts'o bio->bi_end_io = NULL; 320bd2d0210STheodore Ts'o if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 321bd2d0210STheodore Ts'o error = 0; 3220058f965SJan Kara 323f7ad6d2eSTheodore Ts'o if (error) { 324b0857d30SJan Kara struct inode *inode = io_end->inode; 325b0857d30SJan Kara 326f7ad6d2eSTheodore Ts'o ext4_warning(inode->i_sb, "I/O error writing to inode %lu " 327f7ad6d2eSTheodore Ts'o "(offset %llu size %ld starting block %llu)", 328f7ad6d2eSTheodore Ts'o inode->i_ino, 329f7ad6d2eSTheodore Ts'o (unsigned long long) io_end->offset, 330f7ad6d2eSTheodore Ts'o (long) io_end->size, 331f7ad6d2eSTheodore Ts'o (unsigned long long) 332d50bdd5aSCurt Wohlgemuth bi_sector >> (inode->i_blkbits - 9)); 333f7ad6d2eSTheodore Ts'o } 334822dbba3SJan Kara 335822dbba3SJan Kara if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 336822dbba3SJan Kara /* 337822dbba3SJan Kara * Link bio into list hanging from io_end. We have to do it 338822dbba3SJan Kara * atomically as bio completions can be racing against each 339822dbba3SJan Kara * other. 340822dbba3SJan Kara */ 341822dbba3SJan Kara bio->bi_private = xchg(&io_end->bio, bio); 34297a851edSJan Kara ext4_put_io_end_defer(io_end); 343822dbba3SJan Kara } else { 344822dbba3SJan Kara /* 345822dbba3SJan Kara * Drop io_end reference early. Inode can get freed once 346822dbba3SJan Kara * we finish the bio. 347822dbba3SJan Kara */ 348822dbba3SJan Kara ext4_put_io_end_defer(io_end); 349822dbba3SJan Kara ext4_finish_bio(bio); 350822dbba3SJan Kara bio_put(bio); 351822dbba3SJan Kara } 352bd2d0210STheodore Ts'o } 353bd2d0210STheodore Ts'o 354bd2d0210STheodore Ts'o void ext4_io_submit(struct ext4_io_submit *io) 355bd2d0210STheodore Ts'o { 356bd2d0210STheodore Ts'o struct bio *bio = io->io_bio; 357bd2d0210STheodore Ts'o 358bd2d0210STheodore Ts'o if (bio) { 359bd2d0210STheodore Ts'o bio_get(io->io_bio); 360bd2d0210STheodore Ts'o submit_bio(io->io_op, io->io_bio); 361bd2d0210STheodore Ts'o BUG_ON(bio_flagged(io->io_bio, BIO_EOPNOTSUPP)); 362bd2d0210STheodore Ts'o bio_put(io->io_bio); 363bd2d0210STheodore Ts'o } 3647dc57615SPeter Huewe io->io_bio = NULL; 36597a851edSJan Kara } 36697a851edSJan Kara 36797a851edSJan Kara void ext4_io_submit_init(struct ext4_io_submit *io, 36897a851edSJan Kara struct writeback_control *wbc) 36997a851edSJan Kara { 37097a851edSJan Kara io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE); 37197a851edSJan Kara io->io_bio = NULL; 3727dc57615SPeter Huewe io->io_end = NULL; 373bd2d0210STheodore Ts'o } 374bd2d0210STheodore Ts'o 37597a851edSJan Kara static int io_submit_init_bio(struct ext4_io_submit *io, 376bd2d0210STheodore Ts'o struct buffer_head *bh) 377bd2d0210STheodore Ts'o { 378bd2d0210STheodore Ts'o int nvecs = bio_get_nr_vecs(bh->b_bdev); 379bd2d0210STheodore Ts'o struct bio *bio; 380bd2d0210STheodore Ts'o 381275d3ba6STheodore Ts'o bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES)); 382a1d8d9a7STheodore Ts'o if (!bio) 383a1d8d9a7STheodore Ts'o return -ENOMEM; 384bd2d0210STheodore Ts'o bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 385bd2d0210STheodore Ts'o bio->bi_bdev = bh->b_bdev; 386bd2d0210STheodore Ts'o bio->bi_end_io = ext4_end_bio; 38797a851edSJan Kara bio->bi_private = ext4_get_io_end(io->io_end); 388bd2d0210STheodore Ts'o io->io_bio = bio; 389bd2d0210STheodore Ts'o io->io_next_block = bh->b_blocknr; 390bd2d0210STheodore Ts'o return 0; 391bd2d0210STheodore Ts'o } 392bd2d0210STheodore Ts'o 393bd2d0210STheodore Ts'o static int io_submit_add_bh(struct ext4_io_submit *io, 394bd2d0210STheodore Ts'o struct inode *inode, 395bd2d0210STheodore Ts'o struct buffer_head *bh) 396bd2d0210STheodore Ts'o { 397bd2d0210STheodore Ts'o int ret; 398bd2d0210STheodore Ts'o 399bd2d0210STheodore Ts'o if (io->io_bio && bh->b_blocknr != io->io_next_block) { 400bd2d0210STheodore Ts'o submit_and_retry: 401bd2d0210STheodore Ts'o ext4_io_submit(io); 402bd2d0210STheodore Ts'o } 403bd2d0210STheodore Ts'o if (io->io_bio == NULL) { 40497a851edSJan Kara ret = io_submit_init_bio(io, bh); 405bd2d0210STheodore Ts'o if (ret) 406bd2d0210STheodore Ts'o return ret; 407bd2d0210STheodore Ts'o } 408a549984bSTheodore Ts'o ret = bio_add_page(io->io_bio, bh->b_page, bh->b_size, bh_offset(bh)); 409a549984bSTheodore Ts'o if (ret != bh->b_size) 410a549984bSTheodore Ts'o goto submit_and_retry; 41197a851edSJan Kara io->io_next_block++; 412bd2d0210STheodore Ts'o return 0; 413bd2d0210STheodore Ts'o } 414bd2d0210STheodore Ts'o 415bd2d0210STheodore Ts'o int ext4_bio_write_page(struct ext4_io_submit *io, 416bd2d0210STheodore Ts'o struct page *page, 417bd2d0210STheodore Ts'o int len, 418bd2d0210STheodore Ts'o struct writeback_control *wbc) 419bd2d0210STheodore Ts'o { 420bd2d0210STheodore Ts'o struct inode *inode = page->mapping->host; 4210058f965SJan Kara unsigned block_start, blocksize; 422bd2d0210STheodore Ts'o struct buffer_head *bh, *head; 423bd2d0210STheodore Ts'o int ret = 0; 4240058f965SJan Kara int nr_submitted = 0; 425bd2d0210STheodore Ts'o 426bd2d0210STheodore Ts'o blocksize = 1 << inode->i_blkbits; 427bd2d0210STheodore Ts'o 428d50bdd5aSCurt Wohlgemuth BUG_ON(!PageLocked(page)); 429bd2d0210STheodore Ts'o BUG_ON(PageWriteback(page)); 430bd2d0210STheodore Ts'o 431a54aa761STheodore Ts'o set_page_writeback(page); 432a54aa761STheodore Ts'o ClearPageError(page); 433bd2d0210STheodore Ts'o 4340058f965SJan Kara /* 4350058f965SJan Kara * In the first loop we prepare and mark buffers to submit. We have to 4360058f965SJan Kara * mark all buffers in the page before submitting so that 4370058f965SJan Kara * end_page_writeback() cannot be called from ext4_bio_end_io() when IO 4380058f965SJan Kara * on the first buffer finishes and we are still working on submitting 4390058f965SJan Kara * the second buffer. 4400058f965SJan Kara */ 4410058f965SJan Kara bh = head = page_buffers(page); 4420058f965SJan Kara do { 4430058f965SJan Kara block_start = bh_offset(bh); 444bd2d0210STheodore Ts'o if (block_start >= len) { 4455a0dc736SYongqiang Yang /* 4465a0dc736SYongqiang Yang * Comments copied from block_write_full_page_endio: 4475a0dc736SYongqiang Yang * 4485a0dc736SYongqiang Yang * The page straddles i_size. It must be zeroed out on 4495a0dc736SYongqiang Yang * each and every writepage invocation because it may 4505a0dc736SYongqiang Yang * be mmapped. "A file is mapped in multiples of the 4515a0dc736SYongqiang Yang * page size. For a file that is not a multiple of 4525a0dc736SYongqiang Yang * the page size, the remaining memory is zeroed when 4535a0dc736SYongqiang Yang * mapped, and writes to that region are not written 4545a0dc736SYongqiang Yang * out to the file." 4555a0dc736SYongqiang Yang */ 4560058f965SJan Kara zero_user_segment(page, block_start, 4570058f965SJan Kara block_start + blocksize); 458bd2d0210STheodore Ts'o clear_buffer_dirty(bh); 459bd2d0210STheodore Ts'o set_buffer_uptodate(bh); 460bd2d0210STheodore Ts'o continue; 461bd2d0210STheodore Ts'o } 4628a850c3fSJan Kara if (!buffer_dirty(bh) || buffer_delay(bh) || 4638a850c3fSJan Kara !buffer_mapped(bh) || buffer_unwritten(bh)) { 4648a850c3fSJan Kara /* A hole? We can safely clear the dirty bit */ 4658a850c3fSJan Kara if (!buffer_mapped(bh)) 4668a850c3fSJan Kara clear_buffer_dirty(bh); 4678a850c3fSJan Kara if (io->io_bio) 4688a850c3fSJan Kara ext4_io_submit(io); 4698a850c3fSJan Kara continue; 4708a850c3fSJan Kara } 4710058f965SJan Kara if (buffer_new(bh)) { 4720058f965SJan Kara clear_buffer_new(bh); 4730058f965SJan Kara unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr); 4740058f965SJan Kara } 4750058f965SJan Kara set_buffer_async_write(bh); 4760058f965SJan Kara } while ((bh = bh->b_this_page) != head); 4770058f965SJan Kara 4780058f965SJan Kara /* Now submit buffers to write */ 4790058f965SJan Kara bh = head = page_buffers(page); 4800058f965SJan Kara do { 4810058f965SJan Kara if (!buffer_async_write(bh)) 4820058f965SJan Kara continue; 48397a851edSJan Kara ret = io_submit_add_bh(io, inode, bh); 484bd2d0210STheodore Ts'o if (ret) { 485bd2d0210STheodore Ts'o /* 486bd2d0210STheodore Ts'o * We only get here on ENOMEM. Not much else 487bd2d0210STheodore Ts'o * we can do but mark the page as dirty, and 488bd2d0210STheodore Ts'o * better luck next time. 489bd2d0210STheodore Ts'o */ 4901ae48a63SJan Kara redirty_page_for_writepage(wbc, page); 491bd2d0210STheodore Ts'o break; 492bd2d0210STheodore Ts'o } 4930058f965SJan Kara nr_submitted++; 4941ae48a63SJan Kara clear_buffer_dirty(bh); 4950058f965SJan Kara } while ((bh = bh->b_this_page) != head); 4960058f965SJan Kara 4970058f965SJan Kara /* Error stopped previous loop? Clean up buffers... */ 4980058f965SJan Kara if (ret) { 4990058f965SJan Kara do { 5000058f965SJan Kara clear_buffer_async_write(bh); 5010058f965SJan Kara bh = bh->b_this_page; 5020058f965SJan Kara } while (bh != head); 503bd2d0210STheodore Ts'o } 504bd2d0210STheodore Ts'o unlock_page(page); 5050058f965SJan Kara /* Nothing submitted - we have to end page writeback */ 5060058f965SJan Kara if (!nr_submitted) 5070058f965SJan Kara end_page_writeback(page); 508bd2d0210STheodore Ts'o return ret; 509bd2d0210STheodore Ts'o } 510