1bd2d0210STheodore Ts'o /* 2bd2d0210STheodore Ts'o * linux/fs/ext4/page-io.c 3bd2d0210STheodore Ts'o * 4bd2d0210STheodore Ts'o * This contains the new page_io functions for ext4 5bd2d0210STheodore Ts'o * 6bd2d0210STheodore Ts'o * Written by Theodore Ts'o, 2010. 7bd2d0210STheodore Ts'o */ 8bd2d0210STheodore Ts'o 9bd2d0210STheodore Ts'o #include <linux/fs.h> 10bd2d0210STheodore Ts'o #include <linux/time.h> 11bd2d0210STheodore Ts'o #include <linux/jbd2.h> 12bd2d0210STheodore Ts'o #include <linux/highuid.h> 13bd2d0210STheodore Ts'o #include <linux/pagemap.h> 14bd2d0210STheodore Ts'o #include <linux/quotaops.h> 15bd2d0210STheodore Ts'o #include <linux/string.h> 16bd2d0210STheodore Ts'o #include <linux/buffer_head.h> 17bd2d0210STheodore Ts'o #include <linux/writeback.h> 18bd2d0210STheodore Ts'o #include <linux/pagevec.h> 19bd2d0210STheodore Ts'o #include <linux/mpage.h> 20bd2d0210STheodore Ts'o #include <linux/namei.h> 21*a27bb332SKent Overstreet #include <linux/aio.h> 22bd2d0210STheodore Ts'o #include <linux/uio.h> 23bd2d0210STheodore Ts'o #include <linux/bio.h> 24bd2d0210STheodore Ts'o #include <linux/workqueue.h> 25bd2d0210STheodore Ts'o #include <linux/kernel.h> 26bd2d0210STheodore Ts'o #include <linux/slab.h> 271ae48a63SJan Kara #include <linux/mm.h> 28bd2d0210STheodore Ts'o 29bd2d0210STheodore Ts'o #include "ext4_jbd2.h" 30bd2d0210STheodore Ts'o #include "xattr.h" 31bd2d0210STheodore Ts'o #include "acl.h" 32bd2d0210STheodore Ts'o 330058f965SJan Kara static struct kmem_cache *io_end_cachep; 34bd2d0210STheodore Ts'o 355dabfc78STheodore Ts'o int __init ext4_init_pageio(void) 36bd2d0210STheodore Ts'o { 37bd2d0210STheodore Ts'o io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT); 380058f965SJan Kara if (io_end_cachep == NULL) 39bd2d0210STheodore Ts'o return -ENOMEM; 40bd2d0210STheodore Ts'o return 0; 41bd2d0210STheodore Ts'o } 42bd2d0210STheodore Ts'o 435dabfc78STheodore Ts'o void ext4_exit_pageio(void) 44bd2d0210STheodore Ts'o { 45bd2d0210STheodore Ts'o kmem_cache_destroy(io_end_cachep); 46bd2d0210STheodore Ts'o } 47bd2d0210STheodore Ts'o 481ada47d9STheodore Ts'o /* 491ada47d9STheodore Ts'o * This function is called by ext4_evict_inode() to make sure there is 501ada47d9STheodore Ts'o * no more pending I/O completion work left to do. 511ada47d9STheodore Ts'o */ 521ada47d9STheodore Ts'o void ext4_ioend_shutdown(struct inode *inode) 53f7ad6d2eSTheodore Ts'o { 54e9e3bcecSEric Sandeen wait_queue_head_t *wq = ext4_ioend_wq(inode); 55f7ad6d2eSTheodore Ts'o 56f7ad6d2eSTheodore Ts'o wait_event(*wq, (atomic_read(&EXT4_I(inode)->i_ioend_count) == 0)); 571ada47d9STheodore Ts'o /* 581ada47d9STheodore Ts'o * We need to make sure the work structure is finished being 591ada47d9STheodore Ts'o * used before we let the inode get destroyed. 601ada47d9STheodore Ts'o */ 611ada47d9STheodore Ts'o if (work_pending(&EXT4_I(inode)->i_unwritten_work)) 621ada47d9STheodore Ts'o cancel_work_sync(&EXT4_I(inode)->i_unwritten_work); 63f7ad6d2eSTheodore Ts'o } 64f7ad6d2eSTheodore Ts'o 654eec708dSJan Kara static void ext4_release_io_end(ext4_io_end_t *io_end) 66bd2d0210STheodore Ts'o { 674eec708dSJan Kara BUG_ON(!list_empty(&io_end->list)); 684eec708dSJan Kara BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN); 69bd2d0210STheodore Ts'o 704eec708dSJan Kara if (atomic_dec_and_test(&EXT4_I(io_end->inode)->i_ioend_count)) 714eec708dSJan Kara wake_up_all(ext4_ioend_wq(io_end->inode)); 724eec708dSJan Kara if (io_end->flag & EXT4_IO_END_DIRECT) 734eec708dSJan Kara inode_dio_done(io_end->inode); 744eec708dSJan Kara if (io_end->iocb) 754eec708dSJan Kara aio_complete(io_end->iocb, io_end->result, 0); 764eec708dSJan Kara kmem_cache_free(io_end_cachep, io_end); 774eec708dSJan Kara } 7882e54229SDmitry Monakhov 794eec708dSJan Kara static void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) 804eec708dSJan Kara { 814eec708dSJan Kara struct inode *inode = io_end->inode; 824eec708dSJan Kara 834eec708dSJan Kara io_end->flag &= ~EXT4_IO_END_UNWRITTEN; 844eec708dSJan Kara /* Wake up anyone waiting on unwritten extent conversion */ 854eec708dSJan Kara if (atomic_dec_and_test(&EXT4_I(inode)->i_unwritten)) 864eec708dSJan Kara wake_up_all(ext4_ioend_wq(inode)); 87bd2d0210STheodore Ts'o } 88bd2d0210STheodore Ts'o 8928a535f9SDmitry Monakhov /* check a range of space and convert unwritten extents to written. */ 9028a535f9SDmitry Monakhov static int ext4_end_io(ext4_io_end_t *io) 91bd2d0210STheodore Ts'o { 92bd2d0210STheodore Ts'o struct inode *inode = io->inode; 93bd2d0210STheodore Ts'o loff_t offset = io->offset; 94bd2d0210STheodore Ts'o ssize_t size = io->size; 95bd2d0210STheodore Ts'o int ret = 0; 96bd2d0210STheodore Ts'o 97bd2d0210STheodore Ts'o ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," 98bd2d0210STheodore Ts'o "list->prev 0x%p\n", 99bd2d0210STheodore Ts'o io, inode->i_ino, io->list.next, io->list.prev); 100bd2d0210STheodore Ts'o 101bd2d0210STheodore Ts'o ret = ext4_convert_unwritten_extents(inode, offset, size); 102bd2d0210STheodore Ts'o if (ret < 0) { 103b82e384cSTheodore Ts'o ext4_msg(inode->i_sb, KERN_EMERG, 104b82e384cSTheodore Ts'o "failed to convert unwritten extents to written " 105b82e384cSTheodore Ts'o "extents -- potential data loss! " 106b82e384cSTheodore Ts'o "(inode %lu, offset %llu, size %zd, error %d)", 107b82e384cSTheodore Ts'o inode->i_ino, offset, size, ret); 108bd2d0210STheodore Ts'o } 1094eec708dSJan Kara ext4_clear_io_unwritten_flag(io); 1104eec708dSJan Kara ext4_release_io_end(io); 111bd2d0210STheodore Ts'o return ret; 112bd2d0210STheodore Ts'o } 113bd2d0210STheodore Ts'o 11428a535f9SDmitry Monakhov static void dump_completed_IO(struct inode *inode) 11528a535f9SDmitry Monakhov { 11628a535f9SDmitry Monakhov #ifdef EXT4FS_DEBUG 11728a535f9SDmitry Monakhov struct list_head *cur, *before, *after; 11828a535f9SDmitry Monakhov ext4_io_end_t *io, *io0, *io1; 11928a535f9SDmitry Monakhov 12028a535f9SDmitry Monakhov if (list_empty(&EXT4_I(inode)->i_completed_io_list)) { 12128a535f9SDmitry Monakhov ext4_debug("inode %lu completed_io list is empty\n", 12228a535f9SDmitry Monakhov inode->i_ino); 12328a535f9SDmitry Monakhov return; 12428a535f9SDmitry Monakhov } 12528a535f9SDmitry Monakhov 12628a535f9SDmitry Monakhov ext4_debug("Dump inode %lu completed_io list\n", inode->i_ino); 12728a535f9SDmitry Monakhov list_for_each_entry(io, &EXT4_I(inode)->i_completed_io_list, list) { 12828a535f9SDmitry Monakhov cur = &io->list; 12928a535f9SDmitry Monakhov before = cur->prev; 13028a535f9SDmitry Monakhov io0 = container_of(before, ext4_io_end_t, list); 13128a535f9SDmitry Monakhov after = cur->next; 13228a535f9SDmitry Monakhov io1 = container_of(after, ext4_io_end_t, list); 13328a535f9SDmitry Monakhov 13428a535f9SDmitry Monakhov ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n", 13528a535f9SDmitry Monakhov io, inode->i_ino, io0, io1); 13628a535f9SDmitry Monakhov } 13728a535f9SDmitry Monakhov #endif 13828a535f9SDmitry Monakhov } 13928a535f9SDmitry Monakhov 14028a535f9SDmitry Monakhov /* Add the io_end to per-inode completed end_io list. */ 1414eec708dSJan Kara static void ext4_add_complete_io(ext4_io_end_t *io_end) 14228a535f9SDmitry Monakhov { 14328a535f9SDmitry Monakhov struct ext4_inode_info *ei = EXT4_I(io_end->inode); 14428a535f9SDmitry Monakhov struct workqueue_struct *wq; 14528a535f9SDmitry Monakhov unsigned long flags; 14628a535f9SDmitry Monakhov 14728a535f9SDmitry Monakhov BUG_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN)); 14828a535f9SDmitry Monakhov wq = EXT4_SB(io_end->inode->i_sb)->dio_unwritten_wq; 14928a535f9SDmitry Monakhov 15028a535f9SDmitry Monakhov spin_lock_irqsave(&ei->i_completed_io_lock, flags); 15184c17543SJan Kara if (list_empty(&ei->i_completed_io_list)) 15284c17543SJan Kara queue_work(wq, &ei->i_unwritten_work); 15328a535f9SDmitry Monakhov list_add_tail(&io_end->list, &ei->i_completed_io_list); 15428a535f9SDmitry Monakhov spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 15528a535f9SDmitry Monakhov } 15628a535f9SDmitry Monakhov 15784c17543SJan Kara static int ext4_do_flush_completed_IO(struct inode *inode) 15828a535f9SDmitry Monakhov { 15928a535f9SDmitry Monakhov ext4_io_end_t *io; 160002bd7faSJan Kara struct list_head unwritten; 16128a535f9SDmitry Monakhov unsigned long flags; 16228a535f9SDmitry Monakhov struct ext4_inode_info *ei = EXT4_I(inode); 16328a535f9SDmitry Monakhov int err, ret = 0; 16428a535f9SDmitry Monakhov 16528a535f9SDmitry Monakhov spin_lock_irqsave(&ei->i_completed_io_lock, flags); 16628a535f9SDmitry Monakhov dump_completed_IO(inode); 16728a535f9SDmitry Monakhov list_replace_init(&ei->i_completed_io_list, &unwritten); 16828a535f9SDmitry Monakhov spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 16928a535f9SDmitry Monakhov 17028a535f9SDmitry Monakhov while (!list_empty(&unwritten)) { 17128a535f9SDmitry Monakhov io = list_entry(unwritten.next, ext4_io_end_t, list); 17228a535f9SDmitry Monakhov BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN)); 17328a535f9SDmitry Monakhov list_del_init(&io->list); 17428a535f9SDmitry Monakhov 17528a535f9SDmitry Monakhov err = ext4_end_io(io); 17628a535f9SDmitry Monakhov if (unlikely(!ret && err)) 17728a535f9SDmitry Monakhov ret = err; 17828a535f9SDmitry Monakhov } 17928a535f9SDmitry Monakhov return ret; 18028a535f9SDmitry Monakhov } 18128a535f9SDmitry Monakhov 182bd2d0210STheodore Ts'o /* 183bd2d0210STheodore Ts'o * work on completed aio dio IO, to convert unwritten extents to extents 184bd2d0210STheodore Ts'o */ 18584c17543SJan Kara void ext4_end_io_work(struct work_struct *work) 186bd2d0210STheodore Ts'o { 18784c17543SJan Kara struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info, 18884c17543SJan Kara i_unwritten_work); 18984c17543SJan Kara ext4_do_flush_completed_IO(&ei->vfs_inode); 190d73d5046STao Ma } 191d73d5046STao Ma 192c278531dSDmitry Monakhov int ext4_flush_unwritten_io(struct inode *inode) 19328a535f9SDmitry Monakhov { 194c278531dSDmitry Monakhov int ret; 195c278531dSDmitry Monakhov WARN_ON_ONCE(!mutex_is_locked(&inode->i_mutex) && 196c278531dSDmitry Monakhov !(inode->i_state & I_FREEING)); 19784c17543SJan Kara ret = ext4_do_flush_completed_IO(inode); 198c278531dSDmitry Monakhov ext4_unwritten_wait(inode); 199c278531dSDmitry Monakhov return ret; 200bd2d0210STheodore Ts'o } 201bd2d0210STheodore Ts'o 202bd2d0210STheodore Ts'o ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags) 203bd2d0210STheodore Ts'o { 204b17b35ecSJesper Juhl ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags); 205bd2d0210STheodore Ts'o if (io) { 206f7ad6d2eSTheodore Ts'o atomic_inc(&EXT4_I(inode)->i_ioend_count); 207f7ad6d2eSTheodore Ts'o io->inode = inode; 208bd2d0210STheodore Ts'o INIT_LIST_HEAD(&io->list); 2094eec708dSJan Kara atomic_set(&io->count, 1); 210bd2d0210STheodore Ts'o } 211bd2d0210STheodore Ts'o return io; 212bd2d0210STheodore Ts'o } 213bd2d0210STheodore Ts'o 2144eec708dSJan Kara void ext4_put_io_end_defer(ext4_io_end_t *io_end) 2154eec708dSJan Kara { 2164eec708dSJan Kara if (atomic_dec_and_test(&io_end->count)) { 2174eec708dSJan Kara if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) || !io_end->size) { 2184eec708dSJan Kara ext4_release_io_end(io_end); 2194eec708dSJan Kara return; 2204eec708dSJan Kara } 2214eec708dSJan Kara ext4_add_complete_io(io_end); 2224eec708dSJan Kara } 2234eec708dSJan Kara } 2244eec708dSJan Kara 2254eec708dSJan Kara int ext4_put_io_end(ext4_io_end_t *io_end) 2264eec708dSJan Kara { 2274eec708dSJan Kara int err = 0; 2284eec708dSJan Kara 2294eec708dSJan Kara if (atomic_dec_and_test(&io_end->count)) { 2304eec708dSJan Kara if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 2314eec708dSJan Kara err = ext4_convert_unwritten_extents(io_end->inode, 2324eec708dSJan Kara io_end->offset, io_end->size); 2334eec708dSJan Kara ext4_clear_io_unwritten_flag(io_end); 2344eec708dSJan Kara } 2354eec708dSJan Kara ext4_release_io_end(io_end); 2364eec708dSJan Kara } 2374eec708dSJan Kara return err; 2384eec708dSJan Kara } 2394eec708dSJan Kara 2404eec708dSJan Kara ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end) 2414eec708dSJan Kara { 2424eec708dSJan Kara atomic_inc(&io_end->count); 2434eec708dSJan Kara return io_end; 2444eec708dSJan Kara } 2454eec708dSJan Kara 246bd2d0210STheodore Ts'o /* 247bd2d0210STheodore Ts'o * Print an buffer I/O error compatible with the fs/buffer.c. This 248bd2d0210STheodore Ts'o * provides compatibility with dmesg scrapers that look for a specific 249bd2d0210STheodore Ts'o * buffer I/O error message. We really need a unified error reporting 250bd2d0210STheodore Ts'o * structure to userspace ala Digital Unix's uerf system, but it's 251bd2d0210STheodore Ts'o * probably not going to happen in my lifetime, due to LKML politics... 252bd2d0210STheodore Ts'o */ 253bd2d0210STheodore Ts'o static void buffer_io_error(struct buffer_head *bh) 254bd2d0210STheodore Ts'o { 255bd2d0210STheodore Ts'o char b[BDEVNAME_SIZE]; 256bd2d0210STheodore Ts'o printk(KERN_ERR "Buffer I/O error on device %s, logical block %llu\n", 257bd2d0210STheodore Ts'o bdevname(bh->b_bdev, b), 258bd2d0210STheodore Ts'o (unsigned long long)bh->b_blocknr); 259bd2d0210STheodore Ts'o } 260bd2d0210STheodore Ts'o 261bd2d0210STheodore Ts'o static void ext4_end_bio(struct bio *bio, int error) 262bd2d0210STheodore Ts'o { 263bd2d0210STheodore Ts'o ext4_io_end_t *io_end = bio->bi_private; 264bd2d0210STheodore Ts'o struct inode *inode; 265bd2d0210STheodore Ts'o int i; 2660058f965SJan Kara int blocksize; 267d50bdd5aSCurt Wohlgemuth sector_t bi_sector = bio->bi_sector; 268bd2d0210STheodore Ts'o 269bd2d0210STheodore Ts'o BUG_ON(!io_end); 2700058f965SJan Kara inode = io_end->inode; 2710058f965SJan Kara blocksize = 1 << inode->i_blkbits; 272bd2d0210STheodore Ts'o bio->bi_private = NULL; 273bd2d0210STheodore Ts'o bio->bi_end_io = NULL; 274bd2d0210STheodore Ts'o if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 275bd2d0210STheodore Ts'o error = 0; 2760058f965SJan Kara for (i = 0; i < bio->bi_vcnt; i++) { 2770058f965SJan Kara struct bio_vec *bvec = &bio->bi_io_vec[i]; 2780058f965SJan Kara struct page *page = bvec->bv_page; 279bd2d0210STheodore Ts'o struct buffer_head *bh, *head; 2800058f965SJan Kara unsigned bio_start = bvec->bv_offset; 2810058f965SJan Kara unsigned bio_end = bio_start + bvec->bv_len; 2820058f965SJan Kara unsigned under_io = 0; 2830058f965SJan Kara unsigned long flags; 2840058f965SJan Kara 2850058f965SJan Kara if (!page) 2860058f965SJan Kara continue; 28739db00f1SCurt Wohlgemuth 28839db00f1SCurt Wohlgemuth if (error) { 28939db00f1SCurt Wohlgemuth SetPageError(page); 29039db00f1SCurt Wohlgemuth set_bit(AS_EIO, &page->mapping->flags); 2910058f965SJan Kara } 2920058f965SJan Kara bh = head = page_buffers(page); 2930058f965SJan Kara /* 2940058f965SJan Kara * We check all buffers in the page under BH_Uptodate_Lock 2950058f965SJan Kara * to avoid races with other end io clearing async_write flags 2960058f965SJan Kara */ 2970058f965SJan Kara local_irq_save(flags); 2980058f965SJan Kara bit_spin_lock(BH_Uptodate_Lock, &head->b_state); 299bd2d0210STheodore Ts'o do { 3000058f965SJan Kara if (bh_offset(bh) < bio_start || 3010058f965SJan Kara bh_offset(bh) + blocksize > bio_end) { 3020058f965SJan Kara if (buffer_async_write(bh)) 3030058f965SJan Kara under_io++; 3040058f965SJan Kara continue; 3050058f965SJan Kara } 3060058f965SJan Kara clear_buffer_async_write(bh); 3070058f965SJan Kara if (error) 308bd2d0210STheodore Ts'o buffer_io_error(bh); 3090058f965SJan Kara } while ((bh = bh->b_this_page) != head); 3100058f965SJan Kara bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); 3110058f965SJan Kara local_irq_restore(flags); 3120058f965SJan Kara if (!under_io) 3130058f965SJan Kara end_page_writeback(page); 314bd2d0210STheodore Ts'o } 3150058f965SJan Kara bio_put(bio); 316f7ad6d2eSTheodore Ts'o 317f7ad6d2eSTheodore Ts'o if (error) { 318f7ad6d2eSTheodore Ts'o io_end->flag |= EXT4_IO_END_ERROR; 319f7ad6d2eSTheodore Ts'o ext4_warning(inode->i_sb, "I/O error writing to inode %lu " 320f7ad6d2eSTheodore Ts'o "(offset %llu size %ld starting block %llu)", 321f7ad6d2eSTheodore Ts'o inode->i_ino, 322f7ad6d2eSTheodore Ts'o (unsigned long long) io_end->offset, 323f7ad6d2eSTheodore Ts'o (long) io_end->size, 324f7ad6d2eSTheodore Ts'o (unsigned long long) 325d50bdd5aSCurt Wohlgemuth bi_sector >> (inode->i_blkbits - 9)); 326f7ad6d2eSTheodore Ts'o } 327bd2d0210STheodore Ts'o 3284eec708dSJan Kara ext4_put_io_end_defer(io_end); 329bd2d0210STheodore Ts'o } 330bd2d0210STheodore Ts'o 331bd2d0210STheodore Ts'o void ext4_io_submit(struct ext4_io_submit *io) 332bd2d0210STheodore Ts'o { 333bd2d0210STheodore Ts'o struct bio *bio = io->io_bio; 334bd2d0210STheodore Ts'o 335bd2d0210STheodore Ts'o if (bio) { 336bd2d0210STheodore Ts'o bio_get(io->io_bio); 337bd2d0210STheodore Ts'o submit_bio(io->io_op, io->io_bio); 338bd2d0210STheodore Ts'o BUG_ON(bio_flagged(io->io_bio, BIO_EOPNOTSUPP)); 339bd2d0210STheodore Ts'o bio_put(io->io_bio); 340bd2d0210STheodore Ts'o } 3417dc57615SPeter Huewe io->io_bio = NULL; 3424eec708dSJan Kara } 3434eec708dSJan Kara 3444eec708dSJan Kara void ext4_io_submit_init(struct ext4_io_submit *io, 3454eec708dSJan Kara struct writeback_control *wbc) 3464eec708dSJan Kara { 3474eec708dSJan Kara io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE); 3484eec708dSJan Kara io->io_bio = NULL; 3497dc57615SPeter Huewe io->io_end = NULL; 350bd2d0210STheodore Ts'o } 351bd2d0210STheodore Ts'o 3524eec708dSJan Kara static int io_submit_init_bio(struct ext4_io_submit *io, 353bd2d0210STheodore Ts'o struct buffer_head *bh) 354bd2d0210STheodore Ts'o { 355bd2d0210STheodore Ts'o int nvecs = bio_get_nr_vecs(bh->b_bdev); 356bd2d0210STheodore Ts'o struct bio *bio; 357bd2d0210STheodore Ts'o 358275d3ba6STheodore Ts'o bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES)); 359bd2d0210STheodore Ts'o bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 360bd2d0210STheodore Ts'o bio->bi_bdev = bh->b_bdev; 361bd2d0210STheodore Ts'o bio->bi_end_io = ext4_end_bio; 3624eec708dSJan Kara bio->bi_private = ext4_get_io_end(io->io_end); 3634eec708dSJan Kara if (!io->io_end->size) 3644eec708dSJan Kara io->io_end->offset = (bh->b_page->index << PAGE_CACHE_SHIFT) 3654eec708dSJan Kara + bh_offset(bh); 366bd2d0210STheodore Ts'o io->io_bio = bio; 367bd2d0210STheodore Ts'o io->io_next_block = bh->b_blocknr; 368bd2d0210STheodore Ts'o return 0; 369bd2d0210STheodore Ts'o } 370bd2d0210STheodore Ts'o 371bd2d0210STheodore Ts'o static int io_submit_add_bh(struct ext4_io_submit *io, 372bd2d0210STheodore Ts'o struct inode *inode, 373bd2d0210STheodore Ts'o struct buffer_head *bh) 374bd2d0210STheodore Ts'o { 375bd2d0210STheodore Ts'o ext4_io_end_t *io_end; 376bd2d0210STheodore Ts'o int ret; 377bd2d0210STheodore Ts'o 378bd2d0210STheodore Ts'o if (io->io_bio && bh->b_blocknr != io->io_next_block) { 379bd2d0210STheodore Ts'o submit_and_retry: 380bd2d0210STheodore Ts'o ext4_io_submit(io); 381bd2d0210STheodore Ts'o } 382bd2d0210STheodore Ts'o if (io->io_bio == NULL) { 3834eec708dSJan Kara ret = io_submit_init_bio(io, bh); 384bd2d0210STheodore Ts'o if (ret) 385bd2d0210STheodore Ts'o return ret; 386bd2d0210STheodore Ts'o } 387bd2d0210STheodore Ts'o ret = bio_add_page(io->io_bio, bh->b_page, bh->b_size, bh_offset(bh)); 388bd2d0210STheodore Ts'o if (ret != bh->b_size) 389bd2d0210STheodore Ts'o goto submit_and_retry; 3904eec708dSJan Kara io_end = io->io_end; 3917b001d6aSJan Kara if (test_clear_buffer_uninit(bh)) 3924eec708dSJan Kara ext4_set_io_unwritten_flag(inode, io_end); 3934eec708dSJan Kara io_end->size += bh->b_size; 3944eec708dSJan Kara io->io_next_block++; 395bd2d0210STheodore Ts'o return 0; 396bd2d0210STheodore Ts'o } 397bd2d0210STheodore Ts'o 398bd2d0210STheodore Ts'o int ext4_bio_write_page(struct ext4_io_submit *io, 399bd2d0210STheodore Ts'o struct page *page, 400bd2d0210STheodore Ts'o int len, 401bd2d0210STheodore Ts'o struct writeback_control *wbc) 402bd2d0210STheodore Ts'o { 403bd2d0210STheodore Ts'o struct inode *inode = page->mapping->host; 4040058f965SJan Kara unsigned block_start, blocksize; 405bd2d0210STheodore Ts'o struct buffer_head *bh, *head; 406bd2d0210STheodore Ts'o int ret = 0; 4070058f965SJan Kara int nr_submitted = 0; 408bd2d0210STheodore Ts'o 409bd2d0210STheodore Ts'o blocksize = 1 << inode->i_blkbits; 410bd2d0210STheodore Ts'o 411d50bdd5aSCurt Wohlgemuth BUG_ON(!PageLocked(page)); 412bd2d0210STheodore Ts'o BUG_ON(PageWriteback(page)); 413bd2d0210STheodore Ts'o 414a54aa761STheodore Ts'o set_page_writeback(page); 415a54aa761STheodore Ts'o ClearPageError(page); 416bd2d0210STheodore Ts'o 4170058f965SJan Kara /* 4180058f965SJan Kara * In the first loop we prepare and mark buffers to submit. We have to 4190058f965SJan Kara * mark all buffers in the page before submitting so that 4200058f965SJan Kara * end_page_writeback() cannot be called from ext4_bio_end_io() when IO 4210058f965SJan Kara * on the first buffer finishes and we are still working on submitting 4220058f965SJan Kara * the second buffer. 4230058f965SJan Kara */ 4240058f965SJan Kara bh = head = page_buffers(page); 4250058f965SJan Kara do { 4260058f965SJan Kara block_start = bh_offset(bh); 427bd2d0210STheodore Ts'o if (block_start >= len) { 4285a0dc736SYongqiang Yang /* 4295a0dc736SYongqiang Yang * Comments copied from block_write_full_page_endio: 4305a0dc736SYongqiang Yang * 4315a0dc736SYongqiang Yang * The page straddles i_size. It must be zeroed out on 4325a0dc736SYongqiang Yang * each and every writepage invocation because it may 4335a0dc736SYongqiang Yang * be mmapped. "A file is mapped in multiples of the 4345a0dc736SYongqiang Yang * page size. For a file that is not a multiple of 4355a0dc736SYongqiang Yang * the page size, the remaining memory is zeroed when 4365a0dc736SYongqiang Yang * mapped, and writes to that region are not written 4375a0dc736SYongqiang Yang * out to the file." 4385a0dc736SYongqiang Yang */ 4390058f965SJan Kara zero_user_segment(page, block_start, 4400058f965SJan Kara block_start + blocksize); 441bd2d0210STheodore Ts'o clear_buffer_dirty(bh); 442bd2d0210STheodore Ts'o set_buffer_uptodate(bh); 443bd2d0210STheodore Ts'o continue; 444bd2d0210STheodore Ts'o } 4458a850c3fSJan Kara if (!buffer_dirty(bh) || buffer_delay(bh) || 4468a850c3fSJan Kara !buffer_mapped(bh) || buffer_unwritten(bh)) { 4478a850c3fSJan Kara /* A hole? We can safely clear the dirty bit */ 4488a850c3fSJan Kara if (!buffer_mapped(bh)) 4498a850c3fSJan Kara clear_buffer_dirty(bh); 4508a850c3fSJan Kara if (io->io_bio) 4518a850c3fSJan Kara ext4_io_submit(io); 4528a850c3fSJan Kara continue; 4538a850c3fSJan Kara } 4540058f965SJan Kara if (buffer_new(bh)) { 4550058f965SJan Kara clear_buffer_new(bh); 4560058f965SJan Kara unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr); 4570058f965SJan Kara } 4580058f965SJan Kara set_buffer_async_write(bh); 4590058f965SJan Kara } while ((bh = bh->b_this_page) != head); 4600058f965SJan Kara 4610058f965SJan Kara /* Now submit buffers to write */ 4620058f965SJan Kara bh = head = page_buffers(page); 4630058f965SJan Kara do { 4640058f965SJan Kara if (!buffer_async_write(bh)) 4650058f965SJan Kara continue; 4664eec708dSJan Kara ret = io_submit_add_bh(io, inode, bh); 467bd2d0210STheodore Ts'o if (ret) { 468bd2d0210STheodore Ts'o /* 469bd2d0210STheodore Ts'o * We only get here on ENOMEM. Not much else 470bd2d0210STheodore Ts'o * we can do but mark the page as dirty, and 471bd2d0210STheodore Ts'o * better luck next time. 472bd2d0210STheodore Ts'o */ 4731ae48a63SJan Kara redirty_page_for_writepage(wbc, page); 474bd2d0210STheodore Ts'o break; 475bd2d0210STheodore Ts'o } 4760058f965SJan Kara nr_submitted++; 4771ae48a63SJan Kara clear_buffer_dirty(bh); 4780058f965SJan Kara } while ((bh = bh->b_this_page) != head); 4790058f965SJan Kara 4800058f965SJan Kara /* Error stopped previous loop? Clean up buffers... */ 4810058f965SJan Kara if (ret) { 4820058f965SJan Kara do { 4830058f965SJan Kara clear_buffer_async_write(bh); 4840058f965SJan Kara bh = bh->b_this_page; 4850058f965SJan Kara } while (bh != head); 486bd2d0210STheodore Ts'o } 487bd2d0210STheodore Ts'o unlock_page(page); 4880058f965SJan Kara /* Nothing submitted - we have to end page writeback */ 4890058f965SJan Kara if (!nr_submitted) 4900058f965SJan Kara end_page_writeback(page); 491bd2d0210STheodore Ts'o return ret; 492bd2d0210STheodore Ts'o } 493