11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/fs/buffer.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 2002 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 81da177e4SLinus Torvalds * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * Removed a lot of unnecessary code and simplified things now that 111da177e4SLinus Torvalds * the buffer cache isn't our primary cache - Andrew Tridgell 12/96 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * Speed up hash, lru, and free list operations. Use gfp() for allocating 141da177e4SLinus Torvalds * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * Added 32k buffer block sizes - these are required older ARM systems. - RMK 171da177e4SLinus Torvalds * 181da177e4SLinus Torvalds * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de> 191da177e4SLinus Torvalds */ 201da177e4SLinus Torvalds 211da177e4SLinus Torvalds #include <linux/kernel.h> 221da177e4SLinus Torvalds #include <linux/syscalls.h> 231da177e4SLinus Torvalds #include <linux/fs.h> 241da177e4SLinus Torvalds #include <linux/mm.h> 251da177e4SLinus Torvalds #include <linux/percpu.h> 261da177e4SLinus Torvalds #include <linux/slab.h> 2716f7e0feSRandy Dunlap #include <linux/capability.h> 281da177e4SLinus Torvalds #include <linux/blkdev.h> 291da177e4SLinus Torvalds #include <linux/file.h> 301da177e4SLinus Torvalds #include <linux/quotaops.h> 311da177e4SLinus Torvalds #include <linux/highmem.h> 32630d9c47SPaul Gortmaker #include <linux/export.h> 331da177e4SLinus Torvalds #include <linux/writeback.h> 341da177e4SLinus Torvalds #include <linux/hash.h> 351da177e4SLinus Torvalds #include <linux/suspend.h> 361da177e4SLinus Torvalds #include <linux/buffer_head.h> 3755e829afSAndrew Morton #include <linux/task_io_accounting_ops.h> 381da177e4SLinus Torvalds #include <linux/bio.h> 391da177e4SLinus Torvalds #include <linux/notifier.h> 401da177e4SLinus Torvalds #include <linux/cpu.h> 411da177e4SLinus Torvalds #include <linux/bitops.h> 421da177e4SLinus Torvalds #include <linux/mpage.h> 43fb1c8f93SIngo Molnar #include <linux/bit_spinlock.h> 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); 461da177e4SLinus Torvalds 471da177e4SLinus Torvalds #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) 481da177e4SLinus Torvalds 491da177e4SLinus Torvalds inline void 501da177e4SLinus Torvalds init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private) 511da177e4SLinus Torvalds { 521da177e4SLinus Torvalds bh->b_end_io = handler; 531da177e4SLinus Torvalds bh->b_private = private; 541da177e4SLinus Torvalds } 551fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(init_buffer); 561da177e4SLinus Torvalds 577eaceaccSJens Axboe static int sleep_on_buffer(void *word) 581da177e4SLinus Torvalds { 591da177e4SLinus Torvalds io_schedule(); 601da177e4SLinus Torvalds return 0; 611da177e4SLinus Torvalds } 621da177e4SLinus Torvalds 63fc9b52cdSHarvey Harrison void __lock_buffer(struct buffer_head *bh) 641da177e4SLinus Torvalds { 657eaceaccSJens Axboe wait_on_bit_lock(&bh->b_state, BH_Lock, sleep_on_buffer, 661da177e4SLinus Torvalds TASK_UNINTERRUPTIBLE); 671da177e4SLinus Torvalds } 681da177e4SLinus Torvalds EXPORT_SYMBOL(__lock_buffer); 691da177e4SLinus Torvalds 70fc9b52cdSHarvey Harrison void unlock_buffer(struct buffer_head *bh) 711da177e4SLinus Torvalds { 7251b07fc3SNick Piggin clear_bit_unlock(BH_Lock, &bh->b_state); 731da177e4SLinus Torvalds smp_mb__after_clear_bit(); 741da177e4SLinus Torvalds wake_up_bit(&bh->b_state, BH_Lock); 751da177e4SLinus Torvalds } 761fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(unlock_buffer); 771da177e4SLinus Torvalds 781da177e4SLinus Torvalds /* 791da177e4SLinus Torvalds * Block until a buffer comes unlocked. This doesn't stop it 801da177e4SLinus Torvalds * from becoming locked again - you have to lock it yourself 811da177e4SLinus Torvalds * if you want to preserve its state. 821da177e4SLinus Torvalds */ 831da177e4SLinus Torvalds void __wait_on_buffer(struct buffer_head * bh) 841da177e4SLinus Torvalds { 857eaceaccSJens Axboe wait_on_bit(&bh->b_state, BH_Lock, sleep_on_buffer, TASK_UNINTERRUPTIBLE); 861da177e4SLinus Torvalds } 871fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(__wait_on_buffer); 881da177e4SLinus Torvalds 891da177e4SLinus Torvalds static void 901da177e4SLinus Torvalds __clear_page_buffers(struct page *page) 911da177e4SLinus Torvalds { 921da177e4SLinus Torvalds ClearPagePrivate(page); 934c21e2f2SHugh Dickins set_page_private(page, 0); 941da177e4SLinus Torvalds page_cache_release(page); 951da177e4SLinus Torvalds } 961da177e4SLinus Torvalds 9708bafc03SKeith Mannthey 9808bafc03SKeith Mannthey static int quiet_error(struct buffer_head *bh) 9908bafc03SKeith Mannthey { 10008bafc03SKeith Mannthey if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit()) 10108bafc03SKeith Mannthey return 0; 10208bafc03SKeith Mannthey return 1; 10308bafc03SKeith Mannthey } 10408bafc03SKeith Mannthey 10508bafc03SKeith Mannthey 1061da177e4SLinus Torvalds static void buffer_io_error(struct buffer_head *bh) 1071da177e4SLinus Torvalds { 1081da177e4SLinus Torvalds char b[BDEVNAME_SIZE]; 1091da177e4SLinus Torvalds printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n", 1101da177e4SLinus Torvalds bdevname(bh->b_bdev, b), 1111da177e4SLinus Torvalds (unsigned long long)bh->b_blocknr); 1121da177e4SLinus Torvalds } 1131da177e4SLinus Torvalds 1141da177e4SLinus Torvalds /* 11568671f35SDmitry Monakhov * End-of-IO handler helper function which does not touch the bh after 11668671f35SDmitry Monakhov * unlocking it. 11768671f35SDmitry Monakhov * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but 11868671f35SDmitry Monakhov * a race there is benign: unlock_buffer() only use the bh's address for 11968671f35SDmitry Monakhov * hashing after unlocking the buffer, so it doesn't actually touch the bh 12068671f35SDmitry Monakhov * itself. 1211da177e4SLinus Torvalds */ 12268671f35SDmitry Monakhov static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) 1231da177e4SLinus Torvalds { 1241da177e4SLinus Torvalds if (uptodate) { 1251da177e4SLinus Torvalds set_buffer_uptodate(bh); 1261da177e4SLinus Torvalds } else { 1271da177e4SLinus Torvalds /* This happens, due to failed READA attempts. */ 1281da177e4SLinus Torvalds clear_buffer_uptodate(bh); 1291da177e4SLinus Torvalds } 1301da177e4SLinus Torvalds unlock_buffer(bh); 13168671f35SDmitry Monakhov } 13268671f35SDmitry Monakhov 13368671f35SDmitry Monakhov /* 13468671f35SDmitry Monakhov * Default synchronous end-of-IO handler.. Just mark it up-to-date and 13568671f35SDmitry Monakhov * unlock the buffer. This is what ll_rw_block uses too. 13668671f35SDmitry Monakhov */ 13768671f35SDmitry Monakhov void end_buffer_read_sync(struct buffer_head *bh, int uptodate) 13868671f35SDmitry Monakhov { 13968671f35SDmitry Monakhov __end_buffer_read_notouch(bh, uptodate); 1401da177e4SLinus Torvalds put_bh(bh); 1411da177e4SLinus Torvalds } 1421fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(end_buffer_read_sync); 1431da177e4SLinus Torvalds 1441da177e4SLinus Torvalds void end_buffer_write_sync(struct buffer_head *bh, int uptodate) 1451da177e4SLinus Torvalds { 1461da177e4SLinus Torvalds char b[BDEVNAME_SIZE]; 1471da177e4SLinus Torvalds 1481da177e4SLinus Torvalds if (uptodate) { 1491da177e4SLinus Torvalds set_buffer_uptodate(bh); 1501da177e4SLinus Torvalds } else { 1510edd55faSChristoph Hellwig if (!quiet_error(bh)) { 1521da177e4SLinus Torvalds buffer_io_error(bh); 1531da177e4SLinus Torvalds printk(KERN_WARNING "lost page write due to " 1541da177e4SLinus Torvalds "I/O error on %s\n", 1551da177e4SLinus Torvalds bdevname(bh->b_bdev, b)); 1561da177e4SLinus Torvalds } 1571da177e4SLinus Torvalds set_buffer_write_io_error(bh); 1581da177e4SLinus Torvalds clear_buffer_uptodate(bh); 1591da177e4SLinus Torvalds } 1601da177e4SLinus Torvalds unlock_buffer(bh); 1611da177e4SLinus Torvalds put_bh(bh); 1621da177e4SLinus Torvalds } 1631fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(end_buffer_write_sync); 1641da177e4SLinus Torvalds 1651da177e4SLinus Torvalds /* 1661da177e4SLinus Torvalds * Various filesystems appear to want __find_get_block to be non-blocking. 1671da177e4SLinus Torvalds * But it's the page lock which protects the buffers. To get around this, 1681da177e4SLinus Torvalds * we get exclusion from try_to_free_buffers with the blockdev mapping's 1691da177e4SLinus Torvalds * private_lock. 1701da177e4SLinus Torvalds * 1711da177e4SLinus Torvalds * Hack idea: for the blockdev mapping, i_bufferlist_lock contention 1721da177e4SLinus Torvalds * may be quite high. This code could TryLock the page, and if that 1731da177e4SLinus Torvalds * succeeds, there is no need to take private_lock. (But if 1741da177e4SLinus Torvalds * private_lock is contended then so is mapping->tree_lock). 1751da177e4SLinus Torvalds */ 1761da177e4SLinus Torvalds static struct buffer_head * 177385fd4c5SCoywolf Qi Hunt __find_get_block_slow(struct block_device *bdev, sector_t block) 1781da177e4SLinus Torvalds { 1791da177e4SLinus Torvalds struct inode *bd_inode = bdev->bd_inode; 1801da177e4SLinus Torvalds struct address_space *bd_mapping = bd_inode->i_mapping; 1811da177e4SLinus Torvalds struct buffer_head *ret = NULL; 1821da177e4SLinus Torvalds pgoff_t index; 1831da177e4SLinus Torvalds struct buffer_head *bh; 1841da177e4SLinus Torvalds struct buffer_head *head; 1851da177e4SLinus Torvalds struct page *page; 1861da177e4SLinus Torvalds int all_mapped = 1; 1871da177e4SLinus Torvalds 1881da177e4SLinus Torvalds index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits); 1891da177e4SLinus Torvalds page = find_get_page(bd_mapping, index); 1901da177e4SLinus Torvalds if (!page) 1911da177e4SLinus Torvalds goto out; 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds spin_lock(&bd_mapping->private_lock); 1941da177e4SLinus Torvalds if (!page_has_buffers(page)) 1951da177e4SLinus Torvalds goto out_unlock; 1961da177e4SLinus Torvalds head = page_buffers(page); 1971da177e4SLinus Torvalds bh = head; 1981da177e4SLinus Torvalds do { 19997f76d3dSNikanth Karthikesan if (!buffer_mapped(bh)) 20097f76d3dSNikanth Karthikesan all_mapped = 0; 20197f76d3dSNikanth Karthikesan else if (bh->b_blocknr == block) { 2021da177e4SLinus Torvalds ret = bh; 2031da177e4SLinus Torvalds get_bh(bh); 2041da177e4SLinus Torvalds goto out_unlock; 2051da177e4SLinus Torvalds } 2061da177e4SLinus Torvalds bh = bh->b_this_page; 2071da177e4SLinus Torvalds } while (bh != head); 2081da177e4SLinus Torvalds 2091da177e4SLinus Torvalds /* we might be here because some of the buffers on this page are 2101da177e4SLinus Torvalds * not mapped. This is due to various races between 2111da177e4SLinus Torvalds * file io on the block device and getblk. It gets dealt with 2121da177e4SLinus Torvalds * elsewhere, don't buffer_error if we had some unmapped buffers 2131da177e4SLinus Torvalds */ 2141da177e4SLinus Torvalds if (all_mapped) { 21572a2ebd8STao Ma char b[BDEVNAME_SIZE]; 21672a2ebd8STao Ma 2171da177e4SLinus Torvalds printk("__find_get_block_slow() failed. " 2181da177e4SLinus Torvalds "block=%llu, b_blocknr=%llu\n", 219205f87f6SBadari Pulavarty (unsigned long long)block, 220205f87f6SBadari Pulavarty (unsigned long long)bh->b_blocknr); 221205f87f6SBadari Pulavarty printk("b_state=0x%08lx, b_size=%zu\n", 222205f87f6SBadari Pulavarty bh->b_state, bh->b_size); 22372a2ebd8STao Ma printk("device %s blocksize: %d\n", bdevname(bdev, b), 22472a2ebd8STao Ma 1 << bd_inode->i_blkbits); 2251da177e4SLinus Torvalds } 2261da177e4SLinus Torvalds out_unlock: 2271da177e4SLinus Torvalds spin_unlock(&bd_mapping->private_lock); 2281da177e4SLinus Torvalds page_cache_release(page); 2291da177e4SLinus Torvalds out: 2301da177e4SLinus Torvalds return ret; 2311da177e4SLinus Torvalds } 2321da177e4SLinus Torvalds 2331da177e4SLinus Torvalds /* 2345b0830cbSJens Axboe * Kick the writeback threads then try to free up some ZONE_NORMAL memory. 2351da177e4SLinus Torvalds */ 2361da177e4SLinus Torvalds static void free_more_memory(void) 2371da177e4SLinus Torvalds { 23819770b32SMel Gorman struct zone *zone; 2390e88460dSMel Gorman int nid; 2401da177e4SLinus Torvalds 2410e175a18SCurt Wohlgemuth wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM); 2421da177e4SLinus Torvalds yield(); 2431da177e4SLinus Torvalds 2440e88460dSMel Gorman for_each_online_node(nid) { 24519770b32SMel Gorman (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS), 24619770b32SMel Gorman gfp_zone(GFP_NOFS), NULL, 24719770b32SMel Gorman &zone); 24819770b32SMel Gorman if (zone) 24954a6eb5cSMel Gorman try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0, 250327c0e96SKAMEZAWA Hiroyuki GFP_NOFS, NULL); 2511da177e4SLinus Torvalds } 2521da177e4SLinus Torvalds } 2531da177e4SLinus Torvalds 2541da177e4SLinus Torvalds /* 2551da177e4SLinus Torvalds * I/O completion handler for block_read_full_page() - pages 2561da177e4SLinus Torvalds * which come unlocked at the end of I/O. 2571da177e4SLinus Torvalds */ 2581da177e4SLinus Torvalds static void end_buffer_async_read(struct buffer_head *bh, int uptodate) 2591da177e4SLinus Torvalds { 2601da177e4SLinus Torvalds unsigned long flags; 261a3972203SNick Piggin struct buffer_head *first; 2621da177e4SLinus Torvalds struct buffer_head *tmp; 2631da177e4SLinus Torvalds struct page *page; 2641da177e4SLinus Torvalds int page_uptodate = 1; 2651da177e4SLinus Torvalds 2661da177e4SLinus Torvalds BUG_ON(!buffer_async_read(bh)); 2671da177e4SLinus Torvalds 2681da177e4SLinus Torvalds page = bh->b_page; 2691da177e4SLinus Torvalds if (uptodate) { 2701da177e4SLinus Torvalds set_buffer_uptodate(bh); 2711da177e4SLinus Torvalds } else { 2721da177e4SLinus Torvalds clear_buffer_uptodate(bh); 27308bafc03SKeith Mannthey if (!quiet_error(bh)) 2741da177e4SLinus Torvalds buffer_io_error(bh); 2751da177e4SLinus Torvalds SetPageError(page); 2761da177e4SLinus Torvalds } 2771da177e4SLinus Torvalds 2781da177e4SLinus Torvalds /* 2791da177e4SLinus Torvalds * Be _very_ careful from here on. Bad things can happen if 2801da177e4SLinus Torvalds * two buffer heads end IO at almost the same time and both 2811da177e4SLinus Torvalds * decide that the page is now completely done. 2821da177e4SLinus Torvalds */ 283a3972203SNick Piggin first = page_buffers(page); 284a3972203SNick Piggin local_irq_save(flags); 285a3972203SNick Piggin bit_spin_lock(BH_Uptodate_Lock, &first->b_state); 2861da177e4SLinus Torvalds clear_buffer_async_read(bh); 2871da177e4SLinus Torvalds unlock_buffer(bh); 2881da177e4SLinus Torvalds tmp = bh; 2891da177e4SLinus Torvalds do { 2901da177e4SLinus Torvalds if (!buffer_uptodate(tmp)) 2911da177e4SLinus Torvalds page_uptodate = 0; 2921da177e4SLinus Torvalds if (buffer_async_read(tmp)) { 2931da177e4SLinus Torvalds BUG_ON(!buffer_locked(tmp)); 2941da177e4SLinus Torvalds goto still_busy; 2951da177e4SLinus Torvalds } 2961da177e4SLinus Torvalds tmp = tmp->b_this_page; 2971da177e4SLinus Torvalds } while (tmp != bh); 298a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 299a3972203SNick Piggin local_irq_restore(flags); 3001da177e4SLinus Torvalds 3011da177e4SLinus Torvalds /* 3021da177e4SLinus Torvalds * If none of the buffers had errors and they are all 3031da177e4SLinus Torvalds * uptodate then we can set the page uptodate. 3041da177e4SLinus Torvalds */ 3051da177e4SLinus Torvalds if (page_uptodate && !PageError(page)) 3061da177e4SLinus Torvalds SetPageUptodate(page); 3071da177e4SLinus Torvalds unlock_page(page); 3081da177e4SLinus Torvalds return; 3091da177e4SLinus Torvalds 3101da177e4SLinus Torvalds still_busy: 311a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 312a3972203SNick Piggin local_irq_restore(flags); 3131da177e4SLinus Torvalds return; 3141da177e4SLinus Torvalds } 3151da177e4SLinus Torvalds 3161da177e4SLinus Torvalds /* 3171da177e4SLinus Torvalds * Completion handler for block_write_full_page() - pages which are unlocked 3181da177e4SLinus Torvalds * during I/O, and which have PageWriteback cleared upon I/O completion. 3191da177e4SLinus Torvalds */ 32035c80d5fSChris Mason void end_buffer_async_write(struct buffer_head *bh, int uptodate) 3211da177e4SLinus Torvalds { 3221da177e4SLinus Torvalds char b[BDEVNAME_SIZE]; 3231da177e4SLinus Torvalds unsigned long flags; 324a3972203SNick Piggin struct buffer_head *first; 3251da177e4SLinus Torvalds struct buffer_head *tmp; 3261da177e4SLinus Torvalds struct page *page; 3271da177e4SLinus Torvalds 3281da177e4SLinus Torvalds BUG_ON(!buffer_async_write(bh)); 3291da177e4SLinus Torvalds 3301da177e4SLinus Torvalds page = bh->b_page; 3311da177e4SLinus Torvalds if (uptodate) { 3321da177e4SLinus Torvalds set_buffer_uptodate(bh); 3331da177e4SLinus Torvalds } else { 33408bafc03SKeith Mannthey if (!quiet_error(bh)) { 3351da177e4SLinus Torvalds buffer_io_error(bh); 3361da177e4SLinus Torvalds printk(KERN_WARNING "lost page write due to " 3371da177e4SLinus Torvalds "I/O error on %s\n", 3381da177e4SLinus Torvalds bdevname(bh->b_bdev, b)); 3391da177e4SLinus Torvalds } 3401da177e4SLinus Torvalds set_bit(AS_EIO, &page->mapping->flags); 34158ff407bSJan Kara set_buffer_write_io_error(bh); 3421da177e4SLinus Torvalds clear_buffer_uptodate(bh); 3431da177e4SLinus Torvalds SetPageError(page); 3441da177e4SLinus Torvalds } 3451da177e4SLinus Torvalds 346a3972203SNick Piggin first = page_buffers(page); 347a3972203SNick Piggin local_irq_save(flags); 348a3972203SNick Piggin bit_spin_lock(BH_Uptodate_Lock, &first->b_state); 349a3972203SNick Piggin 3501da177e4SLinus Torvalds clear_buffer_async_write(bh); 3511da177e4SLinus Torvalds unlock_buffer(bh); 3521da177e4SLinus Torvalds tmp = bh->b_this_page; 3531da177e4SLinus Torvalds while (tmp != bh) { 3541da177e4SLinus Torvalds if (buffer_async_write(tmp)) { 3551da177e4SLinus Torvalds BUG_ON(!buffer_locked(tmp)); 3561da177e4SLinus Torvalds goto still_busy; 3571da177e4SLinus Torvalds } 3581da177e4SLinus Torvalds tmp = tmp->b_this_page; 3591da177e4SLinus Torvalds } 360a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 361a3972203SNick Piggin local_irq_restore(flags); 3621da177e4SLinus Torvalds end_page_writeback(page); 3631da177e4SLinus Torvalds return; 3641da177e4SLinus Torvalds 3651da177e4SLinus Torvalds still_busy: 366a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 367a3972203SNick Piggin local_irq_restore(flags); 3681da177e4SLinus Torvalds return; 3691da177e4SLinus Torvalds } 3701fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(end_buffer_async_write); 3711da177e4SLinus Torvalds 3721da177e4SLinus Torvalds /* 3731da177e4SLinus Torvalds * If a page's buffers are under async readin (end_buffer_async_read 3741da177e4SLinus Torvalds * completion) then there is a possibility that another thread of 3751da177e4SLinus Torvalds * control could lock one of the buffers after it has completed 3761da177e4SLinus Torvalds * but while some of the other buffers have not completed. This 3771da177e4SLinus Torvalds * locked buffer would confuse end_buffer_async_read() into not unlocking 3781da177e4SLinus Torvalds * the page. So the absence of BH_Async_Read tells end_buffer_async_read() 3791da177e4SLinus Torvalds * that this buffer is not under async I/O. 3801da177e4SLinus Torvalds * 3811da177e4SLinus Torvalds * The page comes unlocked when it has no locked buffer_async buffers 3821da177e4SLinus Torvalds * left. 3831da177e4SLinus Torvalds * 3841da177e4SLinus Torvalds * PageLocked prevents anyone starting new async I/O reads any of 3851da177e4SLinus Torvalds * the buffers. 3861da177e4SLinus Torvalds * 3871da177e4SLinus Torvalds * PageWriteback is used to prevent simultaneous writeout of the same 3881da177e4SLinus Torvalds * page. 3891da177e4SLinus Torvalds * 3901da177e4SLinus Torvalds * PageLocked prevents anyone from starting writeback of a page which is 3911da177e4SLinus Torvalds * under read I/O (PageWriteback is only ever set against a locked page). 3921da177e4SLinus Torvalds */ 3931da177e4SLinus Torvalds static void mark_buffer_async_read(struct buffer_head *bh) 3941da177e4SLinus Torvalds { 3951da177e4SLinus Torvalds bh->b_end_io = end_buffer_async_read; 3961da177e4SLinus Torvalds set_buffer_async_read(bh); 3971da177e4SLinus Torvalds } 3981da177e4SLinus Torvalds 3991fe72eaaSH Hartley Sweeten static void mark_buffer_async_write_endio(struct buffer_head *bh, 40035c80d5fSChris Mason bh_end_io_t *handler) 40135c80d5fSChris Mason { 40235c80d5fSChris Mason bh->b_end_io = handler; 40335c80d5fSChris Mason set_buffer_async_write(bh); 40435c80d5fSChris Mason } 40535c80d5fSChris Mason 4061da177e4SLinus Torvalds void mark_buffer_async_write(struct buffer_head *bh) 4071da177e4SLinus Torvalds { 40835c80d5fSChris Mason mark_buffer_async_write_endio(bh, end_buffer_async_write); 4091da177e4SLinus Torvalds } 4101da177e4SLinus Torvalds EXPORT_SYMBOL(mark_buffer_async_write); 4111da177e4SLinus Torvalds 4121da177e4SLinus Torvalds 4131da177e4SLinus Torvalds /* 4141da177e4SLinus Torvalds * fs/buffer.c contains helper functions for buffer-backed address space's 4151da177e4SLinus Torvalds * fsync functions. A common requirement for buffer-based filesystems is 4161da177e4SLinus Torvalds * that certain data from the backing blockdev needs to be written out for 4171da177e4SLinus Torvalds * a successful fsync(). For example, ext2 indirect blocks need to be 4181da177e4SLinus Torvalds * written back and waited upon before fsync() returns. 4191da177e4SLinus Torvalds * 4201da177e4SLinus Torvalds * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(), 4211da177e4SLinus Torvalds * inode_has_buffers() and invalidate_inode_buffers() are provided for the 4221da177e4SLinus Torvalds * management of a list of dependent buffers at ->i_mapping->private_list. 4231da177e4SLinus Torvalds * 4241da177e4SLinus Torvalds * Locking is a little subtle: try_to_free_buffers() will remove buffers 4251da177e4SLinus Torvalds * from their controlling inode's queue when they are being freed. But 4261da177e4SLinus Torvalds * try_to_free_buffers() will be operating against the *blockdev* mapping 4271da177e4SLinus Torvalds * at the time, not against the S_ISREG file which depends on those buffers. 4281da177e4SLinus Torvalds * So the locking for private_list is via the private_lock in the address_space 4291da177e4SLinus Torvalds * which backs the buffers. Which is different from the address_space 4301da177e4SLinus Torvalds * against which the buffers are listed. So for a particular address_space, 4311da177e4SLinus Torvalds * mapping->private_lock does *not* protect mapping->private_list! In fact, 4321da177e4SLinus Torvalds * mapping->private_list will always be protected by the backing blockdev's 4331da177e4SLinus Torvalds * ->private_lock. 4341da177e4SLinus Torvalds * 4351da177e4SLinus Torvalds * Which introduces a requirement: all buffers on an address_space's 4361da177e4SLinus Torvalds * ->private_list must be from the same address_space: the blockdev's. 4371da177e4SLinus Torvalds * 4381da177e4SLinus Torvalds * address_spaces which do not place buffers at ->private_list via these 4391da177e4SLinus Torvalds * utility functions are free to use private_lock and private_list for 4401da177e4SLinus Torvalds * whatever they want. The only requirement is that list_empty(private_list) 4411da177e4SLinus Torvalds * be true at clear_inode() time. 4421da177e4SLinus Torvalds * 4431da177e4SLinus Torvalds * FIXME: clear_inode should not call invalidate_inode_buffers(). The 4441da177e4SLinus Torvalds * filesystems should do that. invalidate_inode_buffers() should just go 4451da177e4SLinus Torvalds * BUG_ON(!list_empty). 4461da177e4SLinus Torvalds * 4471da177e4SLinus Torvalds * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should 4481da177e4SLinus Torvalds * take an address_space, not an inode. And it should be called 4491da177e4SLinus Torvalds * mark_buffer_dirty_fsync() to clearly define why those buffers are being 4501da177e4SLinus Torvalds * queued up. 4511da177e4SLinus Torvalds * 4521da177e4SLinus Torvalds * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the 4531da177e4SLinus Torvalds * list if it is already on a list. Because if the buffer is on a list, 4541da177e4SLinus Torvalds * it *must* already be on the right one. If not, the filesystem is being 4551da177e4SLinus Torvalds * silly. This will save a ton of locking. But first we have to ensure 4561da177e4SLinus Torvalds * that buffers are taken *off* the old inode's list when they are freed 4571da177e4SLinus Torvalds * (presumably in truncate). That requires careful auditing of all 4581da177e4SLinus Torvalds * filesystems (do it inside bforget()). It could also be done by bringing 4591da177e4SLinus Torvalds * b_inode back. 4601da177e4SLinus Torvalds */ 4611da177e4SLinus Torvalds 4621da177e4SLinus Torvalds /* 4631da177e4SLinus Torvalds * The buffer's backing address_space's private_lock must be held 4641da177e4SLinus Torvalds */ 465dbacefc9SThomas Petazzoni static void __remove_assoc_queue(struct buffer_head *bh) 4661da177e4SLinus Torvalds { 4671da177e4SLinus Torvalds list_del_init(&bh->b_assoc_buffers); 46858ff407bSJan Kara WARN_ON(!bh->b_assoc_map); 46958ff407bSJan Kara if (buffer_write_io_error(bh)) 47058ff407bSJan Kara set_bit(AS_EIO, &bh->b_assoc_map->flags); 47158ff407bSJan Kara bh->b_assoc_map = NULL; 4721da177e4SLinus Torvalds } 4731da177e4SLinus Torvalds 4741da177e4SLinus Torvalds int inode_has_buffers(struct inode *inode) 4751da177e4SLinus Torvalds { 4761da177e4SLinus Torvalds return !list_empty(&inode->i_data.private_list); 4771da177e4SLinus Torvalds } 4781da177e4SLinus Torvalds 4791da177e4SLinus Torvalds /* 4801da177e4SLinus Torvalds * osync is designed to support O_SYNC io. It waits synchronously for 4811da177e4SLinus Torvalds * all already-submitted IO to complete, but does not queue any new 4821da177e4SLinus Torvalds * writes to the disk. 4831da177e4SLinus Torvalds * 4841da177e4SLinus Torvalds * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as 4851da177e4SLinus Torvalds * you dirty the buffers, and then use osync_inode_buffers to wait for 4861da177e4SLinus Torvalds * completion. Any other dirty buffers which are not yet queued for 4871da177e4SLinus Torvalds * write will not be flushed to disk by the osync. 4881da177e4SLinus Torvalds */ 4891da177e4SLinus Torvalds static int osync_buffers_list(spinlock_t *lock, struct list_head *list) 4901da177e4SLinus Torvalds { 4911da177e4SLinus Torvalds struct buffer_head *bh; 4921da177e4SLinus Torvalds struct list_head *p; 4931da177e4SLinus Torvalds int err = 0; 4941da177e4SLinus Torvalds 4951da177e4SLinus Torvalds spin_lock(lock); 4961da177e4SLinus Torvalds repeat: 4971da177e4SLinus Torvalds list_for_each_prev(p, list) { 4981da177e4SLinus Torvalds bh = BH_ENTRY(p); 4991da177e4SLinus Torvalds if (buffer_locked(bh)) { 5001da177e4SLinus Torvalds get_bh(bh); 5011da177e4SLinus Torvalds spin_unlock(lock); 5021da177e4SLinus Torvalds wait_on_buffer(bh); 5031da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 5041da177e4SLinus Torvalds err = -EIO; 5051da177e4SLinus Torvalds brelse(bh); 5061da177e4SLinus Torvalds spin_lock(lock); 5071da177e4SLinus Torvalds goto repeat; 5081da177e4SLinus Torvalds } 5091da177e4SLinus Torvalds } 5101da177e4SLinus Torvalds spin_unlock(lock); 5111da177e4SLinus Torvalds return err; 5121da177e4SLinus Torvalds } 5131da177e4SLinus Torvalds 51401a05b33SAl Viro static void do_thaw_one(struct super_block *sb, void *unused) 515c2d75438SEric Sandeen { 516c2d75438SEric Sandeen char b[BDEVNAME_SIZE]; 517c2d75438SEric Sandeen while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb)) 518c2d75438SEric Sandeen printk(KERN_WARNING "Emergency Thaw on %s\n", 519c2d75438SEric Sandeen bdevname(sb->s_bdev, b)); 520c2d75438SEric Sandeen } 52101a05b33SAl Viro 52201a05b33SAl Viro static void do_thaw_all(struct work_struct *work) 52301a05b33SAl Viro { 52401a05b33SAl Viro iterate_supers(do_thaw_one, NULL); 525053c525fSJens Axboe kfree(work); 526c2d75438SEric Sandeen printk(KERN_WARNING "Emergency Thaw complete\n"); 527c2d75438SEric Sandeen } 528c2d75438SEric Sandeen 529c2d75438SEric Sandeen /** 530c2d75438SEric Sandeen * emergency_thaw_all -- forcibly thaw every frozen filesystem 531c2d75438SEric Sandeen * 532c2d75438SEric Sandeen * Used for emergency unfreeze of all filesystems via SysRq 533c2d75438SEric Sandeen */ 534c2d75438SEric Sandeen void emergency_thaw_all(void) 535c2d75438SEric Sandeen { 536053c525fSJens Axboe struct work_struct *work; 537053c525fSJens Axboe 538053c525fSJens Axboe work = kmalloc(sizeof(*work), GFP_ATOMIC); 539053c525fSJens Axboe if (work) { 540053c525fSJens Axboe INIT_WORK(work, do_thaw_all); 541053c525fSJens Axboe schedule_work(work); 542053c525fSJens Axboe } 543c2d75438SEric Sandeen } 544c2d75438SEric Sandeen 5451da177e4SLinus Torvalds /** 54678a4a50aSRandy Dunlap * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers 54767be2dd1SMartin Waitz * @mapping: the mapping which wants those buffers written 5481da177e4SLinus Torvalds * 5491da177e4SLinus Torvalds * Starts I/O against the buffers at mapping->private_list, and waits upon 5501da177e4SLinus Torvalds * that I/O. 5511da177e4SLinus Torvalds * 55267be2dd1SMartin Waitz * Basically, this is a convenience function for fsync(). 55367be2dd1SMartin Waitz * @mapping is a file or directory which needs those buffers to be written for 55467be2dd1SMartin Waitz * a successful fsync(). 5551da177e4SLinus Torvalds */ 5561da177e4SLinus Torvalds int sync_mapping_buffers(struct address_space *mapping) 5571da177e4SLinus Torvalds { 5581da177e4SLinus Torvalds struct address_space *buffer_mapping = mapping->assoc_mapping; 5591da177e4SLinus Torvalds 5601da177e4SLinus Torvalds if (buffer_mapping == NULL || list_empty(&mapping->private_list)) 5611da177e4SLinus Torvalds return 0; 5621da177e4SLinus Torvalds 5631da177e4SLinus Torvalds return fsync_buffers_list(&buffer_mapping->private_lock, 5641da177e4SLinus Torvalds &mapping->private_list); 5651da177e4SLinus Torvalds } 5661da177e4SLinus Torvalds EXPORT_SYMBOL(sync_mapping_buffers); 5671da177e4SLinus Torvalds 5681da177e4SLinus Torvalds /* 5691da177e4SLinus Torvalds * Called when we've recently written block `bblock', and it is known that 5701da177e4SLinus Torvalds * `bblock' was for a buffer_boundary() buffer. This means that the block at 5711da177e4SLinus Torvalds * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's 5721da177e4SLinus Torvalds * dirty, schedule it for IO. So that indirects merge nicely with their data. 5731da177e4SLinus Torvalds */ 5741da177e4SLinus Torvalds void write_boundary_block(struct block_device *bdev, 5751da177e4SLinus Torvalds sector_t bblock, unsigned blocksize) 5761da177e4SLinus Torvalds { 5771da177e4SLinus Torvalds struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize); 5781da177e4SLinus Torvalds if (bh) { 5791da177e4SLinus Torvalds if (buffer_dirty(bh)) 5801da177e4SLinus Torvalds ll_rw_block(WRITE, 1, &bh); 5811da177e4SLinus Torvalds put_bh(bh); 5821da177e4SLinus Torvalds } 5831da177e4SLinus Torvalds } 5841da177e4SLinus Torvalds 5851da177e4SLinus Torvalds void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) 5861da177e4SLinus Torvalds { 5871da177e4SLinus Torvalds struct address_space *mapping = inode->i_mapping; 5881da177e4SLinus Torvalds struct address_space *buffer_mapping = bh->b_page->mapping; 5891da177e4SLinus Torvalds 5901da177e4SLinus Torvalds mark_buffer_dirty(bh); 5911da177e4SLinus Torvalds if (!mapping->assoc_mapping) { 5921da177e4SLinus Torvalds mapping->assoc_mapping = buffer_mapping; 5931da177e4SLinus Torvalds } else { 594e827f923SEric Sesterhenn BUG_ON(mapping->assoc_mapping != buffer_mapping); 5951da177e4SLinus Torvalds } 596535ee2fbSJan Kara if (!bh->b_assoc_map) { 5971da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 5981da177e4SLinus Torvalds list_move_tail(&bh->b_assoc_buffers, 5991da177e4SLinus Torvalds &mapping->private_list); 60058ff407bSJan Kara bh->b_assoc_map = mapping; 6011da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 6021da177e4SLinus Torvalds } 6031da177e4SLinus Torvalds } 6041da177e4SLinus Torvalds EXPORT_SYMBOL(mark_buffer_dirty_inode); 6051da177e4SLinus Torvalds 6061da177e4SLinus Torvalds /* 607787d2214SNick Piggin * Mark the page dirty, and set it dirty in the radix tree, and mark the inode 608787d2214SNick Piggin * dirty. 609787d2214SNick Piggin * 610787d2214SNick Piggin * If warn is true, then emit a warning if the page is not uptodate and has 611787d2214SNick Piggin * not been truncated. 612787d2214SNick Piggin */ 613a8e7d49aSLinus Torvalds static void __set_page_dirty(struct page *page, 614787d2214SNick Piggin struct address_space *mapping, int warn) 615787d2214SNick Piggin { 61619fd6231SNick Piggin spin_lock_irq(&mapping->tree_lock); 617787d2214SNick Piggin if (page->mapping) { /* Race with truncate? */ 618787d2214SNick Piggin WARN_ON_ONCE(warn && !PageUptodate(page)); 619e3a7cca1SEdward Shishkin account_page_dirtied(page, mapping); 620787d2214SNick Piggin radix_tree_tag_set(&mapping->page_tree, 621787d2214SNick Piggin page_index(page), PAGECACHE_TAG_DIRTY); 622787d2214SNick Piggin } 62319fd6231SNick Piggin spin_unlock_irq(&mapping->tree_lock); 624787d2214SNick Piggin __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 625787d2214SNick Piggin } 626787d2214SNick Piggin 627787d2214SNick Piggin /* 6281da177e4SLinus Torvalds * Add a page to the dirty page list. 6291da177e4SLinus Torvalds * 6301da177e4SLinus Torvalds * It is a sad fact of life that this function is called from several places 6311da177e4SLinus Torvalds * deeply under spinlocking. It may not sleep. 6321da177e4SLinus Torvalds * 6331da177e4SLinus Torvalds * If the page has buffers, the uptodate buffers are set dirty, to preserve 6341da177e4SLinus Torvalds * dirty-state coherency between the page and the buffers. It the page does 6351da177e4SLinus Torvalds * not have buffers then when they are later attached they will all be set 6361da177e4SLinus Torvalds * dirty. 6371da177e4SLinus Torvalds * 6381da177e4SLinus Torvalds * The buffers are dirtied before the page is dirtied. There's a small race 6391da177e4SLinus Torvalds * window in which a writepage caller may see the page cleanness but not the 6401da177e4SLinus Torvalds * buffer dirtiness. That's fine. If this code were to set the page dirty 6411da177e4SLinus Torvalds * before the buffers, a concurrent writepage caller could clear the page dirty 6421da177e4SLinus Torvalds * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean 6431da177e4SLinus Torvalds * page on the dirty page list. 6441da177e4SLinus Torvalds * 6451da177e4SLinus Torvalds * We use private_lock to lock against try_to_free_buffers while using the 6461da177e4SLinus Torvalds * page's buffer list. Also use this to protect against clean buffers being 6471da177e4SLinus Torvalds * added to the page after it was set dirty. 6481da177e4SLinus Torvalds * 6491da177e4SLinus Torvalds * FIXME: may need to call ->reservepage here as well. That's rather up to the 6501da177e4SLinus Torvalds * address_space though. 6511da177e4SLinus Torvalds */ 6521da177e4SLinus Torvalds int __set_page_dirty_buffers(struct page *page) 6531da177e4SLinus Torvalds { 654a8e7d49aSLinus Torvalds int newly_dirty; 655787d2214SNick Piggin struct address_space *mapping = page_mapping(page); 656ebf7a227SNick Piggin 657ebf7a227SNick Piggin if (unlikely(!mapping)) 658ebf7a227SNick Piggin return !TestSetPageDirty(page); 6591da177e4SLinus Torvalds 6601da177e4SLinus Torvalds spin_lock(&mapping->private_lock); 6611da177e4SLinus Torvalds if (page_has_buffers(page)) { 6621da177e4SLinus Torvalds struct buffer_head *head = page_buffers(page); 6631da177e4SLinus Torvalds struct buffer_head *bh = head; 6641da177e4SLinus Torvalds 6651da177e4SLinus Torvalds do { 6661da177e4SLinus Torvalds set_buffer_dirty(bh); 6671da177e4SLinus Torvalds bh = bh->b_this_page; 6681da177e4SLinus Torvalds } while (bh != head); 6691da177e4SLinus Torvalds } 670a8e7d49aSLinus Torvalds newly_dirty = !TestSetPageDirty(page); 6711da177e4SLinus Torvalds spin_unlock(&mapping->private_lock); 6721da177e4SLinus Torvalds 673a8e7d49aSLinus Torvalds if (newly_dirty) 674a8e7d49aSLinus Torvalds __set_page_dirty(page, mapping, 1); 675a8e7d49aSLinus Torvalds return newly_dirty; 6761da177e4SLinus Torvalds } 6771da177e4SLinus Torvalds EXPORT_SYMBOL(__set_page_dirty_buffers); 6781da177e4SLinus Torvalds 6791da177e4SLinus Torvalds /* 6801da177e4SLinus Torvalds * Write out and wait upon a list of buffers. 6811da177e4SLinus Torvalds * 6821da177e4SLinus Torvalds * We have conflicting pressures: we want to make sure that all 6831da177e4SLinus Torvalds * initially dirty buffers get waited on, but that any subsequently 6841da177e4SLinus Torvalds * dirtied buffers don't. After all, we don't want fsync to last 6851da177e4SLinus Torvalds * forever if somebody is actively writing to the file. 6861da177e4SLinus Torvalds * 6871da177e4SLinus Torvalds * Do this in two main stages: first we copy dirty buffers to a 6881da177e4SLinus Torvalds * temporary inode list, queueing the writes as we go. Then we clean 6891da177e4SLinus Torvalds * up, waiting for those writes to complete. 6901da177e4SLinus Torvalds * 6911da177e4SLinus Torvalds * During this second stage, any subsequent updates to the file may end 6921da177e4SLinus Torvalds * up refiling the buffer on the original inode's dirty list again, so 6931da177e4SLinus Torvalds * there is a chance we will end up with a buffer queued for write but 6941da177e4SLinus Torvalds * not yet completed on that list. So, as a final cleanup we go through 6951da177e4SLinus Torvalds * the osync code to catch these locked, dirty buffers without requeuing 6961da177e4SLinus Torvalds * any newly dirty buffers for write. 6971da177e4SLinus Torvalds */ 6981da177e4SLinus Torvalds static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) 6991da177e4SLinus Torvalds { 7001da177e4SLinus Torvalds struct buffer_head *bh; 7011da177e4SLinus Torvalds struct list_head tmp; 7027eaceaccSJens Axboe struct address_space *mapping; 7031da177e4SLinus Torvalds int err = 0, err2; 7044ee2491eSJens Axboe struct blk_plug plug; 7051da177e4SLinus Torvalds 7061da177e4SLinus Torvalds INIT_LIST_HEAD(&tmp); 7074ee2491eSJens Axboe blk_start_plug(&plug); 7081da177e4SLinus Torvalds 7091da177e4SLinus Torvalds spin_lock(lock); 7101da177e4SLinus Torvalds while (!list_empty(list)) { 7111da177e4SLinus Torvalds bh = BH_ENTRY(list->next); 712535ee2fbSJan Kara mapping = bh->b_assoc_map; 71358ff407bSJan Kara __remove_assoc_queue(bh); 714535ee2fbSJan Kara /* Avoid race with mark_buffer_dirty_inode() which does 715535ee2fbSJan Kara * a lockless check and we rely on seeing the dirty bit */ 716535ee2fbSJan Kara smp_mb(); 7171da177e4SLinus Torvalds if (buffer_dirty(bh) || buffer_locked(bh)) { 7181da177e4SLinus Torvalds list_add(&bh->b_assoc_buffers, &tmp); 719535ee2fbSJan Kara bh->b_assoc_map = mapping; 7201da177e4SLinus Torvalds if (buffer_dirty(bh)) { 7211da177e4SLinus Torvalds get_bh(bh); 7221da177e4SLinus Torvalds spin_unlock(lock); 7231da177e4SLinus Torvalds /* 7241da177e4SLinus Torvalds * Ensure any pending I/O completes so that 7259cb569d6SChristoph Hellwig * write_dirty_buffer() actually writes the 7269cb569d6SChristoph Hellwig * current contents - it is a noop if I/O is 7279cb569d6SChristoph Hellwig * still in flight on potentially older 7289cb569d6SChristoph Hellwig * contents. 7291da177e4SLinus Torvalds */ 730721a9602SJens Axboe write_dirty_buffer(bh, WRITE_SYNC); 7319cf6b720SJens Axboe 7329cf6b720SJens Axboe /* 7339cf6b720SJens Axboe * Kick off IO for the previous mapping. Note 7349cf6b720SJens Axboe * that we will not run the very last mapping, 7359cf6b720SJens Axboe * wait_on_buffer() will do that for us 7369cf6b720SJens Axboe * through sync_buffer(). 7379cf6b720SJens Axboe */ 7381da177e4SLinus Torvalds brelse(bh); 7391da177e4SLinus Torvalds spin_lock(lock); 7401da177e4SLinus Torvalds } 7411da177e4SLinus Torvalds } 7421da177e4SLinus Torvalds } 7431da177e4SLinus Torvalds 7444ee2491eSJens Axboe spin_unlock(lock); 7454ee2491eSJens Axboe blk_finish_plug(&plug); 7464ee2491eSJens Axboe spin_lock(lock); 7474ee2491eSJens Axboe 7481da177e4SLinus Torvalds while (!list_empty(&tmp)) { 7491da177e4SLinus Torvalds bh = BH_ENTRY(tmp.prev); 7501da177e4SLinus Torvalds get_bh(bh); 751535ee2fbSJan Kara mapping = bh->b_assoc_map; 752535ee2fbSJan Kara __remove_assoc_queue(bh); 753535ee2fbSJan Kara /* Avoid race with mark_buffer_dirty_inode() which does 754535ee2fbSJan Kara * a lockless check and we rely on seeing the dirty bit */ 755535ee2fbSJan Kara smp_mb(); 756535ee2fbSJan Kara if (buffer_dirty(bh)) { 757535ee2fbSJan Kara list_add(&bh->b_assoc_buffers, 758e3892296SJan Kara &mapping->private_list); 759535ee2fbSJan Kara bh->b_assoc_map = mapping; 760535ee2fbSJan Kara } 7611da177e4SLinus Torvalds spin_unlock(lock); 7621da177e4SLinus Torvalds wait_on_buffer(bh); 7631da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 7641da177e4SLinus Torvalds err = -EIO; 7651da177e4SLinus Torvalds brelse(bh); 7661da177e4SLinus Torvalds spin_lock(lock); 7671da177e4SLinus Torvalds } 7681da177e4SLinus Torvalds 7691da177e4SLinus Torvalds spin_unlock(lock); 7701da177e4SLinus Torvalds err2 = osync_buffers_list(lock, list); 7711da177e4SLinus Torvalds if (err) 7721da177e4SLinus Torvalds return err; 7731da177e4SLinus Torvalds else 7741da177e4SLinus Torvalds return err2; 7751da177e4SLinus Torvalds } 7761da177e4SLinus Torvalds 7771da177e4SLinus Torvalds /* 7781da177e4SLinus Torvalds * Invalidate any and all dirty buffers on a given inode. We are 7791da177e4SLinus Torvalds * probably unmounting the fs, but that doesn't mean we have already 7801da177e4SLinus Torvalds * done a sync(). Just drop the buffers from the inode list. 7811da177e4SLinus Torvalds * 7821da177e4SLinus Torvalds * NOTE: we take the inode's blockdev's mapping's private_lock. Which 7831da177e4SLinus Torvalds * assumes that all the buffers are against the blockdev. Not true 7841da177e4SLinus Torvalds * for reiserfs. 7851da177e4SLinus Torvalds */ 7861da177e4SLinus Torvalds void invalidate_inode_buffers(struct inode *inode) 7871da177e4SLinus Torvalds { 7881da177e4SLinus Torvalds if (inode_has_buffers(inode)) { 7891da177e4SLinus Torvalds struct address_space *mapping = &inode->i_data; 7901da177e4SLinus Torvalds struct list_head *list = &mapping->private_list; 7911da177e4SLinus Torvalds struct address_space *buffer_mapping = mapping->assoc_mapping; 7921da177e4SLinus Torvalds 7931da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 7941da177e4SLinus Torvalds while (!list_empty(list)) 7951da177e4SLinus Torvalds __remove_assoc_queue(BH_ENTRY(list->next)); 7961da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 7971da177e4SLinus Torvalds } 7981da177e4SLinus Torvalds } 79952b19ac9SJan Kara EXPORT_SYMBOL(invalidate_inode_buffers); 8001da177e4SLinus Torvalds 8011da177e4SLinus Torvalds /* 8021da177e4SLinus Torvalds * Remove any clean buffers from the inode's buffer list. This is called 8031da177e4SLinus Torvalds * when we're trying to free the inode itself. Those buffers can pin it. 8041da177e4SLinus Torvalds * 8051da177e4SLinus Torvalds * Returns true if all buffers were removed. 8061da177e4SLinus Torvalds */ 8071da177e4SLinus Torvalds int remove_inode_buffers(struct inode *inode) 8081da177e4SLinus Torvalds { 8091da177e4SLinus Torvalds int ret = 1; 8101da177e4SLinus Torvalds 8111da177e4SLinus Torvalds if (inode_has_buffers(inode)) { 8121da177e4SLinus Torvalds struct address_space *mapping = &inode->i_data; 8131da177e4SLinus Torvalds struct list_head *list = &mapping->private_list; 8141da177e4SLinus Torvalds struct address_space *buffer_mapping = mapping->assoc_mapping; 8151da177e4SLinus Torvalds 8161da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 8171da177e4SLinus Torvalds while (!list_empty(list)) { 8181da177e4SLinus Torvalds struct buffer_head *bh = BH_ENTRY(list->next); 8191da177e4SLinus Torvalds if (buffer_dirty(bh)) { 8201da177e4SLinus Torvalds ret = 0; 8211da177e4SLinus Torvalds break; 8221da177e4SLinus Torvalds } 8231da177e4SLinus Torvalds __remove_assoc_queue(bh); 8241da177e4SLinus Torvalds } 8251da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 8261da177e4SLinus Torvalds } 8271da177e4SLinus Torvalds return ret; 8281da177e4SLinus Torvalds } 8291da177e4SLinus Torvalds 8301da177e4SLinus Torvalds /* 8311da177e4SLinus Torvalds * Create the appropriate buffers when given a page for data area and 8321da177e4SLinus Torvalds * the size of each buffer.. Use the bh->b_this_page linked list to 8331da177e4SLinus Torvalds * follow the buffers created. Return NULL if unable to create more 8341da177e4SLinus Torvalds * buffers. 8351da177e4SLinus Torvalds * 8361da177e4SLinus Torvalds * The retry flag is used to differentiate async IO (paging, swapping) 8371da177e4SLinus Torvalds * which may not fail from ordinary buffer allocations. 8381da177e4SLinus Torvalds */ 8391da177e4SLinus Torvalds struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, 8401da177e4SLinus Torvalds int retry) 8411da177e4SLinus Torvalds { 8421da177e4SLinus Torvalds struct buffer_head *bh, *head; 8431da177e4SLinus Torvalds long offset; 8441da177e4SLinus Torvalds 8451da177e4SLinus Torvalds try_again: 8461da177e4SLinus Torvalds head = NULL; 8471da177e4SLinus Torvalds offset = PAGE_SIZE; 8481da177e4SLinus Torvalds while ((offset -= size) >= 0) { 8491da177e4SLinus Torvalds bh = alloc_buffer_head(GFP_NOFS); 8501da177e4SLinus Torvalds if (!bh) 8511da177e4SLinus Torvalds goto no_grow; 8521da177e4SLinus Torvalds 8531da177e4SLinus Torvalds bh->b_bdev = NULL; 8541da177e4SLinus Torvalds bh->b_this_page = head; 8551da177e4SLinus Torvalds bh->b_blocknr = -1; 8561da177e4SLinus Torvalds head = bh; 8571da177e4SLinus Torvalds 8581da177e4SLinus Torvalds bh->b_state = 0; 8591da177e4SLinus Torvalds atomic_set(&bh->b_count, 0); 8601da177e4SLinus Torvalds bh->b_size = size; 8611da177e4SLinus Torvalds 8621da177e4SLinus Torvalds /* Link the buffer to its page */ 8631da177e4SLinus Torvalds set_bh_page(bh, page, offset); 8641da177e4SLinus Torvalds 86501ffe339SNathan Scott init_buffer(bh, NULL, NULL); 8661da177e4SLinus Torvalds } 8671da177e4SLinus Torvalds return head; 8681da177e4SLinus Torvalds /* 8691da177e4SLinus Torvalds * In case anything failed, we just free everything we got. 8701da177e4SLinus Torvalds */ 8711da177e4SLinus Torvalds no_grow: 8721da177e4SLinus Torvalds if (head) { 8731da177e4SLinus Torvalds do { 8741da177e4SLinus Torvalds bh = head; 8751da177e4SLinus Torvalds head = head->b_this_page; 8761da177e4SLinus Torvalds free_buffer_head(bh); 8771da177e4SLinus Torvalds } while (head); 8781da177e4SLinus Torvalds } 8791da177e4SLinus Torvalds 8801da177e4SLinus Torvalds /* 8811da177e4SLinus Torvalds * Return failure for non-async IO requests. Async IO requests 8821da177e4SLinus Torvalds * are not allowed to fail, so we have to wait until buffer heads 8831da177e4SLinus Torvalds * become available. But we don't want tasks sleeping with 8841da177e4SLinus Torvalds * partially complete buffers, so all were released above. 8851da177e4SLinus Torvalds */ 8861da177e4SLinus Torvalds if (!retry) 8871da177e4SLinus Torvalds return NULL; 8881da177e4SLinus Torvalds 8891da177e4SLinus Torvalds /* We're _really_ low on memory. Now we just 8901da177e4SLinus Torvalds * wait for old buffer heads to become free due to 8911da177e4SLinus Torvalds * finishing IO. Since this is an async request and 8921da177e4SLinus Torvalds * the reserve list is empty, we're sure there are 8931da177e4SLinus Torvalds * async buffer heads in use. 8941da177e4SLinus Torvalds */ 8951da177e4SLinus Torvalds free_more_memory(); 8961da177e4SLinus Torvalds goto try_again; 8971da177e4SLinus Torvalds } 8981da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(alloc_page_buffers); 8991da177e4SLinus Torvalds 9001da177e4SLinus Torvalds static inline void 9011da177e4SLinus Torvalds link_dev_buffers(struct page *page, struct buffer_head *head) 9021da177e4SLinus Torvalds { 9031da177e4SLinus Torvalds struct buffer_head *bh, *tail; 9041da177e4SLinus Torvalds 9051da177e4SLinus Torvalds bh = head; 9061da177e4SLinus Torvalds do { 9071da177e4SLinus Torvalds tail = bh; 9081da177e4SLinus Torvalds bh = bh->b_this_page; 9091da177e4SLinus Torvalds } while (bh); 9101da177e4SLinus Torvalds tail->b_this_page = head; 9111da177e4SLinus Torvalds attach_page_buffers(page, head); 9121da177e4SLinus Torvalds } 9131da177e4SLinus Torvalds 9141da177e4SLinus Torvalds /* 9151da177e4SLinus Torvalds * Initialise the state of a blockdev page's buffers. 9161da177e4SLinus Torvalds */ 9171da177e4SLinus Torvalds static void 9181da177e4SLinus Torvalds init_page_buffers(struct page *page, struct block_device *bdev, 9191da177e4SLinus Torvalds sector_t block, int size) 9201da177e4SLinus Torvalds { 9211da177e4SLinus Torvalds struct buffer_head *head = page_buffers(page); 9221da177e4SLinus Torvalds struct buffer_head *bh = head; 9231da177e4SLinus Torvalds int uptodate = PageUptodate(page); 924080399aaSJeff Moyer sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode)); 9251da177e4SLinus Torvalds 9261da177e4SLinus Torvalds do { 9271da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 9281da177e4SLinus Torvalds init_buffer(bh, NULL, NULL); 9291da177e4SLinus Torvalds bh->b_bdev = bdev; 9301da177e4SLinus Torvalds bh->b_blocknr = block; 9311da177e4SLinus Torvalds if (uptodate) 9321da177e4SLinus Torvalds set_buffer_uptodate(bh); 933080399aaSJeff Moyer if (block < end_block) 9341da177e4SLinus Torvalds set_buffer_mapped(bh); 9351da177e4SLinus Torvalds } 9361da177e4SLinus Torvalds block++; 9371da177e4SLinus Torvalds bh = bh->b_this_page; 9381da177e4SLinus Torvalds } while (bh != head); 9391da177e4SLinus Torvalds } 9401da177e4SLinus Torvalds 9411da177e4SLinus Torvalds /* 9421da177e4SLinus Torvalds * Create the page-cache page that contains the requested block. 9431da177e4SLinus Torvalds * 9441da177e4SLinus Torvalds * This is user purely for blockdev mappings. 9451da177e4SLinus Torvalds */ 9461da177e4SLinus Torvalds static struct page * 9471da177e4SLinus Torvalds grow_dev_page(struct block_device *bdev, sector_t block, 9481da177e4SLinus Torvalds pgoff_t index, int size) 9491da177e4SLinus Torvalds { 9501da177e4SLinus Torvalds struct inode *inode = bdev->bd_inode; 9511da177e4SLinus Torvalds struct page *page; 9521da177e4SLinus Torvalds struct buffer_head *bh; 9531da177e4SLinus Torvalds 954ea125892SChristoph Lameter page = find_or_create_page(inode->i_mapping, index, 955769848c0SMel Gorman (mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE); 9561da177e4SLinus Torvalds if (!page) 9571da177e4SLinus Torvalds return NULL; 9581da177e4SLinus Torvalds 959e827f923SEric Sesterhenn BUG_ON(!PageLocked(page)); 9601da177e4SLinus Torvalds 9611da177e4SLinus Torvalds if (page_has_buffers(page)) { 9621da177e4SLinus Torvalds bh = page_buffers(page); 9631da177e4SLinus Torvalds if (bh->b_size == size) { 9641da177e4SLinus Torvalds init_page_buffers(page, bdev, block, size); 9651da177e4SLinus Torvalds return page; 9661da177e4SLinus Torvalds } 9671da177e4SLinus Torvalds if (!try_to_free_buffers(page)) 9681da177e4SLinus Torvalds goto failed; 9691da177e4SLinus Torvalds } 9701da177e4SLinus Torvalds 9711da177e4SLinus Torvalds /* 9721da177e4SLinus Torvalds * Allocate some buffers for this page 9731da177e4SLinus Torvalds */ 9741da177e4SLinus Torvalds bh = alloc_page_buffers(page, size, 0); 9751da177e4SLinus Torvalds if (!bh) 9761da177e4SLinus Torvalds goto failed; 9771da177e4SLinus Torvalds 9781da177e4SLinus Torvalds /* 9791da177e4SLinus Torvalds * Link the page to the buffers and initialise them. Take the 9801da177e4SLinus Torvalds * lock to be atomic wrt __find_get_block(), which does not 9811da177e4SLinus Torvalds * run under the page lock. 9821da177e4SLinus Torvalds */ 9831da177e4SLinus Torvalds spin_lock(&inode->i_mapping->private_lock); 9841da177e4SLinus Torvalds link_dev_buffers(page, bh); 9851da177e4SLinus Torvalds init_page_buffers(page, bdev, block, size); 9861da177e4SLinus Torvalds spin_unlock(&inode->i_mapping->private_lock); 9871da177e4SLinus Torvalds return page; 9881da177e4SLinus Torvalds 9891da177e4SLinus Torvalds failed: 9901da177e4SLinus Torvalds unlock_page(page); 9911da177e4SLinus Torvalds page_cache_release(page); 9921da177e4SLinus Torvalds return NULL; 9931da177e4SLinus Torvalds } 9941da177e4SLinus Torvalds 9951da177e4SLinus Torvalds /* 9961da177e4SLinus Torvalds * Create buffers for the specified block device block's page. If 9971da177e4SLinus Torvalds * that page was dirty, the buffers are set dirty also. 9981da177e4SLinus Torvalds */ 999858119e1SArjan van de Ven static int 10001da177e4SLinus Torvalds grow_buffers(struct block_device *bdev, sector_t block, int size) 10011da177e4SLinus Torvalds { 10021da177e4SLinus Torvalds struct page *page; 10031da177e4SLinus Torvalds pgoff_t index; 10041da177e4SLinus Torvalds int sizebits; 10051da177e4SLinus Torvalds 10061da177e4SLinus Torvalds sizebits = -1; 10071da177e4SLinus Torvalds do { 10081da177e4SLinus Torvalds sizebits++; 10091da177e4SLinus Torvalds } while ((size << sizebits) < PAGE_SIZE); 10101da177e4SLinus Torvalds 10111da177e4SLinus Torvalds index = block >> sizebits; 10121da177e4SLinus Torvalds 1013e5657933SAndrew Morton /* 1014e5657933SAndrew Morton * Check for a block which wants to lie outside our maximum possible 1015e5657933SAndrew Morton * pagecache index. (this comparison is done using sector_t types). 1016e5657933SAndrew Morton */ 1017e5657933SAndrew Morton if (unlikely(index != block >> sizebits)) { 1018e5657933SAndrew Morton char b[BDEVNAME_SIZE]; 1019e5657933SAndrew Morton 1020e5657933SAndrew Morton printk(KERN_ERR "%s: requested out-of-range block %llu for " 1021e5657933SAndrew Morton "device %s\n", 10228e24eea7SHarvey Harrison __func__, (unsigned long long)block, 1023e5657933SAndrew Morton bdevname(bdev, b)); 1024e5657933SAndrew Morton return -EIO; 1025e5657933SAndrew Morton } 1026e5657933SAndrew Morton block = index << sizebits; 10271da177e4SLinus Torvalds /* Create a page with the proper size buffers.. */ 10281da177e4SLinus Torvalds page = grow_dev_page(bdev, block, index, size); 10291da177e4SLinus Torvalds if (!page) 10301da177e4SLinus Torvalds return 0; 10311da177e4SLinus Torvalds unlock_page(page); 10321da177e4SLinus Torvalds page_cache_release(page); 10331da177e4SLinus Torvalds return 1; 10341da177e4SLinus Torvalds } 10351da177e4SLinus Torvalds 103675c96f85SAdrian Bunk static struct buffer_head * 10371da177e4SLinus Torvalds __getblk_slow(struct block_device *bdev, sector_t block, int size) 10381da177e4SLinus Torvalds { 1039*91f68c89SJeff Moyer int ret; 1040*91f68c89SJeff Moyer struct buffer_head *bh; 1041*91f68c89SJeff Moyer 10421da177e4SLinus Torvalds /* Size must be multiple of hard sectorsize */ 1043e1defc4fSMartin K. Petersen if (unlikely(size & (bdev_logical_block_size(bdev)-1) || 10441da177e4SLinus Torvalds (size < 512 || size > PAGE_SIZE))) { 10451da177e4SLinus Torvalds printk(KERN_ERR "getblk(): invalid block size %d requested\n", 10461da177e4SLinus Torvalds size); 1047e1defc4fSMartin K. Petersen printk(KERN_ERR "logical block size: %d\n", 1048e1defc4fSMartin K. Petersen bdev_logical_block_size(bdev)); 10491da177e4SLinus Torvalds 10501da177e4SLinus Torvalds dump_stack(); 10511da177e4SLinus Torvalds return NULL; 10521da177e4SLinus Torvalds } 10531da177e4SLinus Torvalds 1054*91f68c89SJeff Moyer retry: 10551da177e4SLinus Torvalds bh = __find_get_block(bdev, block, size); 10561da177e4SLinus Torvalds if (bh) 10571da177e4SLinus Torvalds return bh; 10581da177e4SLinus Torvalds 1059e5657933SAndrew Morton ret = grow_buffers(bdev, block, size); 1060*91f68c89SJeff Moyer if (ret == 0) { 10611da177e4SLinus Torvalds free_more_memory(); 1062*91f68c89SJeff Moyer goto retry; 1063*91f68c89SJeff Moyer } else if (ret > 0) { 1064*91f68c89SJeff Moyer bh = __find_get_block(bdev, block, size); 1065*91f68c89SJeff Moyer if (bh) 1066*91f68c89SJeff Moyer return bh; 10671da177e4SLinus Torvalds } 1068*91f68c89SJeff Moyer return NULL; 10691da177e4SLinus Torvalds } 10701da177e4SLinus Torvalds 10711da177e4SLinus Torvalds /* 10721da177e4SLinus Torvalds * The relationship between dirty buffers and dirty pages: 10731da177e4SLinus Torvalds * 10741da177e4SLinus Torvalds * Whenever a page has any dirty buffers, the page's dirty bit is set, and 10751da177e4SLinus Torvalds * the page is tagged dirty in its radix tree. 10761da177e4SLinus Torvalds * 10771da177e4SLinus Torvalds * At all times, the dirtiness of the buffers represents the dirtiness of 10781da177e4SLinus Torvalds * subsections of the page. If the page has buffers, the page dirty bit is 10791da177e4SLinus Torvalds * merely a hint about the true dirty state. 10801da177e4SLinus Torvalds * 10811da177e4SLinus Torvalds * When a page is set dirty in its entirety, all its buffers are marked dirty 10821da177e4SLinus Torvalds * (if the page has buffers). 10831da177e4SLinus Torvalds * 10841da177e4SLinus Torvalds * When a buffer is marked dirty, its page is dirtied, but the page's other 10851da177e4SLinus Torvalds * buffers are not. 10861da177e4SLinus Torvalds * 10871da177e4SLinus Torvalds * Also. When blockdev buffers are explicitly read with bread(), they 10881da177e4SLinus Torvalds * individually become uptodate. But their backing page remains not 10891da177e4SLinus Torvalds * uptodate - even if all of its buffers are uptodate. A subsequent 10901da177e4SLinus Torvalds * block_read_full_page() against that page will discover all the uptodate 10911da177e4SLinus Torvalds * buffers, will set the page uptodate and will perform no I/O. 10921da177e4SLinus Torvalds */ 10931da177e4SLinus Torvalds 10941da177e4SLinus Torvalds /** 10951da177e4SLinus Torvalds * mark_buffer_dirty - mark a buffer_head as needing writeout 109667be2dd1SMartin Waitz * @bh: the buffer_head to mark dirty 10971da177e4SLinus Torvalds * 10981da177e4SLinus Torvalds * mark_buffer_dirty() will set the dirty bit against the buffer, then set its 10991da177e4SLinus Torvalds * backing page dirty, then tag the page as dirty in its address_space's radix 11001da177e4SLinus Torvalds * tree and then attach the address_space's inode to its superblock's dirty 11011da177e4SLinus Torvalds * inode list. 11021da177e4SLinus Torvalds * 11031da177e4SLinus Torvalds * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, 1104250df6edSDave Chinner * mapping->tree_lock and mapping->host->i_lock. 11051da177e4SLinus Torvalds */ 1106fc9b52cdSHarvey Harrison void mark_buffer_dirty(struct buffer_head *bh) 11071da177e4SLinus Torvalds { 1108787d2214SNick Piggin WARN_ON_ONCE(!buffer_uptodate(bh)); 11091be62dc1SLinus Torvalds 11101be62dc1SLinus Torvalds /* 11111be62dc1SLinus Torvalds * Very *carefully* optimize the it-is-already-dirty case. 11121be62dc1SLinus Torvalds * 11131be62dc1SLinus Torvalds * Don't let the final "is it dirty" escape to before we 11141be62dc1SLinus Torvalds * perhaps modified the buffer. 11151be62dc1SLinus Torvalds */ 11161be62dc1SLinus Torvalds if (buffer_dirty(bh)) { 11171be62dc1SLinus Torvalds smp_mb(); 11181be62dc1SLinus Torvalds if (buffer_dirty(bh)) 11191be62dc1SLinus Torvalds return; 11201be62dc1SLinus Torvalds } 11211be62dc1SLinus Torvalds 1122a8e7d49aSLinus Torvalds if (!test_set_buffer_dirty(bh)) { 1123a8e7d49aSLinus Torvalds struct page *page = bh->b_page; 11248e9d78edSLinus Torvalds if (!TestSetPageDirty(page)) { 11258e9d78edSLinus Torvalds struct address_space *mapping = page_mapping(page); 11268e9d78edSLinus Torvalds if (mapping) 11278e9d78edSLinus Torvalds __set_page_dirty(page, mapping, 0); 11288e9d78edSLinus Torvalds } 1129a8e7d49aSLinus Torvalds } 11301da177e4SLinus Torvalds } 11311fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(mark_buffer_dirty); 11321da177e4SLinus Torvalds 11331da177e4SLinus Torvalds /* 11341da177e4SLinus Torvalds * Decrement a buffer_head's reference count. If all buffers against a page 11351da177e4SLinus Torvalds * have zero reference count, are clean and unlocked, and if the page is clean 11361da177e4SLinus Torvalds * and unlocked then try_to_free_buffers() may strip the buffers from the page 11371da177e4SLinus Torvalds * in preparation for freeing it (sometimes, rarely, buffers are removed from 11381da177e4SLinus Torvalds * a page but it ends up not being freed, and buffers may later be reattached). 11391da177e4SLinus Torvalds */ 11401da177e4SLinus Torvalds void __brelse(struct buffer_head * buf) 11411da177e4SLinus Torvalds { 11421da177e4SLinus Torvalds if (atomic_read(&buf->b_count)) { 11431da177e4SLinus Torvalds put_bh(buf); 11441da177e4SLinus Torvalds return; 11451da177e4SLinus Torvalds } 11465c752ad9SArjan van de Ven WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n"); 11471da177e4SLinus Torvalds } 11481fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(__brelse); 11491da177e4SLinus Torvalds 11501da177e4SLinus Torvalds /* 11511da177e4SLinus Torvalds * bforget() is like brelse(), except it discards any 11521da177e4SLinus Torvalds * potentially dirty data. 11531da177e4SLinus Torvalds */ 11541da177e4SLinus Torvalds void __bforget(struct buffer_head *bh) 11551da177e4SLinus Torvalds { 11561da177e4SLinus Torvalds clear_buffer_dirty(bh); 1157535ee2fbSJan Kara if (bh->b_assoc_map) { 11581da177e4SLinus Torvalds struct address_space *buffer_mapping = bh->b_page->mapping; 11591da177e4SLinus Torvalds 11601da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 11611da177e4SLinus Torvalds list_del_init(&bh->b_assoc_buffers); 116258ff407bSJan Kara bh->b_assoc_map = NULL; 11631da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 11641da177e4SLinus Torvalds } 11651da177e4SLinus Torvalds __brelse(bh); 11661da177e4SLinus Torvalds } 11671fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(__bforget); 11681da177e4SLinus Torvalds 11691da177e4SLinus Torvalds static struct buffer_head *__bread_slow(struct buffer_head *bh) 11701da177e4SLinus Torvalds { 11711da177e4SLinus Torvalds lock_buffer(bh); 11721da177e4SLinus Torvalds if (buffer_uptodate(bh)) { 11731da177e4SLinus Torvalds unlock_buffer(bh); 11741da177e4SLinus Torvalds return bh; 11751da177e4SLinus Torvalds } else { 11761da177e4SLinus Torvalds get_bh(bh); 11771da177e4SLinus Torvalds bh->b_end_io = end_buffer_read_sync; 11781da177e4SLinus Torvalds submit_bh(READ, bh); 11791da177e4SLinus Torvalds wait_on_buffer(bh); 11801da177e4SLinus Torvalds if (buffer_uptodate(bh)) 11811da177e4SLinus Torvalds return bh; 11821da177e4SLinus Torvalds } 11831da177e4SLinus Torvalds brelse(bh); 11841da177e4SLinus Torvalds return NULL; 11851da177e4SLinus Torvalds } 11861da177e4SLinus Torvalds 11871da177e4SLinus Torvalds /* 11881da177e4SLinus Torvalds * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block(). 11891da177e4SLinus Torvalds * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their 11901da177e4SLinus Torvalds * refcount elevated by one when they're in an LRU. A buffer can only appear 11911da177e4SLinus Torvalds * once in a particular CPU's LRU. A single buffer can be present in multiple 11921da177e4SLinus Torvalds * CPU's LRUs at the same time. 11931da177e4SLinus Torvalds * 11941da177e4SLinus Torvalds * This is a transparent caching front-end to sb_bread(), sb_getblk() and 11951da177e4SLinus Torvalds * sb_find_get_block(). 11961da177e4SLinus Torvalds * 11971da177e4SLinus Torvalds * The LRUs themselves only need locking against invalidate_bh_lrus. We use 11981da177e4SLinus Torvalds * a local interrupt disable for that. 11991da177e4SLinus Torvalds */ 12001da177e4SLinus Torvalds 12011da177e4SLinus Torvalds #define BH_LRU_SIZE 8 12021da177e4SLinus Torvalds 12031da177e4SLinus Torvalds struct bh_lru { 12041da177e4SLinus Torvalds struct buffer_head *bhs[BH_LRU_SIZE]; 12051da177e4SLinus Torvalds }; 12061da177e4SLinus Torvalds 12071da177e4SLinus Torvalds static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }}; 12081da177e4SLinus Torvalds 12091da177e4SLinus Torvalds #ifdef CONFIG_SMP 12101da177e4SLinus Torvalds #define bh_lru_lock() local_irq_disable() 12111da177e4SLinus Torvalds #define bh_lru_unlock() local_irq_enable() 12121da177e4SLinus Torvalds #else 12131da177e4SLinus Torvalds #define bh_lru_lock() preempt_disable() 12141da177e4SLinus Torvalds #define bh_lru_unlock() preempt_enable() 12151da177e4SLinus Torvalds #endif 12161da177e4SLinus Torvalds 12171da177e4SLinus Torvalds static inline void check_irqs_on(void) 12181da177e4SLinus Torvalds { 12191da177e4SLinus Torvalds #ifdef irqs_disabled 12201da177e4SLinus Torvalds BUG_ON(irqs_disabled()); 12211da177e4SLinus Torvalds #endif 12221da177e4SLinus Torvalds } 12231da177e4SLinus Torvalds 12241da177e4SLinus Torvalds /* 12251da177e4SLinus Torvalds * The LRU management algorithm is dopey-but-simple. Sorry. 12261da177e4SLinus Torvalds */ 12271da177e4SLinus Torvalds static void bh_lru_install(struct buffer_head *bh) 12281da177e4SLinus Torvalds { 12291da177e4SLinus Torvalds struct buffer_head *evictee = NULL; 12301da177e4SLinus Torvalds 12311da177e4SLinus Torvalds check_irqs_on(); 12321da177e4SLinus Torvalds bh_lru_lock(); 1233c7b92516SChristoph Lameter if (__this_cpu_read(bh_lrus.bhs[0]) != bh) { 12341da177e4SLinus Torvalds struct buffer_head *bhs[BH_LRU_SIZE]; 12351da177e4SLinus Torvalds int in; 12361da177e4SLinus Torvalds int out = 0; 12371da177e4SLinus Torvalds 12381da177e4SLinus Torvalds get_bh(bh); 12391da177e4SLinus Torvalds bhs[out++] = bh; 12401da177e4SLinus Torvalds for (in = 0; in < BH_LRU_SIZE; in++) { 1241c7b92516SChristoph Lameter struct buffer_head *bh2 = 1242c7b92516SChristoph Lameter __this_cpu_read(bh_lrus.bhs[in]); 12431da177e4SLinus Torvalds 12441da177e4SLinus Torvalds if (bh2 == bh) { 12451da177e4SLinus Torvalds __brelse(bh2); 12461da177e4SLinus Torvalds } else { 12471da177e4SLinus Torvalds if (out >= BH_LRU_SIZE) { 12481da177e4SLinus Torvalds BUG_ON(evictee != NULL); 12491da177e4SLinus Torvalds evictee = bh2; 12501da177e4SLinus Torvalds } else { 12511da177e4SLinus Torvalds bhs[out++] = bh2; 12521da177e4SLinus Torvalds } 12531da177e4SLinus Torvalds } 12541da177e4SLinus Torvalds } 12551da177e4SLinus Torvalds while (out < BH_LRU_SIZE) 12561da177e4SLinus Torvalds bhs[out++] = NULL; 1257c7b92516SChristoph Lameter memcpy(__this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs)); 12581da177e4SLinus Torvalds } 12591da177e4SLinus Torvalds bh_lru_unlock(); 12601da177e4SLinus Torvalds 12611da177e4SLinus Torvalds if (evictee) 12621da177e4SLinus Torvalds __brelse(evictee); 12631da177e4SLinus Torvalds } 12641da177e4SLinus Torvalds 12651da177e4SLinus Torvalds /* 12661da177e4SLinus Torvalds * Look up the bh in this cpu's LRU. If it's there, move it to the head. 12671da177e4SLinus Torvalds */ 1268858119e1SArjan van de Ven static struct buffer_head * 12693991d3bdSTomasz Kvarsin lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size) 12701da177e4SLinus Torvalds { 12711da177e4SLinus Torvalds struct buffer_head *ret = NULL; 12723991d3bdSTomasz Kvarsin unsigned int i; 12731da177e4SLinus Torvalds 12741da177e4SLinus Torvalds check_irqs_on(); 12751da177e4SLinus Torvalds bh_lru_lock(); 12761da177e4SLinus Torvalds for (i = 0; i < BH_LRU_SIZE; i++) { 1277c7b92516SChristoph Lameter struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]); 12781da177e4SLinus Torvalds 12791da177e4SLinus Torvalds if (bh && bh->b_bdev == bdev && 12801da177e4SLinus Torvalds bh->b_blocknr == block && bh->b_size == size) { 12811da177e4SLinus Torvalds if (i) { 12821da177e4SLinus Torvalds while (i) { 1283c7b92516SChristoph Lameter __this_cpu_write(bh_lrus.bhs[i], 1284c7b92516SChristoph Lameter __this_cpu_read(bh_lrus.bhs[i - 1])); 12851da177e4SLinus Torvalds i--; 12861da177e4SLinus Torvalds } 1287c7b92516SChristoph Lameter __this_cpu_write(bh_lrus.bhs[0], bh); 12881da177e4SLinus Torvalds } 12891da177e4SLinus Torvalds get_bh(bh); 12901da177e4SLinus Torvalds ret = bh; 12911da177e4SLinus Torvalds break; 12921da177e4SLinus Torvalds } 12931da177e4SLinus Torvalds } 12941da177e4SLinus Torvalds bh_lru_unlock(); 12951da177e4SLinus Torvalds return ret; 12961da177e4SLinus Torvalds } 12971da177e4SLinus Torvalds 12981da177e4SLinus Torvalds /* 12991da177e4SLinus Torvalds * Perform a pagecache lookup for the matching buffer. If it's there, refresh 13001da177e4SLinus Torvalds * it in the LRU and mark it as accessed. If it is not present then return 13011da177e4SLinus Torvalds * NULL 13021da177e4SLinus Torvalds */ 13031da177e4SLinus Torvalds struct buffer_head * 13043991d3bdSTomasz Kvarsin __find_get_block(struct block_device *bdev, sector_t block, unsigned size) 13051da177e4SLinus Torvalds { 13061da177e4SLinus Torvalds struct buffer_head *bh = lookup_bh_lru(bdev, block, size); 13071da177e4SLinus Torvalds 13081da177e4SLinus Torvalds if (bh == NULL) { 1309385fd4c5SCoywolf Qi Hunt bh = __find_get_block_slow(bdev, block); 13101da177e4SLinus Torvalds if (bh) 13111da177e4SLinus Torvalds bh_lru_install(bh); 13121da177e4SLinus Torvalds } 13131da177e4SLinus Torvalds if (bh) 13141da177e4SLinus Torvalds touch_buffer(bh); 13151da177e4SLinus Torvalds return bh; 13161da177e4SLinus Torvalds } 13171da177e4SLinus Torvalds EXPORT_SYMBOL(__find_get_block); 13181da177e4SLinus Torvalds 13191da177e4SLinus Torvalds /* 13201da177e4SLinus Torvalds * __getblk will locate (and, if necessary, create) the buffer_head 13211da177e4SLinus Torvalds * which corresponds to the passed block_device, block and size. The 13221da177e4SLinus Torvalds * returned buffer has its reference count incremented. 13231da177e4SLinus Torvalds * 13241da177e4SLinus Torvalds * __getblk() cannot fail - it just keeps trying. If you pass it an 13251da177e4SLinus Torvalds * illegal block number, __getblk() will happily return a buffer_head 13261da177e4SLinus Torvalds * which represents the non-existent block. Very weird. 13271da177e4SLinus Torvalds * 13281da177e4SLinus Torvalds * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers() 13291da177e4SLinus Torvalds * attempt is failing. FIXME, perhaps? 13301da177e4SLinus Torvalds */ 13311da177e4SLinus Torvalds struct buffer_head * 13323991d3bdSTomasz Kvarsin __getblk(struct block_device *bdev, sector_t block, unsigned size) 13331da177e4SLinus Torvalds { 13341da177e4SLinus Torvalds struct buffer_head *bh = __find_get_block(bdev, block, size); 13351da177e4SLinus Torvalds 13361da177e4SLinus Torvalds might_sleep(); 13371da177e4SLinus Torvalds if (bh == NULL) 13381da177e4SLinus Torvalds bh = __getblk_slow(bdev, block, size); 13391da177e4SLinus Torvalds return bh; 13401da177e4SLinus Torvalds } 13411da177e4SLinus Torvalds EXPORT_SYMBOL(__getblk); 13421da177e4SLinus Torvalds 13431da177e4SLinus Torvalds /* 13441da177e4SLinus Torvalds * Do async read-ahead on a buffer.. 13451da177e4SLinus Torvalds */ 13463991d3bdSTomasz Kvarsin void __breadahead(struct block_device *bdev, sector_t block, unsigned size) 13471da177e4SLinus Torvalds { 13481da177e4SLinus Torvalds struct buffer_head *bh = __getblk(bdev, block, size); 1349a3e713b5SAndrew Morton if (likely(bh)) { 13501da177e4SLinus Torvalds ll_rw_block(READA, 1, &bh); 13511da177e4SLinus Torvalds brelse(bh); 13521da177e4SLinus Torvalds } 1353a3e713b5SAndrew Morton } 13541da177e4SLinus Torvalds EXPORT_SYMBOL(__breadahead); 13551da177e4SLinus Torvalds 13561da177e4SLinus Torvalds /** 13571da177e4SLinus Torvalds * __bread() - reads a specified block and returns the bh 135867be2dd1SMartin Waitz * @bdev: the block_device to read from 13591da177e4SLinus Torvalds * @block: number of block 13601da177e4SLinus Torvalds * @size: size (in bytes) to read 13611da177e4SLinus Torvalds * 13621da177e4SLinus Torvalds * Reads a specified block, and returns buffer head that contains it. 13631da177e4SLinus Torvalds * It returns NULL if the block was unreadable. 13641da177e4SLinus Torvalds */ 13651da177e4SLinus Torvalds struct buffer_head * 13663991d3bdSTomasz Kvarsin __bread(struct block_device *bdev, sector_t block, unsigned size) 13671da177e4SLinus Torvalds { 13681da177e4SLinus Torvalds struct buffer_head *bh = __getblk(bdev, block, size); 13691da177e4SLinus Torvalds 1370a3e713b5SAndrew Morton if (likely(bh) && !buffer_uptodate(bh)) 13711da177e4SLinus Torvalds bh = __bread_slow(bh); 13721da177e4SLinus Torvalds return bh; 13731da177e4SLinus Torvalds } 13741da177e4SLinus Torvalds EXPORT_SYMBOL(__bread); 13751da177e4SLinus Torvalds 13761da177e4SLinus Torvalds /* 13771da177e4SLinus Torvalds * invalidate_bh_lrus() is called rarely - but not only at unmount. 13781da177e4SLinus Torvalds * This doesn't race because it runs in each cpu either in irq 13791da177e4SLinus Torvalds * or with preempt disabled. 13801da177e4SLinus Torvalds */ 13811da177e4SLinus Torvalds static void invalidate_bh_lru(void *arg) 13821da177e4SLinus Torvalds { 13831da177e4SLinus Torvalds struct bh_lru *b = &get_cpu_var(bh_lrus); 13841da177e4SLinus Torvalds int i; 13851da177e4SLinus Torvalds 13861da177e4SLinus Torvalds for (i = 0; i < BH_LRU_SIZE; i++) { 13871da177e4SLinus Torvalds brelse(b->bhs[i]); 13881da177e4SLinus Torvalds b->bhs[i] = NULL; 13891da177e4SLinus Torvalds } 13901da177e4SLinus Torvalds put_cpu_var(bh_lrus); 13911da177e4SLinus Torvalds } 13921da177e4SLinus Torvalds 139342be35d0SGilad Ben-Yossef static bool has_bh_in_lru(int cpu, void *dummy) 139442be35d0SGilad Ben-Yossef { 139542be35d0SGilad Ben-Yossef struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu); 139642be35d0SGilad Ben-Yossef int i; 139742be35d0SGilad Ben-Yossef 139842be35d0SGilad Ben-Yossef for (i = 0; i < BH_LRU_SIZE; i++) { 139942be35d0SGilad Ben-Yossef if (b->bhs[i]) 140042be35d0SGilad Ben-Yossef return 1; 140142be35d0SGilad Ben-Yossef } 140242be35d0SGilad Ben-Yossef 140342be35d0SGilad Ben-Yossef return 0; 140442be35d0SGilad Ben-Yossef } 140542be35d0SGilad Ben-Yossef 1406f9a14399SPeter Zijlstra void invalidate_bh_lrus(void) 14071da177e4SLinus Torvalds { 140842be35d0SGilad Ben-Yossef on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL); 14091da177e4SLinus Torvalds } 14109db5579bSNick Piggin EXPORT_SYMBOL_GPL(invalidate_bh_lrus); 14111da177e4SLinus Torvalds 14121da177e4SLinus Torvalds void set_bh_page(struct buffer_head *bh, 14131da177e4SLinus Torvalds struct page *page, unsigned long offset) 14141da177e4SLinus Torvalds { 14151da177e4SLinus Torvalds bh->b_page = page; 1416e827f923SEric Sesterhenn BUG_ON(offset >= PAGE_SIZE); 14171da177e4SLinus Torvalds if (PageHighMem(page)) 14181da177e4SLinus Torvalds /* 14191da177e4SLinus Torvalds * This catches illegal uses and preserves the offset: 14201da177e4SLinus Torvalds */ 14211da177e4SLinus Torvalds bh->b_data = (char *)(0 + offset); 14221da177e4SLinus Torvalds else 14231da177e4SLinus Torvalds bh->b_data = page_address(page) + offset; 14241da177e4SLinus Torvalds } 14251da177e4SLinus Torvalds EXPORT_SYMBOL(set_bh_page); 14261da177e4SLinus Torvalds 14271da177e4SLinus Torvalds /* 14281da177e4SLinus Torvalds * Called when truncating a buffer on a page completely. 14291da177e4SLinus Torvalds */ 1430858119e1SArjan van de Ven static void discard_buffer(struct buffer_head * bh) 14311da177e4SLinus Torvalds { 14321da177e4SLinus Torvalds lock_buffer(bh); 14331da177e4SLinus Torvalds clear_buffer_dirty(bh); 14341da177e4SLinus Torvalds bh->b_bdev = NULL; 14351da177e4SLinus Torvalds clear_buffer_mapped(bh); 14361da177e4SLinus Torvalds clear_buffer_req(bh); 14371da177e4SLinus Torvalds clear_buffer_new(bh); 14381da177e4SLinus Torvalds clear_buffer_delay(bh); 143933a266ddSDavid Chinner clear_buffer_unwritten(bh); 14401da177e4SLinus Torvalds unlock_buffer(bh); 14411da177e4SLinus Torvalds } 14421da177e4SLinus Torvalds 14431da177e4SLinus Torvalds /** 1444814e1d25SWang Sheng-Hui * block_invalidatepage - invalidate part or all of a buffer-backed page 14451da177e4SLinus Torvalds * 14461da177e4SLinus Torvalds * @page: the page which is affected 14471da177e4SLinus Torvalds * @offset: the index of the truncation point 14481da177e4SLinus Torvalds * 14491da177e4SLinus Torvalds * block_invalidatepage() is called when all or part of the page has become 14501da177e4SLinus Torvalds * invalidated by a truncate operation. 14511da177e4SLinus Torvalds * 14521da177e4SLinus Torvalds * block_invalidatepage() does not have to release all buffers, but it must 14531da177e4SLinus Torvalds * ensure that no dirty buffer is left outside @offset and that no I/O 14541da177e4SLinus Torvalds * is underway against any of the blocks which are outside the truncation 14551da177e4SLinus Torvalds * point. Because the caller is about to free (and possibly reuse) those 14561da177e4SLinus Torvalds * blocks on-disk. 14571da177e4SLinus Torvalds */ 14582ff28e22SNeilBrown void block_invalidatepage(struct page *page, unsigned long offset) 14591da177e4SLinus Torvalds { 14601da177e4SLinus Torvalds struct buffer_head *head, *bh, *next; 14611da177e4SLinus Torvalds unsigned int curr_off = 0; 14621da177e4SLinus Torvalds 14631da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 14641da177e4SLinus Torvalds if (!page_has_buffers(page)) 14651da177e4SLinus Torvalds goto out; 14661da177e4SLinus Torvalds 14671da177e4SLinus Torvalds head = page_buffers(page); 14681da177e4SLinus Torvalds bh = head; 14691da177e4SLinus Torvalds do { 14701da177e4SLinus Torvalds unsigned int next_off = curr_off + bh->b_size; 14711da177e4SLinus Torvalds next = bh->b_this_page; 14721da177e4SLinus Torvalds 14731da177e4SLinus Torvalds /* 14741da177e4SLinus Torvalds * is this block fully invalidated? 14751da177e4SLinus Torvalds */ 14761da177e4SLinus Torvalds if (offset <= curr_off) 14771da177e4SLinus Torvalds discard_buffer(bh); 14781da177e4SLinus Torvalds curr_off = next_off; 14791da177e4SLinus Torvalds bh = next; 14801da177e4SLinus Torvalds } while (bh != head); 14811da177e4SLinus Torvalds 14821da177e4SLinus Torvalds /* 14831da177e4SLinus Torvalds * We release buffers only if the entire page is being invalidated. 14841da177e4SLinus Torvalds * The get_block cached value has been unconditionally invalidated, 14851da177e4SLinus Torvalds * so real IO is not possible anymore. 14861da177e4SLinus Torvalds */ 14871da177e4SLinus Torvalds if (offset == 0) 14882ff28e22SNeilBrown try_to_release_page(page, 0); 14891da177e4SLinus Torvalds out: 14902ff28e22SNeilBrown return; 14911da177e4SLinus Torvalds } 14921da177e4SLinus Torvalds EXPORT_SYMBOL(block_invalidatepage); 14931da177e4SLinus Torvalds 14941da177e4SLinus Torvalds /* 14951da177e4SLinus Torvalds * We attach and possibly dirty the buffers atomically wrt 14961da177e4SLinus Torvalds * __set_page_dirty_buffers() via private_lock. try_to_free_buffers 14971da177e4SLinus Torvalds * is already excluded via the page lock. 14981da177e4SLinus Torvalds */ 14991da177e4SLinus Torvalds void create_empty_buffers(struct page *page, 15001da177e4SLinus Torvalds unsigned long blocksize, unsigned long b_state) 15011da177e4SLinus Torvalds { 15021da177e4SLinus Torvalds struct buffer_head *bh, *head, *tail; 15031da177e4SLinus Torvalds 15041da177e4SLinus Torvalds head = alloc_page_buffers(page, blocksize, 1); 15051da177e4SLinus Torvalds bh = head; 15061da177e4SLinus Torvalds do { 15071da177e4SLinus Torvalds bh->b_state |= b_state; 15081da177e4SLinus Torvalds tail = bh; 15091da177e4SLinus Torvalds bh = bh->b_this_page; 15101da177e4SLinus Torvalds } while (bh); 15111da177e4SLinus Torvalds tail->b_this_page = head; 15121da177e4SLinus Torvalds 15131da177e4SLinus Torvalds spin_lock(&page->mapping->private_lock); 15141da177e4SLinus Torvalds if (PageUptodate(page) || PageDirty(page)) { 15151da177e4SLinus Torvalds bh = head; 15161da177e4SLinus Torvalds do { 15171da177e4SLinus Torvalds if (PageDirty(page)) 15181da177e4SLinus Torvalds set_buffer_dirty(bh); 15191da177e4SLinus Torvalds if (PageUptodate(page)) 15201da177e4SLinus Torvalds set_buffer_uptodate(bh); 15211da177e4SLinus Torvalds bh = bh->b_this_page; 15221da177e4SLinus Torvalds } while (bh != head); 15231da177e4SLinus Torvalds } 15241da177e4SLinus Torvalds attach_page_buffers(page, head); 15251da177e4SLinus Torvalds spin_unlock(&page->mapping->private_lock); 15261da177e4SLinus Torvalds } 15271da177e4SLinus Torvalds EXPORT_SYMBOL(create_empty_buffers); 15281da177e4SLinus Torvalds 15291da177e4SLinus Torvalds /* 15301da177e4SLinus Torvalds * We are taking a block for data and we don't want any output from any 15311da177e4SLinus Torvalds * buffer-cache aliases starting from return from that function and 15321da177e4SLinus Torvalds * until the moment when something will explicitly mark the buffer 15331da177e4SLinus Torvalds * dirty (hopefully that will not happen until we will free that block ;-) 15341da177e4SLinus Torvalds * We don't even need to mark it not-uptodate - nobody can expect 15351da177e4SLinus Torvalds * anything from a newly allocated buffer anyway. We used to used 15361da177e4SLinus Torvalds * unmap_buffer() for such invalidation, but that was wrong. We definitely 15371da177e4SLinus Torvalds * don't want to mark the alias unmapped, for example - it would confuse 15381da177e4SLinus Torvalds * anyone who might pick it with bread() afterwards... 15391da177e4SLinus Torvalds * 15401da177e4SLinus Torvalds * Also.. Note that bforget() doesn't lock the buffer. So there can 15411da177e4SLinus Torvalds * be writeout I/O going on against recently-freed buffers. We don't 15421da177e4SLinus Torvalds * wait on that I/O in bforget() - it's more efficient to wait on the I/O 15431da177e4SLinus Torvalds * only if we really need to. That happens here. 15441da177e4SLinus Torvalds */ 15451da177e4SLinus Torvalds void unmap_underlying_metadata(struct block_device *bdev, sector_t block) 15461da177e4SLinus Torvalds { 15471da177e4SLinus Torvalds struct buffer_head *old_bh; 15481da177e4SLinus Torvalds 15491da177e4SLinus Torvalds might_sleep(); 15501da177e4SLinus Torvalds 1551385fd4c5SCoywolf Qi Hunt old_bh = __find_get_block_slow(bdev, block); 15521da177e4SLinus Torvalds if (old_bh) { 15531da177e4SLinus Torvalds clear_buffer_dirty(old_bh); 15541da177e4SLinus Torvalds wait_on_buffer(old_bh); 15551da177e4SLinus Torvalds clear_buffer_req(old_bh); 15561da177e4SLinus Torvalds __brelse(old_bh); 15571da177e4SLinus Torvalds } 15581da177e4SLinus Torvalds } 15591da177e4SLinus Torvalds EXPORT_SYMBOL(unmap_underlying_metadata); 15601da177e4SLinus Torvalds 15611da177e4SLinus Torvalds /* 15621da177e4SLinus Torvalds * NOTE! All mapped/uptodate combinations are valid: 15631da177e4SLinus Torvalds * 15641da177e4SLinus Torvalds * Mapped Uptodate Meaning 15651da177e4SLinus Torvalds * 15661da177e4SLinus Torvalds * No No "unknown" - must do get_block() 15671da177e4SLinus Torvalds * No Yes "hole" - zero-filled 15681da177e4SLinus Torvalds * Yes No "allocated" - allocated on disk, not read in 15691da177e4SLinus Torvalds * Yes Yes "valid" - allocated and up-to-date in memory. 15701da177e4SLinus Torvalds * 15711da177e4SLinus Torvalds * "Dirty" is valid only with the last case (mapped+uptodate). 15721da177e4SLinus Torvalds */ 15731da177e4SLinus Torvalds 15741da177e4SLinus Torvalds /* 15751da177e4SLinus Torvalds * While block_write_full_page is writing back the dirty buffers under 15761da177e4SLinus Torvalds * the page lock, whoever dirtied the buffers may decide to clean them 15771da177e4SLinus Torvalds * again at any time. We handle that by only looking at the buffer 15781da177e4SLinus Torvalds * state inside lock_buffer(). 15791da177e4SLinus Torvalds * 15801da177e4SLinus Torvalds * If block_write_full_page() is called for regular writeback 15811da177e4SLinus Torvalds * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a 15821da177e4SLinus Torvalds * locked buffer. This only can happen if someone has written the buffer 15831da177e4SLinus Torvalds * directly, with submit_bh(). At the address_space level PageWriteback 15841da177e4SLinus Torvalds * prevents this contention from occurring. 15856e34eeddSTheodore Ts'o * 15866e34eeddSTheodore Ts'o * If block_write_full_page() is called with wbc->sync_mode == 1587721a9602SJens Axboe * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this 1588721a9602SJens Axboe * causes the writes to be flagged as synchronous writes. 15891da177e4SLinus Torvalds */ 15901da177e4SLinus Torvalds static int __block_write_full_page(struct inode *inode, struct page *page, 159135c80d5fSChris Mason get_block_t *get_block, struct writeback_control *wbc, 159235c80d5fSChris Mason bh_end_io_t *handler) 15931da177e4SLinus Torvalds { 15941da177e4SLinus Torvalds int err; 15951da177e4SLinus Torvalds sector_t block; 15961da177e4SLinus Torvalds sector_t last_block; 1597f0fbd5fcSAndrew Morton struct buffer_head *bh, *head; 1598b0cf2321SBadari Pulavarty const unsigned blocksize = 1 << inode->i_blkbits; 15991da177e4SLinus Torvalds int nr_underway = 0; 16006e34eeddSTheodore Ts'o int write_op = (wbc->sync_mode == WB_SYNC_ALL ? 1601721a9602SJens Axboe WRITE_SYNC : WRITE); 16021da177e4SLinus Torvalds 16031da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 16041da177e4SLinus Torvalds 16051da177e4SLinus Torvalds last_block = (i_size_read(inode) - 1) >> inode->i_blkbits; 16061da177e4SLinus Torvalds 16071da177e4SLinus Torvalds if (!page_has_buffers(page)) { 1608b0cf2321SBadari Pulavarty create_empty_buffers(page, blocksize, 16091da177e4SLinus Torvalds (1 << BH_Dirty)|(1 << BH_Uptodate)); 16101da177e4SLinus Torvalds } 16111da177e4SLinus Torvalds 16121da177e4SLinus Torvalds /* 16131da177e4SLinus Torvalds * Be very careful. We have no exclusion from __set_page_dirty_buffers 16141da177e4SLinus Torvalds * here, and the (potentially unmapped) buffers may become dirty at 16151da177e4SLinus Torvalds * any time. If a buffer becomes dirty here after we've inspected it 16161da177e4SLinus Torvalds * then we just miss that fact, and the page stays dirty. 16171da177e4SLinus Torvalds * 16181da177e4SLinus Torvalds * Buffers outside i_size may be dirtied by __set_page_dirty_buffers; 16191da177e4SLinus Torvalds * handle that here by just cleaning them. 16201da177e4SLinus Torvalds */ 16211da177e4SLinus Torvalds 162254b21a79SAndrew Morton block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 16231da177e4SLinus Torvalds head = page_buffers(page); 16241da177e4SLinus Torvalds bh = head; 16251da177e4SLinus Torvalds 16261da177e4SLinus Torvalds /* 16271da177e4SLinus Torvalds * Get all the dirty buffers mapped to disk addresses and 16281da177e4SLinus Torvalds * handle any aliases from the underlying blockdev's mapping. 16291da177e4SLinus Torvalds */ 16301da177e4SLinus Torvalds do { 16311da177e4SLinus Torvalds if (block > last_block) { 16321da177e4SLinus Torvalds /* 16331da177e4SLinus Torvalds * mapped buffers outside i_size will occur, because 16341da177e4SLinus Torvalds * this page can be outside i_size when there is a 16351da177e4SLinus Torvalds * truncate in progress. 16361da177e4SLinus Torvalds */ 16371da177e4SLinus Torvalds /* 16381da177e4SLinus Torvalds * The buffer was zeroed by block_write_full_page() 16391da177e4SLinus Torvalds */ 16401da177e4SLinus Torvalds clear_buffer_dirty(bh); 16411da177e4SLinus Torvalds set_buffer_uptodate(bh); 164229a814d2SAlex Tomas } else if ((!buffer_mapped(bh) || buffer_delay(bh)) && 164329a814d2SAlex Tomas buffer_dirty(bh)) { 1644b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 16451da177e4SLinus Torvalds err = get_block(inode, block, bh, 1); 16461da177e4SLinus Torvalds if (err) 16471da177e4SLinus Torvalds goto recover; 164829a814d2SAlex Tomas clear_buffer_delay(bh); 16491da177e4SLinus Torvalds if (buffer_new(bh)) { 16501da177e4SLinus Torvalds /* blockdev mappings never come here */ 16511da177e4SLinus Torvalds clear_buffer_new(bh); 16521da177e4SLinus Torvalds unmap_underlying_metadata(bh->b_bdev, 16531da177e4SLinus Torvalds bh->b_blocknr); 16541da177e4SLinus Torvalds } 16551da177e4SLinus Torvalds } 16561da177e4SLinus Torvalds bh = bh->b_this_page; 16571da177e4SLinus Torvalds block++; 16581da177e4SLinus Torvalds } while (bh != head); 16591da177e4SLinus Torvalds 16601da177e4SLinus Torvalds do { 16611da177e4SLinus Torvalds if (!buffer_mapped(bh)) 16621da177e4SLinus Torvalds continue; 16631da177e4SLinus Torvalds /* 16641da177e4SLinus Torvalds * If it's a fully non-blocking write attempt and we cannot 16651da177e4SLinus Torvalds * lock the buffer then redirty the page. Note that this can 16665b0830cbSJens Axboe * potentially cause a busy-wait loop from writeback threads 16675b0830cbSJens Axboe * and kswapd activity, but those code paths have their own 16685b0830cbSJens Axboe * higher-level throttling. 16691da177e4SLinus Torvalds */ 16701b430beeSWu Fengguang if (wbc->sync_mode != WB_SYNC_NONE) { 16711da177e4SLinus Torvalds lock_buffer(bh); 1672ca5de404SNick Piggin } else if (!trylock_buffer(bh)) { 16731da177e4SLinus Torvalds redirty_page_for_writepage(wbc, page); 16741da177e4SLinus Torvalds continue; 16751da177e4SLinus Torvalds } 16761da177e4SLinus Torvalds if (test_clear_buffer_dirty(bh)) { 167735c80d5fSChris Mason mark_buffer_async_write_endio(bh, handler); 16781da177e4SLinus Torvalds } else { 16791da177e4SLinus Torvalds unlock_buffer(bh); 16801da177e4SLinus Torvalds } 16811da177e4SLinus Torvalds } while ((bh = bh->b_this_page) != head); 16821da177e4SLinus Torvalds 16831da177e4SLinus Torvalds /* 16841da177e4SLinus Torvalds * The page and its buffers are protected by PageWriteback(), so we can 16851da177e4SLinus Torvalds * drop the bh refcounts early. 16861da177e4SLinus Torvalds */ 16871da177e4SLinus Torvalds BUG_ON(PageWriteback(page)); 16881da177e4SLinus Torvalds set_page_writeback(page); 16891da177e4SLinus Torvalds 16901da177e4SLinus Torvalds do { 16911da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 16921da177e4SLinus Torvalds if (buffer_async_write(bh)) { 1693a64c8610STheodore Ts'o submit_bh(write_op, bh); 16941da177e4SLinus Torvalds nr_underway++; 1695ad576e63SNick Piggin } 16961da177e4SLinus Torvalds bh = next; 16971da177e4SLinus Torvalds } while (bh != head); 169805937baaSAndrew Morton unlock_page(page); 16991da177e4SLinus Torvalds 17001da177e4SLinus Torvalds err = 0; 17011da177e4SLinus Torvalds done: 17021da177e4SLinus Torvalds if (nr_underway == 0) { 17031da177e4SLinus Torvalds /* 17041da177e4SLinus Torvalds * The page was marked dirty, but the buffers were 17051da177e4SLinus Torvalds * clean. Someone wrote them back by hand with 17061da177e4SLinus Torvalds * ll_rw_block/submit_bh. A rare case. 17071da177e4SLinus Torvalds */ 17081da177e4SLinus Torvalds end_page_writeback(page); 17093d67f2d7SNick Piggin 17101da177e4SLinus Torvalds /* 17111da177e4SLinus Torvalds * The page and buffer_heads can be released at any time from 17121da177e4SLinus Torvalds * here on. 17131da177e4SLinus Torvalds */ 17141da177e4SLinus Torvalds } 17151da177e4SLinus Torvalds return err; 17161da177e4SLinus Torvalds 17171da177e4SLinus Torvalds recover: 17181da177e4SLinus Torvalds /* 17191da177e4SLinus Torvalds * ENOSPC, or some other error. We may already have added some 17201da177e4SLinus Torvalds * blocks to the file, so we need to write these out to avoid 17211da177e4SLinus Torvalds * exposing stale data. 17221da177e4SLinus Torvalds * The page is currently locked and not marked for writeback 17231da177e4SLinus Torvalds */ 17241da177e4SLinus Torvalds bh = head; 17251da177e4SLinus Torvalds /* Recovery: lock and submit the mapped buffers */ 17261da177e4SLinus Torvalds do { 172729a814d2SAlex Tomas if (buffer_mapped(bh) && buffer_dirty(bh) && 172829a814d2SAlex Tomas !buffer_delay(bh)) { 17291da177e4SLinus Torvalds lock_buffer(bh); 173035c80d5fSChris Mason mark_buffer_async_write_endio(bh, handler); 17311da177e4SLinus Torvalds } else { 17321da177e4SLinus Torvalds /* 17331da177e4SLinus Torvalds * The buffer may have been set dirty during 17341da177e4SLinus Torvalds * attachment to a dirty page. 17351da177e4SLinus Torvalds */ 17361da177e4SLinus Torvalds clear_buffer_dirty(bh); 17371da177e4SLinus Torvalds } 17381da177e4SLinus Torvalds } while ((bh = bh->b_this_page) != head); 17391da177e4SLinus Torvalds SetPageError(page); 17401da177e4SLinus Torvalds BUG_ON(PageWriteback(page)); 17417e4c3690SAndrew Morton mapping_set_error(page->mapping, err); 17421da177e4SLinus Torvalds set_page_writeback(page); 17431da177e4SLinus Torvalds do { 17441da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 17451da177e4SLinus Torvalds if (buffer_async_write(bh)) { 17461da177e4SLinus Torvalds clear_buffer_dirty(bh); 1747a64c8610STheodore Ts'o submit_bh(write_op, bh); 17481da177e4SLinus Torvalds nr_underway++; 1749ad576e63SNick Piggin } 17501da177e4SLinus Torvalds bh = next; 17511da177e4SLinus Torvalds } while (bh != head); 1752ffda9d30SNick Piggin unlock_page(page); 17531da177e4SLinus Torvalds goto done; 17541da177e4SLinus Torvalds } 17551da177e4SLinus Torvalds 1756afddba49SNick Piggin /* 1757afddba49SNick Piggin * If a page has any new buffers, zero them out here, and mark them uptodate 1758afddba49SNick Piggin * and dirty so they'll be written out (in order to prevent uninitialised 1759afddba49SNick Piggin * block data from leaking). And clear the new bit. 1760afddba49SNick Piggin */ 1761afddba49SNick Piggin void page_zero_new_buffers(struct page *page, unsigned from, unsigned to) 1762afddba49SNick Piggin { 1763afddba49SNick Piggin unsigned int block_start, block_end; 1764afddba49SNick Piggin struct buffer_head *head, *bh; 1765afddba49SNick Piggin 1766afddba49SNick Piggin BUG_ON(!PageLocked(page)); 1767afddba49SNick Piggin if (!page_has_buffers(page)) 1768afddba49SNick Piggin return; 1769afddba49SNick Piggin 1770afddba49SNick Piggin bh = head = page_buffers(page); 1771afddba49SNick Piggin block_start = 0; 1772afddba49SNick Piggin do { 1773afddba49SNick Piggin block_end = block_start + bh->b_size; 1774afddba49SNick Piggin 1775afddba49SNick Piggin if (buffer_new(bh)) { 1776afddba49SNick Piggin if (block_end > from && block_start < to) { 1777afddba49SNick Piggin if (!PageUptodate(page)) { 1778afddba49SNick Piggin unsigned start, size; 1779afddba49SNick Piggin 1780afddba49SNick Piggin start = max(from, block_start); 1781afddba49SNick Piggin size = min(to, block_end) - start; 1782afddba49SNick Piggin 1783eebd2aa3SChristoph Lameter zero_user(page, start, size); 1784afddba49SNick Piggin set_buffer_uptodate(bh); 1785afddba49SNick Piggin } 1786afddba49SNick Piggin 1787afddba49SNick Piggin clear_buffer_new(bh); 1788afddba49SNick Piggin mark_buffer_dirty(bh); 1789afddba49SNick Piggin } 1790afddba49SNick Piggin } 1791afddba49SNick Piggin 1792afddba49SNick Piggin block_start = block_end; 1793afddba49SNick Piggin bh = bh->b_this_page; 1794afddba49SNick Piggin } while (bh != head); 1795afddba49SNick Piggin } 1796afddba49SNick Piggin EXPORT_SYMBOL(page_zero_new_buffers); 1797afddba49SNick Piggin 1798ebdec241SChristoph Hellwig int __block_write_begin(struct page *page, loff_t pos, unsigned len, 17996e1db88dSChristoph Hellwig get_block_t *get_block) 18001da177e4SLinus Torvalds { 1801ebdec241SChristoph Hellwig unsigned from = pos & (PAGE_CACHE_SIZE - 1); 1802ebdec241SChristoph Hellwig unsigned to = from + len; 18036e1db88dSChristoph Hellwig struct inode *inode = page->mapping->host; 18041da177e4SLinus Torvalds unsigned block_start, block_end; 18051da177e4SLinus Torvalds sector_t block; 18061da177e4SLinus Torvalds int err = 0; 18071da177e4SLinus Torvalds unsigned blocksize, bbits; 18081da177e4SLinus Torvalds struct buffer_head *bh, *head, *wait[2], **wait_bh=wait; 18091da177e4SLinus Torvalds 18101da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 18111da177e4SLinus Torvalds BUG_ON(from > PAGE_CACHE_SIZE); 18121da177e4SLinus Torvalds BUG_ON(to > PAGE_CACHE_SIZE); 18131da177e4SLinus Torvalds BUG_ON(from > to); 18141da177e4SLinus Torvalds 18151da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 18161da177e4SLinus Torvalds if (!page_has_buffers(page)) 18171da177e4SLinus Torvalds create_empty_buffers(page, blocksize, 0); 18181da177e4SLinus Torvalds head = page_buffers(page); 18191da177e4SLinus Torvalds 18201da177e4SLinus Torvalds bbits = inode->i_blkbits; 18211da177e4SLinus Torvalds block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits); 18221da177e4SLinus Torvalds 18231da177e4SLinus Torvalds for(bh = head, block_start = 0; bh != head || !block_start; 18241da177e4SLinus Torvalds block++, block_start=block_end, bh = bh->b_this_page) { 18251da177e4SLinus Torvalds block_end = block_start + blocksize; 18261da177e4SLinus Torvalds if (block_end <= from || block_start >= to) { 18271da177e4SLinus Torvalds if (PageUptodate(page)) { 18281da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 18291da177e4SLinus Torvalds set_buffer_uptodate(bh); 18301da177e4SLinus Torvalds } 18311da177e4SLinus Torvalds continue; 18321da177e4SLinus Torvalds } 18331da177e4SLinus Torvalds if (buffer_new(bh)) 18341da177e4SLinus Torvalds clear_buffer_new(bh); 18351da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 1836b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 18371da177e4SLinus Torvalds err = get_block(inode, block, bh, 1); 18381da177e4SLinus Torvalds if (err) 1839f3ddbdc6SNick Piggin break; 18401da177e4SLinus Torvalds if (buffer_new(bh)) { 18411da177e4SLinus Torvalds unmap_underlying_metadata(bh->b_bdev, 18421da177e4SLinus Torvalds bh->b_blocknr); 18431da177e4SLinus Torvalds if (PageUptodate(page)) { 1844637aff46SNick Piggin clear_buffer_new(bh); 18451da177e4SLinus Torvalds set_buffer_uptodate(bh); 1846637aff46SNick Piggin mark_buffer_dirty(bh); 18471da177e4SLinus Torvalds continue; 18481da177e4SLinus Torvalds } 1849eebd2aa3SChristoph Lameter if (block_end > to || block_start < from) 1850eebd2aa3SChristoph Lameter zero_user_segments(page, 1851eebd2aa3SChristoph Lameter to, block_end, 1852eebd2aa3SChristoph Lameter block_start, from); 18531da177e4SLinus Torvalds continue; 18541da177e4SLinus Torvalds } 18551da177e4SLinus Torvalds } 18561da177e4SLinus Torvalds if (PageUptodate(page)) { 18571da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 18581da177e4SLinus Torvalds set_buffer_uptodate(bh); 18591da177e4SLinus Torvalds continue; 18601da177e4SLinus Torvalds } 18611da177e4SLinus Torvalds if (!buffer_uptodate(bh) && !buffer_delay(bh) && 186233a266ddSDavid Chinner !buffer_unwritten(bh) && 18631da177e4SLinus Torvalds (block_start < from || block_end > to)) { 18641da177e4SLinus Torvalds ll_rw_block(READ, 1, &bh); 18651da177e4SLinus Torvalds *wait_bh++=bh; 18661da177e4SLinus Torvalds } 18671da177e4SLinus Torvalds } 18681da177e4SLinus Torvalds /* 18691da177e4SLinus Torvalds * If we issued read requests - let them complete. 18701da177e4SLinus Torvalds */ 18711da177e4SLinus Torvalds while(wait_bh > wait) { 18721da177e4SLinus Torvalds wait_on_buffer(*--wait_bh); 18731da177e4SLinus Torvalds if (!buffer_uptodate(*wait_bh)) 1874f3ddbdc6SNick Piggin err = -EIO; 18751da177e4SLinus Torvalds } 1876f9f07b6cSJan Kara if (unlikely(err)) 1877afddba49SNick Piggin page_zero_new_buffers(page, from, to); 18781da177e4SLinus Torvalds return err; 18791da177e4SLinus Torvalds } 1880ebdec241SChristoph Hellwig EXPORT_SYMBOL(__block_write_begin); 18811da177e4SLinus Torvalds 18821da177e4SLinus Torvalds static int __block_commit_write(struct inode *inode, struct page *page, 18831da177e4SLinus Torvalds unsigned from, unsigned to) 18841da177e4SLinus Torvalds { 18851da177e4SLinus Torvalds unsigned block_start, block_end; 18861da177e4SLinus Torvalds int partial = 0; 18871da177e4SLinus Torvalds unsigned blocksize; 18881da177e4SLinus Torvalds struct buffer_head *bh, *head; 18891da177e4SLinus Torvalds 18901da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 18911da177e4SLinus Torvalds 18921da177e4SLinus Torvalds for(bh = head = page_buffers(page), block_start = 0; 18931da177e4SLinus Torvalds bh != head || !block_start; 18941da177e4SLinus Torvalds block_start=block_end, bh = bh->b_this_page) { 18951da177e4SLinus Torvalds block_end = block_start + blocksize; 18961da177e4SLinus Torvalds if (block_end <= from || block_start >= to) { 18971da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 18981da177e4SLinus Torvalds partial = 1; 18991da177e4SLinus Torvalds } else { 19001da177e4SLinus Torvalds set_buffer_uptodate(bh); 19011da177e4SLinus Torvalds mark_buffer_dirty(bh); 19021da177e4SLinus Torvalds } 1903afddba49SNick Piggin clear_buffer_new(bh); 19041da177e4SLinus Torvalds } 19051da177e4SLinus Torvalds 19061da177e4SLinus Torvalds /* 19071da177e4SLinus Torvalds * If this is a partial write which happened to make all buffers 19081da177e4SLinus Torvalds * uptodate then we can optimize away a bogus readpage() for 19091da177e4SLinus Torvalds * the next read(). Here we 'discover' whether the page went 19101da177e4SLinus Torvalds * uptodate as a result of this (potentially partial) write. 19111da177e4SLinus Torvalds */ 19121da177e4SLinus Torvalds if (!partial) 19131da177e4SLinus Torvalds SetPageUptodate(page); 19141da177e4SLinus Torvalds return 0; 19151da177e4SLinus Torvalds } 19161da177e4SLinus Torvalds 19171da177e4SLinus Torvalds /* 1918155130a4SChristoph Hellwig * block_write_begin takes care of the basic task of block allocation and 1919155130a4SChristoph Hellwig * bringing partial write blocks uptodate first. 1920155130a4SChristoph Hellwig * 19217bb46a67Snpiggin@suse.de * The filesystem needs to handle block truncation upon failure. 1922afddba49SNick Piggin */ 1923155130a4SChristoph Hellwig int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len, 1924155130a4SChristoph Hellwig unsigned flags, struct page **pagep, get_block_t *get_block) 1925afddba49SNick Piggin { 19266e1db88dSChristoph Hellwig pgoff_t index = pos >> PAGE_CACHE_SHIFT; 1927afddba49SNick Piggin struct page *page; 19286e1db88dSChristoph Hellwig int status; 1929afddba49SNick Piggin 193054566b2cSNick Piggin page = grab_cache_page_write_begin(mapping, index, flags); 19316e1db88dSChristoph Hellwig if (!page) 19326e1db88dSChristoph Hellwig return -ENOMEM; 1933afddba49SNick Piggin 19346e1db88dSChristoph Hellwig status = __block_write_begin(page, pos, len, get_block); 1935afddba49SNick Piggin if (unlikely(status)) { 1936afddba49SNick Piggin unlock_page(page); 1937afddba49SNick Piggin page_cache_release(page); 19386e1db88dSChristoph Hellwig page = NULL; 1939afddba49SNick Piggin } 1940afddba49SNick Piggin 19416e1db88dSChristoph Hellwig *pagep = page; 1942afddba49SNick Piggin return status; 1943afddba49SNick Piggin } 1944afddba49SNick Piggin EXPORT_SYMBOL(block_write_begin); 1945afddba49SNick Piggin 1946afddba49SNick Piggin int block_write_end(struct file *file, struct address_space *mapping, 1947afddba49SNick Piggin loff_t pos, unsigned len, unsigned copied, 1948afddba49SNick Piggin struct page *page, void *fsdata) 1949afddba49SNick Piggin { 1950afddba49SNick Piggin struct inode *inode = mapping->host; 1951afddba49SNick Piggin unsigned start; 1952afddba49SNick Piggin 1953afddba49SNick Piggin start = pos & (PAGE_CACHE_SIZE - 1); 1954afddba49SNick Piggin 1955afddba49SNick Piggin if (unlikely(copied < len)) { 1956afddba49SNick Piggin /* 1957afddba49SNick Piggin * The buffers that were written will now be uptodate, so we 1958afddba49SNick Piggin * don't have to worry about a readpage reading them and 1959afddba49SNick Piggin * overwriting a partial write. However if we have encountered 1960afddba49SNick Piggin * a short write and only partially written into a buffer, it 1961afddba49SNick Piggin * will not be marked uptodate, so a readpage might come in and 1962afddba49SNick Piggin * destroy our partial write. 1963afddba49SNick Piggin * 1964afddba49SNick Piggin * Do the simplest thing, and just treat any short write to a 1965afddba49SNick Piggin * non uptodate page as a zero-length write, and force the 1966afddba49SNick Piggin * caller to redo the whole thing. 1967afddba49SNick Piggin */ 1968afddba49SNick Piggin if (!PageUptodate(page)) 1969afddba49SNick Piggin copied = 0; 1970afddba49SNick Piggin 1971afddba49SNick Piggin page_zero_new_buffers(page, start+copied, start+len); 1972afddba49SNick Piggin } 1973afddba49SNick Piggin flush_dcache_page(page); 1974afddba49SNick Piggin 1975afddba49SNick Piggin /* This could be a short (even 0-length) commit */ 1976afddba49SNick Piggin __block_commit_write(inode, page, start, start+copied); 1977afddba49SNick Piggin 1978afddba49SNick Piggin return copied; 1979afddba49SNick Piggin } 1980afddba49SNick Piggin EXPORT_SYMBOL(block_write_end); 1981afddba49SNick Piggin 1982afddba49SNick Piggin int generic_write_end(struct file *file, struct address_space *mapping, 1983afddba49SNick Piggin loff_t pos, unsigned len, unsigned copied, 1984afddba49SNick Piggin struct page *page, void *fsdata) 1985afddba49SNick Piggin { 1986afddba49SNick Piggin struct inode *inode = mapping->host; 1987c7d206b3SJan Kara int i_size_changed = 0; 1988afddba49SNick Piggin 1989afddba49SNick Piggin copied = block_write_end(file, mapping, pos, len, copied, page, fsdata); 1990afddba49SNick Piggin 1991afddba49SNick Piggin /* 1992afddba49SNick Piggin * No need to use i_size_read() here, the i_size 1993afddba49SNick Piggin * cannot change under us because we hold i_mutex. 1994afddba49SNick Piggin * 1995afddba49SNick Piggin * But it's important to update i_size while still holding page lock: 1996afddba49SNick Piggin * page writeout could otherwise come in and zero beyond i_size. 1997afddba49SNick Piggin */ 1998afddba49SNick Piggin if (pos+copied > inode->i_size) { 1999afddba49SNick Piggin i_size_write(inode, pos+copied); 2000c7d206b3SJan Kara i_size_changed = 1; 2001afddba49SNick Piggin } 2002afddba49SNick Piggin 2003afddba49SNick Piggin unlock_page(page); 2004afddba49SNick Piggin page_cache_release(page); 2005afddba49SNick Piggin 2006c7d206b3SJan Kara /* 2007c7d206b3SJan Kara * Don't mark the inode dirty under page lock. First, it unnecessarily 2008c7d206b3SJan Kara * makes the holding time of page lock longer. Second, it forces lock 2009c7d206b3SJan Kara * ordering of page lock and transaction start for journaling 2010c7d206b3SJan Kara * filesystems. 2011c7d206b3SJan Kara */ 2012c7d206b3SJan Kara if (i_size_changed) 2013c7d206b3SJan Kara mark_inode_dirty(inode); 2014c7d206b3SJan Kara 2015afddba49SNick Piggin return copied; 2016afddba49SNick Piggin } 2017afddba49SNick Piggin EXPORT_SYMBOL(generic_write_end); 2018afddba49SNick Piggin 2019afddba49SNick Piggin /* 20208ab22b9aSHisashi Hifumi * block_is_partially_uptodate checks whether buffers within a page are 20218ab22b9aSHisashi Hifumi * uptodate or not. 20228ab22b9aSHisashi Hifumi * 20238ab22b9aSHisashi Hifumi * Returns true if all buffers which correspond to a file portion 20248ab22b9aSHisashi Hifumi * we want to read are uptodate. 20258ab22b9aSHisashi Hifumi */ 20268ab22b9aSHisashi Hifumi int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc, 20278ab22b9aSHisashi Hifumi unsigned long from) 20288ab22b9aSHisashi Hifumi { 20298ab22b9aSHisashi Hifumi struct inode *inode = page->mapping->host; 20308ab22b9aSHisashi Hifumi unsigned block_start, block_end, blocksize; 20318ab22b9aSHisashi Hifumi unsigned to; 20328ab22b9aSHisashi Hifumi struct buffer_head *bh, *head; 20338ab22b9aSHisashi Hifumi int ret = 1; 20348ab22b9aSHisashi Hifumi 20358ab22b9aSHisashi Hifumi if (!page_has_buffers(page)) 20368ab22b9aSHisashi Hifumi return 0; 20378ab22b9aSHisashi Hifumi 20388ab22b9aSHisashi Hifumi blocksize = 1 << inode->i_blkbits; 20398ab22b9aSHisashi Hifumi to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count); 20408ab22b9aSHisashi Hifumi to = from + to; 20418ab22b9aSHisashi Hifumi if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize) 20428ab22b9aSHisashi Hifumi return 0; 20438ab22b9aSHisashi Hifumi 20448ab22b9aSHisashi Hifumi head = page_buffers(page); 20458ab22b9aSHisashi Hifumi bh = head; 20468ab22b9aSHisashi Hifumi block_start = 0; 20478ab22b9aSHisashi Hifumi do { 20488ab22b9aSHisashi Hifumi block_end = block_start + blocksize; 20498ab22b9aSHisashi Hifumi if (block_end > from && block_start < to) { 20508ab22b9aSHisashi Hifumi if (!buffer_uptodate(bh)) { 20518ab22b9aSHisashi Hifumi ret = 0; 20528ab22b9aSHisashi Hifumi break; 20538ab22b9aSHisashi Hifumi } 20548ab22b9aSHisashi Hifumi if (block_end >= to) 20558ab22b9aSHisashi Hifumi break; 20568ab22b9aSHisashi Hifumi } 20578ab22b9aSHisashi Hifumi block_start = block_end; 20588ab22b9aSHisashi Hifumi bh = bh->b_this_page; 20598ab22b9aSHisashi Hifumi } while (bh != head); 20608ab22b9aSHisashi Hifumi 20618ab22b9aSHisashi Hifumi return ret; 20628ab22b9aSHisashi Hifumi } 20638ab22b9aSHisashi Hifumi EXPORT_SYMBOL(block_is_partially_uptodate); 20648ab22b9aSHisashi Hifumi 20658ab22b9aSHisashi Hifumi /* 20661da177e4SLinus Torvalds * Generic "read page" function for block devices that have the normal 20671da177e4SLinus Torvalds * get_block functionality. This is most of the block device filesystems. 20681da177e4SLinus Torvalds * Reads the page asynchronously --- the unlock_buffer() and 20691da177e4SLinus Torvalds * set/clear_buffer_uptodate() functions propagate buffer state into the 20701da177e4SLinus Torvalds * page struct once IO has completed. 20711da177e4SLinus Torvalds */ 20721da177e4SLinus Torvalds int block_read_full_page(struct page *page, get_block_t *get_block) 20731da177e4SLinus Torvalds { 20741da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 20751da177e4SLinus Torvalds sector_t iblock, lblock; 20761da177e4SLinus Torvalds struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE]; 20771da177e4SLinus Torvalds unsigned int blocksize; 20781da177e4SLinus Torvalds int nr, i; 20791da177e4SLinus Torvalds int fully_mapped = 1; 20801da177e4SLinus Torvalds 2081cd7619d6SMatt Mackall BUG_ON(!PageLocked(page)); 20821da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 20831da177e4SLinus Torvalds if (!page_has_buffers(page)) 20841da177e4SLinus Torvalds create_empty_buffers(page, blocksize, 0); 20851da177e4SLinus Torvalds head = page_buffers(page); 20861da177e4SLinus Torvalds 20871da177e4SLinus Torvalds iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 20881da177e4SLinus Torvalds lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits; 20891da177e4SLinus Torvalds bh = head; 20901da177e4SLinus Torvalds nr = 0; 20911da177e4SLinus Torvalds i = 0; 20921da177e4SLinus Torvalds 20931da177e4SLinus Torvalds do { 20941da177e4SLinus Torvalds if (buffer_uptodate(bh)) 20951da177e4SLinus Torvalds continue; 20961da177e4SLinus Torvalds 20971da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 2098c64610baSAndrew Morton int err = 0; 2099c64610baSAndrew Morton 21001da177e4SLinus Torvalds fully_mapped = 0; 21011da177e4SLinus Torvalds if (iblock < lblock) { 2102b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 2103c64610baSAndrew Morton err = get_block(inode, iblock, bh, 0); 2104c64610baSAndrew Morton if (err) 21051da177e4SLinus Torvalds SetPageError(page); 21061da177e4SLinus Torvalds } 21071da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 2108eebd2aa3SChristoph Lameter zero_user(page, i * blocksize, blocksize); 2109c64610baSAndrew Morton if (!err) 21101da177e4SLinus Torvalds set_buffer_uptodate(bh); 21111da177e4SLinus Torvalds continue; 21121da177e4SLinus Torvalds } 21131da177e4SLinus Torvalds /* 21141da177e4SLinus Torvalds * get_block() might have updated the buffer 21151da177e4SLinus Torvalds * synchronously 21161da177e4SLinus Torvalds */ 21171da177e4SLinus Torvalds if (buffer_uptodate(bh)) 21181da177e4SLinus Torvalds continue; 21191da177e4SLinus Torvalds } 21201da177e4SLinus Torvalds arr[nr++] = bh; 21211da177e4SLinus Torvalds } while (i++, iblock++, (bh = bh->b_this_page) != head); 21221da177e4SLinus Torvalds 21231da177e4SLinus Torvalds if (fully_mapped) 21241da177e4SLinus Torvalds SetPageMappedToDisk(page); 21251da177e4SLinus Torvalds 21261da177e4SLinus Torvalds if (!nr) { 21271da177e4SLinus Torvalds /* 21281da177e4SLinus Torvalds * All buffers are uptodate - we can set the page uptodate 21291da177e4SLinus Torvalds * as well. But not if get_block() returned an error. 21301da177e4SLinus Torvalds */ 21311da177e4SLinus Torvalds if (!PageError(page)) 21321da177e4SLinus Torvalds SetPageUptodate(page); 21331da177e4SLinus Torvalds unlock_page(page); 21341da177e4SLinus Torvalds return 0; 21351da177e4SLinus Torvalds } 21361da177e4SLinus Torvalds 21371da177e4SLinus Torvalds /* Stage two: lock the buffers */ 21381da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 21391da177e4SLinus Torvalds bh = arr[i]; 21401da177e4SLinus Torvalds lock_buffer(bh); 21411da177e4SLinus Torvalds mark_buffer_async_read(bh); 21421da177e4SLinus Torvalds } 21431da177e4SLinus Torvalds 21441da177e4SLinus Torvalds /* 21451da177e4SLinus Torvalds * Stage 3: start the IO. Check for uptodateness 21461da177e4SLinus Torvalds * inside the buffer lock in case another process reading 21471da177e4SLinus Torvalds * the underlying blockdev brought it uptodate (the sct fix). 21481da177e4SLinus Torvalds */ 21491da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 21501da177e4SLinus Torvalds bh = arr[i]; 21511da177e4SLinus Torvalds if (buffer_uptodate(bh)) 21521da177e4SLinus Torvalds end_buffer_async_read(bh, 1); 21531da177e4SLinus Torvalds else 21541da177e4SLinus Torvalds submit_bh(READ, bh); 21551da177e4SLinus Torvalds } 21561da177e4SLinus Torvalds return 0; 21571da177e4SLinus Torvalds } 21581fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(block_read_full_page); 21591da177e4SLinus Torvalds 21601da177e4SLinus Torvalds /* utility function for filesystems that need to do work on expanding 216189e10787SNick Piggin * truncates. Uses filesystem pagecache writes to allow the filesystem to 21621da177e4SLinus Torvalds * deal with the hole. 21631da177e4SLinus Torvalds */ 216489e10787SNick Piggin int generic_cont_expand_simple(struct inode *inode, loff_t size) 21651da177e4SLinus Torvalds { 21661da177e4SLinus Torvalds struct address_space *mapping = inode->i_mapping; 21671da177e4SLinus Torvalds struct page *page; 216889e10787SNick Piggin void *fsdata; 21691da177e4SLinus Torvalds int err; 21701da177e4SLinus Torvalds 2171c08d3b0eSnpiggin@suse.de err = inode_newsize_ok(inode, size); 2172c08d3b0eSnpiggin@suse.de if (err) 21731da177e4SLinus Torvalds goto out; 21741da177e4SLinus Torvalds 217589e10787SNick Piggin err = pagecache_write_begin(NULL, mapping, size, 0, 217689e10787SNick Piggin AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND, 217789e10787SNick Piggin &page, &fsdata); 217889e10787SNick Piggin if (err) 217905eb0b51SOGAWA Hirofumi goto out; 218005eb0b51SOGAWA Hirofumi 218189e10787SNick Piggin err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata); 218289e10787SNick Piggin BUG_ON(err > 0); 218305eb0b51SOGAWA Hirofumi 218405eb0b51SOGAWA Hirofumi out: 218505eb0b51SOGAWA Hirofumi return err; 218605eb0b51SOGAWA Hirofumi } 21871fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(generic_cont_expand_simple); 218805eb0b51SOGAWA Hirofumi 2189f1e3af72SAdrian Bunk static int cont_expand_zero(struct file *file, struct address_space *mapping, 219089e10787SNick Piggin loff_t pos, loff_t *bytes) 219105eb0b51SOGAWA Hirofumi { 219289e10787SNick Piggin struct inode *inode = mapping->host; 219389e10787SNick Piggin unsigned blocksize = 1 << inode->i_blkbits; 219489e10787SNick Piggin struct page *page; 219589e10787SNick Piggin void *fsdata; 219689e10787SNick Piggin pgoff_t index, curidx; 219789e10787SNick Piggin loff_t curpos; 219889e10787SNick Piggin unsigned zerofrom, offset, len; 219989e10787SNick Piggin int err = 0; 220005eb0b51SOGAWA Hirofumi 220189e10787SNick Piggin index = pos >> PAGE_CACHE_SHIFT; 220289e10787SNick Piggin offset = pos & ~PAGE_CACHE_MASK; 220389e10787SNick Piggin 220489e10787SNick Piggin while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) { 220589e10787SNick Piggin zerofrom = curpos & ~PAGE_CACHE_MASK; 220689e10787SNick Piggin if (zerofrom & (blocksize-1)) { 220789e10787SNick Piggin *bytes |= (blocksize-1); 220889e10787SNick Piggin (*bytes)++; 220989e10787SNick Piggin } 221089e10787SNick Piggin len = PAGE_CACHE_SIZE - zerofrom; 221189e10787SNick Piggin 221289e10787SNick Piggin err = pagecache_write_begin(file, mapping, curpos, len, 221389e10787SNick Piggin AOP_FLAG_UNINTERRUPTIBLE, 221489e10787SNick Piggin &page, &fsdata); 221589e10787SNick Piggin if (err) 221689e10787SNick Piggin goto out; 2217eebd2aa3SChristoph Lameter zero_user(page, zerofrom, len); 221889e10787SNick Piggin err = pagecache_write_end(file, mapping, curpos, len, len, 221989e10787SNick Piggin page, fsdata); 222089e10787SNick Piggin if (err < 0) 222189e10787SNick Piggin goto out; 222289e10787SNick Piggin BUG_ON(err != len); 222389e10787SNick Piggin err = 0; 2224061e9746SOGAWA Hirofumi 2225061e9746SOGAWA Hirofumi balance_dirty_pages_ratelimited(mapping); 222689e10787SNick Piggin } 222789e10787SNick Piggin 222889e10787SNick Piggin /* page covers the boundary, find the boundary offset */ 222989e10787SNick Piggin if (index == curidx) { 223089e10787SNick Piggin zerofrom = curpos & ~PAGE_CACHE_MASK; 223189e10787SNick Piggin /* if we will expand the thing last block will be filled */ 223289e10787SNick Piggin if (offset <= zerofrom) { 223389e10787SNick Piggin goto out; 223489e10787SNick Piggin } 223589e10787SNick Piggin if (zerofrom & (blocksize-1)) { 223689e10787SNick Piggin *bytes |= (blocksize-1); 223789e10787SNick Piggin (*bytes)++; 223889e10787SNick Piggin } 223989e10787SNick Piggin len = offset - zerofrom; 224089e10787SNick Piggin 224189e10787SNick Piggin err = pagecache_write_begin(file, mapping, curpos, len, 224289e10787SNick Piggin AOP_FLAG_UNINTERRUPTIBLE, 224389e10787SNick Piggin &page, &fsdata); 224489e10787SNick Piggin if (err) 224589e10787SNick Piggin goto out; 2246eebd2aa3SChristoph Lameter zero_user(page, zerofrom, len); 224789e10787SNick Piggin err = pagecache_write_end(file, mapping, curpos, len, len, 224889e10787SNick Piggin page, fsdata); 224989e10787SNick Piggin if (err < 0) 225089e10787SNick Piggin goto out; 225189e10787SNick Piggin BUG_ON(err != len); 225289e10787SNick Piggin err = 0; 225389e10787SNick Piggin } 225489e10787SNick Piggin out: 225589e10787SNick Piggin return err; 22561da177e4SLinus Torvalds } 22571da177e4SLinus Torvalds 22581da177e4SLinus Torvalds /* 22591da177e4SLinus Torvalds * For moronic filesystems that do not allow holes in file. 22601da177e4SLinus Torvalds * We may have to extend the file. 22611da177e4SLinus Torvalds */ 2262282dc178SChristoph Hellwig int cont_write_begin(struct file *file, struct address_space *mapping, 226389e10787SNick Piggin loff_t pos, unsigned len, unsigned flags, 226489e10787SNick Piggin struct page **pagep, void **fsdata, 226589e10787SNick Piggin get_block_t *get_block, loff_t *bytes) 22661da177e4SLinus Torvalds { 22671da177e4SLinus Torvalds struct inode *inode = mapping->host; 22681da177e4SLinus Torvalds unsigned blocksize = 1 << inode->i_blkbits; 226989e10787SNick Piggin unsigned zerofrom; 227089e10787SNick Piggin int err; 22711da177e4SLinus Torvalds 227289e10787SNick Piggin err = cont_expand_zero(file, mapping, pos, bytes); 227389e10787SNick Piggin if (err) 2274155130a4SChristoph Hellwig return err; 22751da177e4SLinus Torvalds 22761da177e4SLinus Torvalds zerofrom = *bytes & ~PAGE_CACHE_MASK; 227789e10787SNick Piggin if (pos+len > *bytes && zerofrom & (blocksize-1)) { 22781da177e4SLinus Torvalds *bytes |= (blocksize-1); 22791da177e4SLinus Torvalds (*bytes)++; 22801da177e4SLinus Torvalds } 22811da177e4SLinus Torvalds 2282155130a4SChristoph Hellwig return block_write_begin(mapping, pos, len, flags, pagep, get_block); 22831da177e4SLinus Torvalds } 22841fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(cont_write_begin); 22851da177e4SLinus Torvalds 22861da177e4SLinus Torvalds int block_commit_write(struct page *page, unsigned from, unsigned to) 22871da177e4SLinus Torvalds { 22881da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 22891da177e4SLinus Torvalds __block_commit_write(inode,page,from,to); 22901da177e4SLinus Torvalds return 0; 22911da177e4SLinus Torvalds } 22921fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(block_commit_write); 22931da177e4SLinus Torvalds 229454171690SDavid Chinner /* 229554171690SDavid Chinner * block_page_mkwrite() is not allowed to change the file size as it gets 229654171690SDavid Chinner * called from a page fault handler when a page is first dirtied. Hence we must 229754171690SDavid Chinner * be careful to check for EOF conditions here. We set the page up correctly 229854171690SDavid Chinner * for a written page which means we get ENOSPC checking when writing into 229954171690SDavid Chinner * holes and correct delalloc and unwritten extent mapping on filesystems that 230054171690SDavid Chinner * support these features. 230154171690SDavid Chinner * 230254171690SDavid Chinner * We are not allowed to take the i_mutex here so we have to play games to 230354171690SDavid Chinner * protect against truncate races as the page could now be beyond EOF. Because 23047bb46a67Snpiggin@suse.de * truncate writes the inode size before removing pages, once we have the 230554171690SDavid Chinner * page lock we can determine safely if the page is beyond EOF. If it is not 230654171690SDavid Chinner * beyond EOF, then the page is guaranteed safe against truncation until we 230754171690SDavid Chinner * unlock the page. 2308ea13a864SJan Kara * 2309ea13a864SJan Kara * Direct callers of this function should call vfs_check_frozen() so that page 2310ea13a864SJan Kara * fault does not busyloop until the fs is thawed. 231154171690SDavid Chinner */ 231224da4fabSJan Kara int __block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, 231354171690SDavid Chinner get_block_t get_block) 231454171690SDavid Chinner { 2315c2ec175cSNick Piggin struct page *page = vmf->page; 231654171690SDavid Chinner struct inode *inode = vma->vm_file->f_path.dentry->d_inode; 231754171690SDavid Chinner unsigned long end; 231854171690SDavid Chinner loff_t size; 231924da4fabSJan Kara int ret; 232054171690SDavid Chinner 232154171690SDavid Chinner lock_page(page); 232254171690SDavid Chinner size = i_size_read(inode); 232354171690SDavid Chinner if ((page->mapping != inode->i_mapping) || 232418336338SNick Piggin (page_offset(page) > size)) { 232524da4fabSJan Kara /* We overload EFAULT to mean page got truncated */ 232624da4fabSJan Kara ret = -EFAULT; 232724da4fabSJan Kara goto out_unlock; 232854171690SDavid Chinner } 232954171690SDavid Chinner 233054171690SDavid Chinner /* page is wholly or partially inside EOF */ 233154171690SDavid Chinner if (((page->index + 1) << PAGE_CACHE_SHIFT) > size) 233254171690SDavid Chinner end = size & ~PAGE_CACHE_MASK; 233354171690SDavid Chinner else 233454171690SDavid Chinner end = PAGE_CACHE_SIZE; 233554171690SDavid Chinner 2336ebdec241SChristoph Hellwig ret = __block_write_begin(page, 0, end, get_block); 233754171690SDavid Chinner if (!ret) 233854171690SDavid Chinner ret = block_commit_write(page, 0, end); 233954171690SDavid Chinner 234024da4fabSJan Kara if (unlikely(ret < 0)) 234124da4fabSJan Kara goto out_unlock; 2342ea13a864SJan Kara /* 2343ea13a864SJan Kara * Freezing in progress? We check after the page is marked dirty and 2344ea13a864SJan Kara * with page lock held so if the test here fails, we are sure freezing 2345ea13a864SJan Kara * code will wait during syncing until the page fault is done - at that 2346ea13a864SJan Kara * point page will be dirty and unlocked so freezing code will write it 2347ea13a864SJan Kara * and writeprotect it again. 2348ea13a864SJan Kara */ 2349ea13a864SJan Kara set_page_dirty(page); 2350ea13a864SJan Kara if (inode->i_sb->s_frozen != SB_UNFROZEN) { 2351ea13a864SJan Kara ret = -EAGAIN; 2352ea13a864SJan Kara goto out_unlock; 2353ea13a864SJan Kara } 2354d76ee18aSDarrick J. Wong wait_on_page_writeback(page); 235524da4fabSJan Kara return 0; 235624da4fabSJan Kara out_unlock: 2357b827e496SNick Piggin unlock_page(page); 235854171690SDavid Chinner return ret; 235954171690SDavid Chinner } 236024da4fabSJan Kara EXPORT_SYMBOL(__block_page_mkwrite); 236124da4fabSJan Kara 236224da4fabSJan Kara int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, 236324da4fabSJan Kara get_block_t get_block) 236424da4fabSJan Kara { 2365ea13a864SJan Kara int ret; 2366ea13a864SJan Kara struct super_block *sb = vma->vm_file->f_path.dentry->d_inode->i_sb; 236724da4fabSJan Kara 2368ea13a864SJan Kara /* 2369ea13a864SJan Kara * This check is racy but catches the common case. The check in 2370ea13a864SJan Kara * __block_page_mkwrite() is reliable. 2371ea13a864SJan Kara */ 2372ea13a864SJan Kara vfs_check_frozen(sb, SB_FREEZE_WRITE); 2373ea13a864SJan Kara ret = __block_page_mkwrite(vma, vmf, get_block); 237424da4fabSJan Kara return block_page_mkwrite_return(ret); 237524da4fabSJan Kara } 23761fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(block_page_mkwrite); 23771da177e4SLinus Torvalds 23781da177e4SLinus Torvalds /* 237903158cd7SNick Piggin * nobh_write_begin()'s prereads are special: the buffer_heads are freed 23801da177e4SLinus Torvalds * immediately, while under the page lock. So it needs a special end_io 23811da177e4SLinus Torvalds * handler which does not touch the bh after unlocking it. 23821da177e4SLinus Torvalds */ 23831da177e4SLinus Torvalds static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate) 23841da177e4SLinus Torvalds { 238568671f35SDmitry Monakhov __end_buffer_read_notouch(bh, uptodate); 23861da177e4SLinus Torvalds } 23871da177e4SLinus Torvalds 23881da177e4SLinus Torvalds /* 238903158cd7SNick Piggin * Attach the singly-linked list of buffers created by nobh_write_begin, to 239003158cd7SNick Piggin * the page (converting it to circular linked list and taking care of page 239103158cd7SNick Piggin * dirty races). 239203158cd7SNick Piggin */ 239303158cd7SNick Piggin static void attach_nobh_buffers(struct page *page, struct buffer_head *head) 239403158cd7SNick Piggin { 239503158cd7SNick Piggin struct buffer_head *bh; 239603158cd7SNick Piggin 239703158cd7SNick Piggin BUG_ON(!PageLocked(page)); 239803158cd7SNick Piggin 239903158cd7SNick Piggin spin_lock(&page->mapping->private_lock); 240003158cd7SNick Piggin bh = head; 240103158cd7SNick Piggin do { 240203158cd7SNick Piggin if (PageDirty(page)) 240303158cd7SNick Piggin set_buffer_dirty(bh); 240403158cd7SNick Piggin if (!bh->b_this_page) 240503158cd7SNick Piggin bh->b_this_page = head; 240603158cd7SNick Piggin bh = bh->b_this_page; 240703158cd7SNick Piggin } while (bh != head); 240803158cd7SNick Piggin attach_page_buffers(page, head); 240903158cd7SNick Piggin spin_unlock(&page->mapping->private_lock); 241003158cd7SNick Piggin } 241103158cd7SNick Piggin 241203158cd7SNick Piggin /* 2413ea0f04e5SChristoph Hellwig * On entry, the page is fully not uptodate. 2414ea0f04e5SChristoph Hellwig * On exit the page is fully uptodate in the areas outside (from,to) 24157bb46a67Snpiggin@suse.de * The filesystem needs to handle block truncation upon failure. 24161da177e4SLinus Torvalds */ 2417ea0f04e5SChristoph Hellwig int nobh_write_begin(struct address_space *mapping, 241803158cd7SNick Piggin loff_t pos, unsigned len, unsigned flags, 241903158cd7SNick Piggin struct page **pagep, void **fsdata, 24201da177e4SLinus Torvalds get_block_t *get_block) 24211da177e4SLinus Torvalds { 242203158cd7SNick Piggin struct inode *inode = mapping->host; 24231da177e4SLinus Torvalds const unsigned blkbits = inode->i_blkbits; 24241da177e4SLinus Torvalds const unsigned blocksize = 1 << blkbits; 2425a4b0672dSNick Piggin struct buffer_head *head, *bh; 242603158cd7SNick Piggin struct page *page; 242703158cd7SNick Piggin pgoff_t index; 242803158cd7SNick Piggin unsigned from, to; 24291da177e4SLinus Torvalds unsigned block_in_page; 2430a4b0672dSNick Piggin unsigned block_start, block_end; 24311da177e4SLinus Torvalds sector_t block_in_file; 24321da177e4SLinus Torvalds int nr_reads = 0; 24331da177e4SLinus Torvalds int ret = 0; 24341da177e4SLinus Torvalds int is_mapped_to_disk = 1; 24351da177e4SLinus Torvalds 243603158cd7SNick Piggin index = pos >> PAGE_CACHE_SHIFT; 243703158cd7SNick Piggin from = pos & (PAGE_CACHE_SIZE - 1); 243803158cd7SNick Piggin to = from + len; 243903158cd7SNick Piggin 244054566b2cSNick Piggin page = grab_cache_page_write_begin(mapping, index, flags); 244103158cd7SNick Piggin if (!page) 244203158cd7SNick Piggin return -ENOMEM; 244303158cd7SNick Piggin *pagep = page; 244403158cd7SNick Piggin *fsdata = NULL; 244503158cd7SNick Piggin 244603158cd7SNick Piggin if (page_has_buffers(page)) { 2447309f77adSNamhyung Kim ret = __block_write_begin(page, pos, len, get_block); 2448309f77adSNamhyung Kim if (unlikely(ret)) 2449309f77adSNamhyung Kim goto out_release; 2450309f77adSNamhyung Kim return ret; 245103158cd7SNick Piggin } 2452a4b0672dSNick Piggin 24531da177e4SLinus Torvalds if (PageMappedToDisk(page)) 24541da177e4SLinus Torvalds return 0; 24551da177e4SLinus Torvalds 2456a4b0672dSNick Piggin /* 2457a4b0672dSNick Piggin * Allocate buffers so that we can keep track of state, and potentially 2458a4b0672dSNick Piggin * attach them to the page if an error occurs. In the common case of 2459a4b0672dSNick Piggin * no error, they will just be freed again without ever being attached 2460a4b0672dSNick Piggin * to the page (which is all OK, because we're under the page lock). 2461a4b0672dSNick Piggin * 2462a4b0672dSNick Piggin * Be careful: the buffer linked list is a NULL terminated one, rather 2463a4b0672dSNick Piggin * than the circular one we're used to. 2464a4b0672dSNick Piggin */ 2465a4b0672dSNick Piggin head = alloc_page_buffers(page, blocksize, 0); 246603158cd7SNick Piggin if (!head) { 246703158cd7SNick Piggin ret = -ENOMEM; 246803158cd7SNick Piggin goto out_release; 246903158cd7SNick Piggin } 2470a4b0672dSNick Piggin 24711da177e4SLinus Torvalds block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits); 24721da177e4SLinus Torvalds 24731da177e4SLinus Torvalds /* 24741da177e4SLinus Torvalds * We loop across all blocks in the page, whether or not they are 24751da177e4SLinus Torvalds * part of the affected region. This is so we can discover if the 24761da177e4SLinus Torvalds * page is fully mapped-to-disk. 24771da177e4SLinus Torvalds */ 2478a4b0672dSNick Piggin for (block_start = 0, block_in_page = 0, bh = head; 24791da177e4SLinus Torvalds block_start < PAGE_CACHE_SIZE; 2480a4b0672dSNick Piggin block_in_page++, block_start += blocksize, bh = bh->b_this_page) { 24811da177e4SLinus Torvalds int create; 24821da177e4SLinus Torvalds 2483a4b0672dSNick Piggin block_end = block_start + blocksize; 2484a4b0672dSNick Piggin bh->b_state = 0; 24851da177e4SLinus Torvalds create = 1; 24861da177e4SLinus Torvalds if (block_start >= to) 24871da177e4SLinus Torvalds create = 0; 24881da177e4SLinus Torvalds ret = get_block(inode, block_in_file + block_in_page, 2489a4b0672dSNick Piggin bh, create); 24901da177e4SLinus Torvalds if (ret) 24911da177e4SLinus Torvalds goto failed; 2492a4b0672dSNick Piggin if (!buffer_mapped(bh)) 24931da177e4SLinus Torvalds is_mapped_to_disk = 0; 2494a4b0672dSNick Piggin if (buffer_new(bh)) 2495a4b0672dSNick Piggin unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr); 2496a4b0672dSNick Piggin if (PageUptodate(page)) { 2497a4b0672dSNick Piggin set_buffer_uptodate(bh); 24981da177e4SLinus Torvalds continue; 2499a4b0672dSNick Piggin } 2500a4b0672dSNick Piggin if (buffer_new(bh) || !buffer_mapped(bh)) { 2501eebd2aa3SChristoph Lameter zero_user_segments(page, block_start, from, 2502eebd2aa3SChristoph Lameter to, block_end); 25031da177e4SLinus Torvalds continue; 25041da177e4SLinus Torvalds } 2505a4b0672dSNick Piggin if (buffer_uptodate(bh)) 25061da177e4SLinus Torvalds continue; /* reiserfs does this */ 25071da177e4SLinus Torvalds if (block_start < from || block_end > to) { 2508a4b0672dSNick Piggin lock_buffer(bh); 2509a4b0672dSNick Piggin bh->b_end_io = end_buffer_read_nobh; 2510a4b0672dSNick Piggin submit_bh(READ, bh); 2511a4b0672dSNick Piggin nr_reads++; 25121da177e4SLinus Torvalds } 25131da177e4SLinus Torvalds } 25141da177e4SLinus Torvalds 25151da177e4SLinus Torvalds if (nr_reads) { 25161da177e4SLinus Torvalds /* 25171da177e4SLinus Torvalds * The page is locked, so these buffers are protected from 25181da177e4SLinus Torvalds * any VM or truncate activity. Hence we don't need to care 25191da177e4SLinus Torvalds * for the buffer_head refcounts. 25201da177e4SLinus Torvalds */ 2521a4b0672dSNick Piggin for (bh = head; bh; bh = bh->b_this_page) { 25221da177e4SLinus Torvalds wait_on_buffer(bh); 25231da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 25241da177e4SLinus Torvalds ret = -EIO; 25251da177e4SLinus Torvalds } 25261da177e4SLinus Torvalds if (ret) 25271da177e4SLinus Torvalds goto failed; 25281da177e4SLinus Torvalds } 25291da177e4SLinus Torvalds 25301da177e4SLinus Torvalds if (is_mapped_to_disk) 25311da177e4SLinus Torvalds SetPageMappedToDisk(page); 25321da177e4SLinus Torvalds 253303158cd7SNick Piggin *fsdata = head; /* to be released by nobh_write_end */ 2534a4b0672dSNick Piggin 25351da177e4SLinus Torvalds return 0; 25361da177e4SLinus Torvalds 25371da177e4SLinus Torvalds failed: 253803158cd7SNick Piggin BUG_ON(!ret); 25391da177e4SLinus Torvalds /* 2540a4b0672dSNick Piggin * Error recovery is a bit difficult. We need to zero out blocks that 2541a4b0672dSNick Piggin * were newly allocated, and dirty them to ensure they get written out. 2542a4b0672dSNick Piggin * Buffers need to be attached to the page at this point, otherwise 2543a4b0672dSNick Piggin * the handling of potential IO errors during writeout would be hard 2544a4b0672dSNick Piggin * (could try doing synchronous writeout, but what if that fails too?) 25451da177e4SLinus Torvalds */ 254603158cd7SNick Piggin attach_nobh_buffers(page, head); 254703158cd7SNick Piggin page_zero_new_buffers(page, from, to); 2548a4b0672dSNick Piggin 254903158cd7SNick Piggin out_release: 255003158cd7SNick Piggin unlock_page(page); 255103158cd7SNick Piggin page_cache_release(page); 255203158cd7SNick Piggin *pagep = NULL; 2553a4b0672dSNick Piggin 25547bb46a67Snpiggin@suse.de return ret; 25557bb46a67Snpiggin@suse.de } 255603158cd7SNick Piggin EXPORT_SYMBOL(nobh_write_begin); 25571da177e4SLinus Torvalds 255803158cd7SNick Piggin int nobh_write_end(struct file *file, struct address_space *mapping, 255903158cd7SNick Piggin loff_t pos, unsigned len, unsigned copied, 256003158cd7SNick Piggin struct page *page, void *fsdata) 25611da177e4SLinus Torvalds { 25621da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 2563efdc3131SNick Piggin struct buffer_head *head = fsdata; 256403158cd7SNick Piggin struct buffer_head *bh; 25655b41e74aSDmitri Monakhov BUG_ON(fsdata != NULL && page_has_buffers(page)); 25661da177e4SLinus Torvalds 2567d4cf109fSDave Kleikamp if (unlikely(copied < len) && head) 256803158cd7SNick Piggin attach_nobh_buffers(page, head); 2569a4b0672dSNick Piggin if (page_has_buffers(page)) 257003158cd7SNick Piggin return generic_write_end(file, mapping, pos, len, 257103158cd7SNick Piggin copied, page, fsdata); 2572a4b0672dSNick Piggin 257322c8ca78SNick Piggin SetPageUptodate(page); 25741da177e4SLinus Torvalds set_page_dirty(page); 257503158cd7SNick Piggin if (pos+copied > inode->i_size) { 257603158cd7SNick Piggin i_size_write(inode, pos+copied); 25771da177e4SLinus Torvalds mark_inode_dirty(inode); 25781da177e4SLinus Torvalds } 257903158cd7SNick Piggin 258003158cd7SNick Piggin unlock_page(page); 258103158cd7SNick Piggin page_cache_release(page); 258203158cd7SNick Piggin 258303158cd7SNick Piggin while (head) { 258403158cd7SNick Piggin bh = head; 258503158cd7SNick Piggin head = head->b_this_page; 258603158cd7SNick Piggin free_buffer_head(bh); 25871da177e4SLinus Torvalds } 258803158cd7SNick Piggin 258903158cd7SNick Piggin return copied; 259003158cd7SNick Piggin } 259103158cd7SNick Piggin EXPORT_SYMBOL(nobh_write_end); 25921da177e4SLinus Torvalds 25931da177e4SLinus Torvalds /* 25941da177e4SLinus Torvalds * nobh_writepage() - based on block_full_write_page() except 25951da177e4SLinus Torvalds * that it tries to operate without attaching bufferheads to 25961da177e4SLinus Torvalds * the page. 25971da177e4SLinus Torvalds */ 25981da177e4SLinus Torvalds int nobh_writepage(struct page *page, get_block_t *get_block, 25991da177e4SLinus Torvalds struct writeback_control *wbc) 26001da177e4SLinus Torvalds { 26011da177e4SLinus Torvalds struct inode * const inode = page->mapping->host; 26021da177e4SLinus Torvalds loff_t i_size = i_size_read(inode); 26031da177e4SLinus Torvalds const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; 26041da177e4SLinus Torvalds unsigned offset; 26051da177e4SLinus Torvalds int ret; 26061da177e4SLinus Torvalds 26071da177e4SLinus Torvalds /* Is the page fully inside i_size? */ 26081da177e4SLinus Torvalds if (page->index < end_index) 26091da177e4SLinus Torvalds goto out; 26101da177e4SLinus Torvalds 26111da177e4SLinus Torvalds /* Is the page fully outside i_size? (truncate in progress) */ 26121da177e4SLinus Torvalds offset = i_size & (PAGE_CACHE_SIZE-1); 26131da177e4SLinus Torvalds if (page->index >= end_index+1 || !offset) { 26141da177e4SLinus Torvalds /* 26151da177e4SLinus Torvalds * The page may have dirty, unmapped buffers. For example, 26161da177e4SLinus Torvalds * they may have been added in ext3_writepage(). Make them 26171da177e4SLinus Torvalds * freeable here, so the page does not leak. 26181da177e4SLinus Torvalds */ 26191da177e4SLinus Torvalds #if 0 26201da177e4SLinus Torvalds /* Not really sure about this - do we need this ? */ 26211da177e4SLinus Torvalds if (page->mapping->a_ops->invalidatepage) 26221da177e4SLinus Torvalds page->mapping->a_ops->invalidatepage(page, offset); 26231da177e4SLinus Torvalds #endif 26241da177e4SLinus Torvalds unlock_page(page); 26251da177e4SLinus Torvalds return 0; /* don't care */ 26261da177e4SLinus Torvalds } 26271da177e4SLinus Torvalds 26281da177e4SLinus Torvalds /* 26291da177e4SLinus Torvalds * The page straddles i_size. It must be zeroed out on each and every 26301da177e4SLinus Torvalds * writepage invocation because it may be mmapped. "A file is mapped 26311da177e4SLinus Torvalds * in multiples of the page size. For a file that is not a multiple of 26321da177e4SLinus Torvalds * the page size, the remaining memory is zeroed when mapped, and 26331da177e4SLinus Torvalds * writes to that region are not written out to the file." 26341da177e4SLinus Torvalds */ 2635eebd2aa3SChristoph Lameter zero_user_segment(page, offset, PAGE_CACHE_SIZE); 26361da177e4SLinus Torvalds out: 26371da177e4SLinus Torvalds ret = mpage_writepage(page, get_block, wbc); 26381da177e4SLinus Torvalds if (ret == -EAGAIN) 263935c80d5fSChris Mason ret = __block_write_full_page(inode, page, get_block, wbc, 264035c80d5fSChris Mason end_buffer_async_write); 26411da177e4SLinus Torvalds return ret; 26421da177e4SLinus Torvalds } 26431da177e4SLinus Torvalds EXPORT_SYMBOL(nobh_writepage); 26441da177e4SLinus Torvalds 264503158cd7SNick Piggin int nobh_truncate_page(struct address_space *mapping, 264603158cd7SNick Piggin loff_t from, get_block_t *get_block) 26471da177e4SLinus Torvalds { 26481da177e4SLinus Torvalds pgoff_t index = from >> PAGE_CACHE_SHIFT; 26491da177e4SLinus Torvalds unsigned offset = from & (PAGE_CACHE_SIZE-1); 265003158cd7SNick Piggin unsigned blocksize; 265103158cd7SNick Piggin sector_t iblock; 265203158cd7SNick Piggin unsigned length, pos; 265303158cd7SNick Piggin struct inode *inode = mapping->host; 26541da177e4SLinus Torvalds struct page *page; 265503158cd7SNick Piggin struct buffer_head map_bh; 265603158cd7SNick Piggin int err; 26571da177e4SLinus Torvalds 265803158cd7SNick Piggin blocksize = 1 << inode->i_blkbits; 265903158cd7SNick Piggin length = offset & (blocksize - 1); 26601da177e4SLinus Torvalds 266103158cd7SNick Piggin /* Block boundary? Nothing to do */ 266203158cd7SNick Piggin if (!length) 266303158cd7SNick Piggin return 0; 266403158cd7SNick Piggin 266503158cd7SNick Piggin length = blocksize - length; 266603158cd7SNick Piggin iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 266703158cd7SNick Piggin 26681da177e4SLinus Torvalds page = grab_cache_page(mapping, index); 266903158cd7SNick Piggin err = -ENOMEM; 26701da177e4SLinus Torvalds if (!page) 26711da177e4SLinus Torvalds goto out; 26721da177e4SLinus Torvalds 267303158cd7SNick Piggin if (page_has_buffers(page)) { 267403158cd7SNick Piggin has_buffers: 267503158cd7SNick Piggin unlock_page(page); 267603158cd7SNick Piggin page_cache_release(page); 267703158cd7SNick Piggin return block_truncate_page(mapping, from, get_block); 26781da177e4SLinus Torvalds } 267903158cd7SNick Piggin 268003158cd7SNick Piggin /* Find the buffer that contains "offset" */ 268103158cd7SNick Piggin pos = blocksize; 268203158cd7SNick Piggin while (offset >= pos) { 268303158cd7SNick Piggin iblock++; 268403158cd7SNick Piggin pos += blocksize; 268503158cd7SNick Piggin } 268603158cd7SNick Piggin 2687460bcf57STheodore Ts'o map_bh.b_size = blocksize; 2688460bcf57STheodore Ts'o map_bh.b_state = 0; 268903158cd7SNick Piggin err = get_block(inode, iblock, &map_bh, 0); 269003158cd7SNick Piggin if (err) 269103158cd7SNick Piggin goto unlock; 269203158cd7SNick Piggin /* unmapped? It's a hole - nothing to do */ 269303158cd7SNick Piggin if (!buffer_mapped(&map_bh)) 269403158cd7SNick Piggin goto unlock; 269503158cd7SNick Piggin 269603158cd7SNick Piggin /* Ok, it's mapped. Make sure it's up-to-date */ 269703158cd7SNick Piggin if (!PageUptodate(page)) { 269803158cd7SNick Piggin err = mapping->a_ops->readpage(NULL, page); 269903158cd7SNick Piggin if (err) { 270003158cd7SNick Piggin page_cache_release(page); 270103158cd7SNick Piggin goto out; 270203158cd7SNick Piggin } 270303158cd7SNick Piggin lock_page(page); 270403158cd7SNick Piggin if (!PageUptodate(page)) { 270503158cd7SNick Piggin err = -EIO; 270603158cd7SNick Piggin goto unlock; 270703158cd7SNick Piggin } 270803158cd7SNick Piggin if (page_has_buffers(page)) 270903158cd7SNick Piggin goto has_buffers; 271003158cd7SNick Piggin } 2711eebd2aa3SChristoph Lameter zero_user(page, offset, length); 271203158cd7SNick Piggin set_page_dirty(page); 271303158cd7SNick Piggin err = 0; 271403158cd7SNick Piggin 271503158cd7SNick Piggin unlock: 27161da177e4SLinus Torvalds unlock_page(page); 27171da177e4SLinus Torvalds page_cache_release(page); 27181da177e4SLinus Torvalds out: 271903158cd7SNick Piggin return err; 27201da177e4SLinus Torvalds } 27211da177e4SLinus Torvalds EXPORT_SYMBOL(nobh_truncate_page); 27221da177e4SLinus Torvalds 27231da177e4SLinus Torvalds int block_truncate_page(struct address_space *mapping, 27241da177e4SLinus Torvalds loff_t from, get_block_t *get_block) 27251da177e4SLinus Torvalds { 27261da177e4SLinus Torvalds pgoff_t index = from >> PAGE_CACHE_SHIFT; 27271da177e4SLinus Torvalds unsigned offset = from & (PAGE_CACHE_SIZE-1); 27281da177e4SLinus Torvalds unsigned blocksize; 272954b21a79SAndrew Morton sector_t iblock; 27301da177e4SLinus Torvalds unsigned length, pos; 27311da177e4SLinus Torvalds struct inode *inode = mapping->host; 27321da177e4SLinus Torvalds struct page *page; 27331da177e4SLinus Torvalds struct buffer_head *bh; 27341da177e4SLinus Torvalds int err; 27351da177e4SLinus Torvalds 27361da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 27371da177e4SLinus Torvalds length = offset & (blocksize - 1); 27381da177e4SLinus Torvalds 27391da177e4SLinus Torvalds /* Block boundary? Nothing to do */ 27401da177e4SLinus Torvalds if (!length) 27411da177e4SLinus Torvalds return 0; 27421da177e4SLinus Torvalds 27431da177e4SLinus Torvalds length = blocksize - length; 274454b21a79SAndrew Morton iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 27451da177e4SLinus Torvalds 27461da177e4SLinus Torvalds page = grab_cache_page(mapping, index); 27471da177e4SLinus Torvalds err = -ENOMEM; 27481da177e4SLinus Torvalds if (!page) 27491da177e4SLinus Torvalds goto out; 27501da177e4SLinus Torvalds 27511da177e4SLinus Torvalds if (!page_has_buffers(page)) 27521da177e4SLinus Torvalds create_empty_buffers(page, blocksize, 0); 27531da177e4SLinus Torvalds 27541da177e4SLinus Torvalds /* Find the buffer that contains "offset" */ 27551da177e4SLinus Torvalds bh = page_buffers(page); 27561da177e4SLinus Torvalds pos = blocksize; 27571da177e4SLinus Torvalds while (offset >= pos) { 27581da177e4SLinus Torvalds bh = bh->b_this_page; 27591da177e4SLinus Torvalds iblock++; 27601da177e4SLinus Torvalds pos += blocksize; 27611da177e4SLinus Torvalds } 27621da177e4SLinus Torvalds 27631da177e4SLinus Torvalds err = 0; 27641da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 2765b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 27661da177e4SLinus Torvalds err = get_block(inode, iblock, bh, 0); 27671da177e4SLinus Torvalds if (err) 27681da177e4SLinus Torvalds goto unlock; 27691da177e4SLinus Torvalds /* unmapped? It's a hole - nothing to do */ 27701da177e4SLinus Torvalds if (!buffer_mapped(bh)) 27711da177e4SLinus Torvalds goto unlock; 27721da177e4SLinus Torvalds } 27731da177e4SLinus Torvalds 27741da177e4SLinus Torvalds /* Ok, it's mapped. Make sure it's up-to-date */ 27751da177e4SLinus Torvalds if (PageUptodate(page)) 27761da177e4SLinus Torvalds set_buffer_uptodate(bh); 27771da177e4SLinus Torvalds 277833a266ddSDavid Chinner if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) { 27791da177e4SLinus Torvalds err = -EIO; 27801da177e4SLinus Torvalds ll_rw_block(READ, 1, &bh); 27811da177e4SLinus Torvalds wait_on_buffer(bh); 27821da177e4SLinus Torvalds /* Uhhuh. Read error. Complain and punt. */ 27831da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 27841da177e4SLinus Torvalds goto unlock; 27851da177e4SLinus Torvalds } 27861da177e4SLinus Torvalds 2787eebd2aa3SChristoph Lameter zero_user(page, offset, length); 27881da177e4SLinus Torvalds mark_buffer_dirty(bh); 27891da177e4SLinus Torvalds err = 0; 27901da177e4SLinus Torvalds 27911da177e4SLinus Torvalds unlock: 27921da177e4SLinus Torvalds unlock_page(page); 27931da177e4SLinus Torvalds page_cache_release(page); 27941da177e4SLinus Torvalds out: 27951da177e4SLinus Torvalds return err; 27961da177e4SLinus Torvalds } 27971fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(block_truncate_page); 27981da177e4SLinus Torvalds 27991da177e4SLinus Torvalds /* 28001da177e4SLinus Torvalds * The generic ->writepage function for buffer-backed address_spaces 280135c80d5fSChris Mason * this form passes in the end_io handler used to finish the IO. 28021da177e4SLinus Torvalds */ 280335c80d5fSChris Mason int block_write_full_page_endio(struct page *page, get_block_t *get_block, 280435c80d5fSChris Mason struct writeback_control *wbc, bh_end_io_t *handler) 28051da177e4SLinus Torvalds { 28061da177e4SLinus Torvalds struct inode * const inode = page->mapping->host; 28071da177e4SLinus Torvalds loff_t i_size = i_size_read(inode); 28081da177e4SLinus Torvalds const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; 28091da177e4SLinus Torvalds unsigned offset; 28101da177e4SLinus Torvalds 28111da177e4SLinus Torvalds /* Is the page fully inside i_size? */ 28121da177e4SLinus Torvalds if (page->index < end_index) 281335c80d5fSChris Mason return __block_write_full_page(inode, page, get_block, wbc, 281435c80d5fSChris Mason handler); 28151da177e4SLinus Torvalds 28161da177e4SLinus Torvalds /* Is the page fully outside i_size? (truncate in progress) */ 28171da177e4SLinus Torvalds offset = i_size & (PAGE_CACHE_SIZE-1); 28181da177e4SLinus Torvalds if (page->index >= end_index+1 || !offset) { 28191da177e4SLinus Torvalds /* 28201da177e4SLinus Torvalds * The page may have dirty, unmapped buffers. For example, 28211da177e4SLinus Torvalds * they may have been added in ext3_writepage(). Make them 28221da177e4SLinus Torvalds * freeable here, so the page does not leak. 28231da177e4SLinus Torvalds */ 2824aaa4059bSJan Kara do_invalidatepage(page, 0); 28251da177e4SLinus Torvalds unlock_page(page); 28261da177e4SLinus Torvalds return 0; /* don't care */ 28271da177e4SLinus Torvalds } 28281da177e4SLinus Torvalds 28291da177e4SLinus Torvalds /* 28301da177e4SLinus Torvalds * The page straddles i_size. It must be zeroed out on each and every 28312a61aa40SAdam Buchbinder * writepage invocation because it may be mmapped. "A file is mapped 28321da177e4SLinus Torvalds * in multiples of the page size. For a file that is not a multiple of 28331da177e4SLinus Torvalds * the page size, the remaining memory is zeroed when mapped, and 28341da177e4SLinus Torvalds * writes to that region are not written out to the file." 28351da177e4SLinus Torvalds */ 2836eebd2aa3SChristoph Lameter zero_user_segment(page, offset, PAGE_CACHE_SIZE); 283735c80d5fSChris Mason return __block_write_full_page(inode, page, get_block, wbc, handler); 28381da177e4SLinus Torvalds } 28391fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(block_write_full_page_endio); 28401da177e4SLinus Torvalds 284135c80d5fSChris Mason /* 284235c80d5fSChris Mason * The generic ->writepage function for buffer-backed address_spaces 284335c80d5fSChris Mason */ 284435c80d5fSChris Mason int block_write_full_page(struct page *page, get_block_t *get_block, 284535c80d5fSChris Mason struct writeback_control *wbc) 284635c80d5fSChris Mason { 284735c80d5fSChris Mason return block_write_full_page_endio(page, get_block, wbc, 284835c80d5fSChris Mason end_buffer_async_write); 284935c80d5fSChris Mason } 28501fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(block_write_full_page); 285135c80d5fSChris Mason 28521da177e4SLinus Torvalds sector_t generic_block_bmap(struct address_space *mapping, sector_t block, 28531da177e4SLinus Torvalds get_block_t *get_block) 28541da177e4SLinus Torvalds { 28551da177e4SLinus Torvalds struct buffer_head tmp; 28561da177e4SLinus Torvalds struct inode *inode = mapping->host; 28571da177e4SLinus Torvalds tmp.b_state = 0; 28581da177e4SLinus Torvalds tmp.b_blocknr = 0; 2859b0cf2321SBadari Pulavarty tmp.b_size = 1 << inode->i_blkbits; 28601da177e4SLinus Torvalds get_block(inode, block, &tmp, 0); 28611da177e4SLinus Torvalds return tmp.b_blocknr; 28621da177e4SLinus Torvalds } 28631fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(generic_block_bmap); 28641da177e4SLinus Torvalds 28656712ecf8SNeilBrown static void end_bio_bh_io_sync(struct bio *bio, int err) 28661da177e4SLinus Torvalds { 28671da177e4SLinus Torvalds struct buffer_head *bh = bio->bi_private; 28681da177e4SLinus Torvalds 28691da177e4SLinus Torvalds if (err == -EOPNOTSUPP) { 28701da177e4SLinus Torvalds set_bit(BIO_EOPNOTSUPP, &bio->bi_flags); 28711da177e4SLinus Torvalds } 28721da177e4SLinus Torvalds 287308bafc03SKeith Mannthey if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags))) 287408bafc03SKeith Mannthey set_bit(BH_Quiet, &bh->b_state); 287508bafc03SKeith Mannthey 28761da177e4SLinus Torvalds bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags)); 28771da177e4SLinus Torvalds bio_put(bio); 28781da177e4SLinus Torvalds } 28791da177e4SLinus Torvalds 28801da177e4SLinus Torvalds int submit_bh(int rw, struct buffer_head * bh) 28811da177e4SLinus Torvalds { 28821da177e4SLinus Torvalds struct bio *bio; 28831da177e4SLinus Torvalds int ret = 0; 28841da177e4SLinus Torvalds 28851da177e4SLinus Torvalds BUG_ON(!buffer_locked(bh)); 28861da177e4SLinus Torvalds BUG_ON(!buffer_mapped(bh)); 28871da177e4SLinus Torvalds BUG_ON(!bh->b_end_io); 28888fb0e342SAneesh Kumar K.V BUG_ON(buffer_delay(bh)); 28898fb0e342SAneesh Kumar K.V BUG_ON(buffer_unwritten(bh)); 28901da177e4SLinus Torvalds 289148fd4f93SJens Axboe /* 289248fd4f93SJens Axboe * Only clear out a write error when rewriting 28931da177e4SLinus Torvalds */ 289448fd4f93SJens Axboe if (test_set_buffer_req(bh) && (rw & WRITE)) 28951da177e4SLinus Torvalds clear_buffer_write_io_error(bh); 28961da177e4SLinus Torvalds 28971da177e4SLinus Torvalds /* 28981da177e4SLinus Torvalds * from here on down, it's all bio -- do the initial mapping, 28991da177e4SLinus Torvalds * submit_bio -> generic_make_request may further map this bio around 29001da177e4SLinus Torvalds */ 29011da177e4SLinus Torvalds bio = bio_alloc(GFP_NOIO, 1); 29021da177e4SLinus Torvalds 29031da177e4SLinus Torvalds bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 29041da177e4SLinus Torvalds bio->bi_bdev = bh->b_bdev; 29051da177e4SLinus Torvalds bio->bi_io_vec[0].bv_page = bh->b_page; 29061da177e4SLinus Torvalds bio->bi_io_vec[0].bv_len = bh->b_size; 29071da177e4SLinus Torvalds bio->bi_io_vec[0].bv_offset = bh_offset(bh); 29081da177e4SLinus Torvalds 29091da177e4SLinus Torvalds bio->bi_vcnt = 1; 29101da177e4SLinus Torvalds bio->bi_idx = 0; 29111da177e4SLinus Torvalds bio->bi_size = bh->b_size; 29121da177e4SLinus Torvalds 29131da177e4SLinus Torvalds bio->bi_end_io = end_bio_bh_io_sync; 29141da177e4SLinus Torvalds bio->bi_private = bh; 29151da177e4SLinus Torvalds 29161da177e4SLinus Torvalds bio_get(bio); 29171da177e4SLinus Torvalds submit_bio(rw, bio); 29181da177e4SLinus Torvalds 29191da177e4SLinus Torvalds if (bio_flagged(bio, BIO_EOPNOTSUPP)) 29201da177e4SLinus Torvalds ret = -EOPNOTSUPP; 29211da177e4SLinus Torvalds 29221da177e4SLinus Torvalds bio_put(bio); 29231da177e4SLinus Torvalds return ret; 29241da177e4SLinus Torvalds } 29251fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(submit_bh); 29261da177e4SLinus Torvalds 29271da177e4SLinus Torvalds /** 29281da177e4SLinus Torvalds * ll_rw_block: low-level access to block devices (DEPRECATED) 29299cb569d6SChristoph Hellwig * @rw: whether to %READ or %WRITE or maybe %READA (readahead) 29301da177e4SLinus Torvalds * @nr: number of &struct buffer_heads in the array 29311da177e4SLinus Torvalds * @bhs: array of pointers to &struct buffer_head 29321da177e4SLinus Torvalds * 2933a7662236SJan Kara * ll_rw_block() takes an array of pointers to &struct buffer_heads, and 2934a7662236SJan Kara * requests an I/O operation on them, either a %READ or a %WRITE. The third 29359cb569d6SChristoph Hellwig * %READA option is described in the documentation for generic_make_request() 29369cb569d6SChristoph Hellwig * which ll_rw_block() calls. 29371da177e4SLinus Torvalds * 29381da177e4SLinus Torvalds * This function drops any buffer that it cannot get a lock on (with the 29399cb569d6SChristoph Hellwig * BH_Lock state bit), any buffer that appears to be clean when doing a write 29409cb569d6SChristoph Hellwig * request, and any buffer that appears to be up-to-date when doing read 29419cb569d6SChristoph Hellwig * request. Further it marks as clean buffers that are processed for 29429cb569d6SChristoph Hellwig * writing (the buffer cache won't assume that they are actually clean 29439cb569d6SChristoph Hellwig * until the buffer gets unlocked). 29441da177e4SLinus Torvalds * 29451da177e4SLinus Torvalds * ll_rw_block sets b_end_io to simple completion handler that marks 29461da177e4SLinus Torvalds * the buffer up-to-date (if approriate), unlocks the buffer and wakes 29471da177e4SLinus Torvalds * any waiters. 29481da177e4SLinus Torvalds * 29491da177e4SLinus Torvalds * All of the buffers must be for the same device, and must also be a 29501da177e4SLinus Torvalds * multiple of the current approved size for the device. 29511da177e4SLinus Torvalds */ 29521da177e4SLinus Torvalds void ll_rw_block(int rw, int nr, struct buffer_head *bhs[]) 29531da177e4SLinus Torvalds { 29541da177e4SLinus Torvalds int i; 29551da177e4SLinus Torvalds 29561da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 29571da177e4SLinus Torvalds struct buffer_head *bh = bhs[i]; 29581da177e4SLinus Torvalds 29599cb569d6SChristoph Hellwig if (!trylock_buffer(bh)) 29601da177e4SLinus Torvalds continue; 29619cb569d6SChristoph Hellwig if (rw == WRITE) { 29621da177e4SLinus Torvalds if (test_clear_buffer_dirty(bh)) { 296376c3073aSakpm@osdl.org bh->b_end_io = end_buffer_write_sync; 2964e60e5c50SOGAWA Hirofumi get_bh(bh); 29651da177e4SLinus Torvalds submit_bh(WRITE, bh); 29661da177e4SLinus Torvalds continue; 29671da177e4SLinus Torvalds } 29681da177e4SLinus Torvalds } else { 29691da177e4SLinus Torvalds if (!buffer_uptodate(bh)) { 297076c3073aSakpm@osdl.org bh->b_end_io = end_buffer_read_sync; 2971e60e5c50SOGAWA Hirofumi get_bh(bh); 29721da177e4SLinus Torvalds submit_bh(rw, bh); 29731da177e4SLinus Torvalds continue; 29741da177e4SLinus Torvalds } 29751da177e4SLinus Torvalds } 29761da177e4SLinus Torvalds unlock_buffer(bh); 29771da177e4SLinus Torvalds } 29781da177e4SLinus Torvalds } 29791fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(ll_rw_block); 29801da177e4SLinus Torvalds 29819cb569d6SChristoph Hellwig void write_dirty_buffer(struct buffer_head *bh, int rw) 29829cb569d6SChristoph Hellwig { 29839cb569d6SChristoph Hellwig lock_buffer(bh); 29849cb569d6SChristoph Hellwig if (!test_clear_buffer_dirty(bh)) { 29859cb569d6SChristoph Hellwig unlock_buffer(bh); 29869cb569d6SChristoph Hellwig return; 29879cb569d6SChristoph Hellwig } 29889cb569d6SChristoph Hellwig bh->b_end_io = end_buffer_write_sync; 29899cb569d6SChristoph Hellwig get_bh(bh); 29909cb569d6SChristoph Hellwig submit_bh(rw, bh); 29919cb569d6SChristoph Hellwig } 29929cb569d6SChristoph Hellwig EXPORT_SYMBOL(write_dirty_buffer); 29939cb569d6SChristoph Hellwig 29941da177e4SLinus Torvalds /* 29951da177e4SLinus Torvalds * For a data-integrity writeout, we need to wait upon any in-progress I/O 29961da177e4SLinus Torvalds * and then start new I/O and then wait upon it. The caller must have a ref on 29971da177e4SLinus Torvalds * the buffer_head. 29981da177e4SLinus Torvalds */ 299987e99511SChristoph Hellwig int __sync_dirty_buffer(struct buffer_head *bh, int rw) 30001da177e4SLinus Torvalds { 30011da177e4SLinus Torvalds int ret = 0; 30021da177e4SLinus Torvalds 30031da177e4SLinus Torvalds WARN_ON(atomic_read(&bh->b_count) < 1); 30041da177e4SLinus Torvalds lock_buffer(bh); 30051da177e4SLinus Torvalds if (test_clear_buffer_dirty(bh)) { 30061da177e4SLinus Torvalds get_bh(bh); 30071da177e4SLinus Torvalds bh->b_end_io = end_buffer_write_sync; 300887e99511SChristoph Hellwig ret = submit_bh(rw, bh); 30091da177e4SLinus Torvalds wait_on_buffer(bh); 30101da177e4SLinus Torvalds if (!ret && !buffer_uptodate(bh)) 30111da177e4SLinus Torvalds ret = -EIO; 30121da177e4SLinus Torvalds } else { 30131da177e4SLinus Torvalds unlock_buffer(bh); 30141da177e4SLinus Torvalds } 30151da177e4SLinus Torvalds return ret; 30161da177e4SLinus Torvalds } 301787e99511SChristoph Hellwig EXPORT_SYMBOL(__sync_dirty_buffer); 301887e99511SChristoph Hellwig 301987e99511SChristoph Hellwig int sync_dirty_buffer(struct buffer_head *bh) 302087e99511SChristoph Hellwig { 302187e99511SChristoph Hellwig return __sync_dirty_buffer(bh, WRITE_SYNC); 302287e99511SChristoph Hellwig } 30231fe72eaaSH Hartley Sweeten EXPORT_SYMBOL(sync_dirty_buffer); 30241da177e4SLinus Torvalds 30251da177e4SLinus Torvalds /* 30261da177e4SLinus Torvalds * try_to_free_buffers() checks if all the buffers on this particular page 30271da177e4SLinus Torvalds * are unused, and releases them if so. 30281da177e4SLinus Torvalds * 30291da177e4SLinus Torvalds * Exclusion against try_to_free_buffers may be obtained by either 30301da177e4SLinus Torvalds * locking the page or by holding its mapping's private_lock. 30311da177e4SLinus Torvalds * 30321da177e4SLinus Torvalds * If the page is dirty but all the buffers are clean then we need to 30331da177e4SLinus Torvalds * be sure to mark the page clean as well. This is because the page 30341da177e4SLinus Torvalds * may be against a block device, and a later reattachment of buffers 30351da177e4SLinus Torvalds * to a dirty page will set *all* buffers dirty. Which would corrupt 30361da177e4SLinus Torvalds * filesystem data on the same device. 30371da177e4SLinus Torvalds * 30381da177e4SLinus Torvalds * The same applies to regular filesystem pages: if all the buffers are 30391da177e4SLinus Torvalds * clean then we set the page clean and proceed. To do that, we require 30401da177e4SLinus Torvalds * total exclusion from __set_page_dirty_buffers(). That is obtained with 30411da177e4SLinus Torvalds * private_lock. 30421da177e4SLinus Torvalds * 30431da177e4SLinus Torvalds * try_to_free_buffers() is non-blocking. 30441da177e4SLinus Torvalds */ 30451da177e4SLinus Torvalds static inline int buffer_busy(struct buffer_head *bh) 30461da177e4SLinus Torvalds { 30471da177e4SLinus Torvalds return atomic_read(&bh->b_count) | 30481da177e4SLinus Torvalds (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock))); 30491da177e4SLinus Torvalds } 30501da177e4SLinus Torvalds 30511da177e4SLinus Torvalds static int 30521da177e4SLinus Torvalds drop_buffers(struct page *page, struct buffer_head **buffers_to_free) 30531da177e4SLinus Torvalds { 30541da177e4SLinus Torvalds struct buffer_head *head = page_buffers(page); 30551da177e4SLinus Torvalds struct buffer_head *bh; 30561da177e4SLinus Torvalds 30571da177e4SLinus Torvalds bh = head; 30581da177e4SLinus Torvalds do { 3059de7d5a3bSakpm@osdl.org if (buffer_write_io_error(bh) && page->mapping) 30601da177e4SLinus Torvalds set_bit(AS_EIO, &page->mapping->flags); 30611da177e4SLinus Torvalds if (buffer_busy(bh)) 30621da177e4SLinus Torvalds goto failed; 30631da177e4SLinus Torvalds bh = bh->b_this_page; 30641da177e4SLinus Torvalds } while (bh != head); 30651da177e4SLinus Torvalds 30661da177e4SLinus Torvalds do { 30671da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 30681da177e4SLinus Torvalds 3069535ee2fbSJan Kara if (bh->b_assoc_map) 30701da177e4SLinus Torvalds __remove_assoc_queue(bh); 30711da177e4SLinus Torvalds bh = next; 30721da177e4SLinus Torvalds } while (bh != head); 30731da177e4SLinus Torvalds *buffers_to_free = head; 30741da177e4SLinus Torvalds __clear_page_buffers(page); 30751da177e4SLinus Torvalds return 1; 30761da177e4SLinus Torvalds failed: 30771da177e4SLinus Torvalds return 0; 30781da177e4SLinus Torvalds } 30791da177e4SLinus Torvalds 30801da177e4SLinus Torvalds int try_to_free_buffers(struct page *page) 30811da177e4SLinus Torvalds { 30821da177e4SLinus Torvalds struct address_space * const mapping = page->mapping; 30831da177e4SLinus Torvalds struct buffer_head *buffers_to_free = NULL; 30841da177e4SLinus Torvalds int ret = 0; 30851da177e4SLinus Torvalds 30861da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 3087ecdfc978SLinus Torvalds if (PageWriteback(page)) 30881da177e4SLinus Torvalds return 0; 30891da177e4SLinus Torvalds 30901da177e4SLinus Torvalds if (mapping == NULL) { /* can this still happen? */ 30911da177e4SLinus Torvalds ret = drop_buffers(page, &buffers_to_free); 30921da177e4SLinus Torvalds goto out; 30931da177e4SLinus Torvalds } 30941da177e4SLinus Torvalds 30951da177e4SLinus Torvalds spin_lock(&mapping->private_lock); 30961da177e4SLinus Torvalds ret = drop_buffers(page, &buffers_to_free); 3097ecdfc978SLinus Torvalds 3098ecdfc978SLinus Torvalds /* 3099ecdfc978SLinus Torvalds * If the filesystem writes its buffers by hand (eg ext3) 3100ecdfc978SLinus Torvalds * then we can have clean buffers against a dirty page. We 3101ecdfc978SLinus Torvalds * clean the page here; otherwise the VM will never notice 3102ecdfc978SLinus Torvalds * that the filesystem did any IO at all. 3103ecdfc978SLinus Torvalds * 3104ecdfc978SLinus Torvalds * Also, during truncate, discard_buffer will have marked all 3105ecdfc978SLinus Torvalds * the page's buffers clean. We discover that here and clean 3106ecdfc978SLinus Torvalds * the page also. 310787df7241SNick Piggin * 310887df7241SNick Piggin * private_lock must be held over this entire operation in order 310987df7241SNick Piggin * to synchronise against __set_page_dirty_buffers and prevent the 311087df7241SNick Piggin * dirty bit from being lost. 3111ecdfc978SLinus Torvalds */ 3112ecdfc978SLinus Torvalds if (ret) 3113ecdfc978SLinus Torvalds cancel_dirty_page(page, PAGE_CACHE_SIZE); 311487df7241SNick Piggin spin_unlock(&mapping->private_lock); 31151da177e4SLinus Torvalds out: 31161da177e4SLinus Torvalds if (buffers_to_free) { 31171da177e4SLinus Torvalds struct buffer_head *bh = buffers_to_free; 31181da177e4SLinus Torvalds 31191da177e4SLinus Torvalds do { 31201da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 31211da177e4SLinus Torvalds free_buffer_head(bh); 31221da177e4SLinus Torvalds bh = next; 31231da177e4SLinus Torvalds } while (bh != buffers_to_free); 31241da177e4SLinus Torvalds } 31251da177e4SLinus Torvalds return ret; 31261da177e4SLinus Torvalds } 31271da177e4SLinus Torvalds EXPORT_SYMBOL(try_to_free_buffers); 31281da177e4SLinus Torvalds 31291da177e4SLinus Torvalds /* 31301da177e4SLinus Torvalds * There are no bdflush tunables left. But distributions are 31311da177e4SLinus Torvalds * still running obsolete flush daemons, so we terminate them here. 31321da177e4SLinus Torvalds * 31331da177e4SLinus Torvalds * Use of bdflush() is deprecated and will be removed in a future kernel. 31345b0830cbSJens Axboe * The `flush-X' kernel threads fully replace bdflush daemons and this call. 31351da177e4SLinus Torvalds */ 3136bdc480e3SHeiko Carstens SYSCALL_DEFINE2(bdflush, int, func, long, data) 31371da177e4SLinus Torvalds { 31381da177e4SLinus Torvalds static int msg_count; 31391da177e4SLinus Torvalds 31401da177e4SLinus Torvalds if (!capable(CAP_SYS_ADMIN)) 31411da177e4SLinus Torvalds return -EPERM; 31421da177e4SLinus Torvalds 31431da177e4SLinus Torvalds if (msg_count < 5) { 31441da177e4SLinus Torvalds msg_count++; 31451da177e4SLinus Torvalds printk(KERN_INFO 31461da177e4SLinus Torvalds "warning: process `%s' used the obsolete bdflush" 31471da177e4SLinus Torvalds " system call\n", current->comm); 31481da177e4SLinus Torvalds printk(KERN_INFO "Fix your initscripts?\n"); 31491da177e4SLinus Torvalds } 31501da177e4SLinus Torvalds 31511da177e4SLinus Torvalds if (func == 1) 31521da177e4SLinus Torvalds do_exit(0); 31531da177e4SLinus Torvalds return 0; 31541da177e4SLinus Torvalds } 31551da177e4SLinus Torvalds 31561da177e4SLinus Torvalds /* 31571da177e4SLinus Torvalds * Buffer-head allocation 31581da177e4SLinus Torvalds */ 3159a0a9b043SShai Fultheim static struct kmem_cache *bh_cachep __read_mostly; 31601da177e4SLinus Torvalds 31611da177e4SLinus Torvalds /* 31621da177e4SLinus Torvalds * Once the number of bh's in the machine exceeds this level, we start 31631da177e4SLinus Torvalds * stripping them in writeback. 31641da177e4SLinus Torvalds */ 31651da177e4SLinus Torvalds static int max_buffer_heads; 31661da177e4SLinus Torvalds 31671da177e4SLinus Torvalds int buffer_heads_over_limit; 31681da177e4SLinus Torvalds 31691da177e4SLinus Torvalds struct bh_accounting { 31701da177e4SLinus Torvalds int nr; /* Number of live bh's */ 31711da177e4SLinus Torvalds int ratelimit; /* Limit cacheline bouncing */ 31721da177e4SLinus Torvalds }; 31731da177e4SLinus Torvalds 31741da177e4SLinus Torvalds static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0}; 31751da177e4SLinus Torvalds 31761da177e4SLinus Torvalds static void recalc_bh_state(void) 31771da177e4SLinus Torvalds { 31781da177e4SLinus Torvalds int i; 31791da177e4SLinus Torvalds int tot = 0; 31801da177e4SLinus Torvalds 3181ee1be862SChristoph Lameter if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096) 31821da177e4SLinus Torvalds return; 3183c7b92516SChristoph Lameter __this_cpu_write(bh_accounting.ratelimit, 0); 31848a143426SEric Dumazet for_each_online_cpu(i) 31851da177e4SLinus Torvalds tot += per_cpu(bh_accounting, i).nr; 31861da177e4SLinus Torvalds buffer_heads_over_limit = (tot > max_buffer_heads); 31871da177e4SLinus Torvalds } 31881da177e4SLinus Torvalds 3189dd0fc66fSAl Viro struct buffer_head *alloc_buffer_head(gfp_t gfp_flags) 31901da177e4SLinus Torvalds { 3191019b4d12SRichard Kennedy struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); 31921da177e4SLinus Torvalds if (ret) { 3193a35afb83SChristoph Lameter INIT_LIST_HEAD(&ret->b_assoc_buffers); 3194c7b92516SChristoph Lameter preempt_disable(); 3195c7b92516SChristoph Lameter __this_cpu_inc(bh_accounting.nr); 31961da177e4SLinus Torvalds recalc_bh_state(); 3197c7b92516SChristoph Lameter preempt_enable(); 31981da177e4SLinus Torvalds } 31991da177e4SLinus Torvalds return ret; 32001da177e4SLinus Torvalds } 32011da177e4SLinus Torvalds EXPORT_SYMBOL(alloc_buffer_head); 32021da177e4SLinus Torvalds 32031da177e4SLinus Torvalds void free_buffer_head(struct buffer_head *bh) 32041da177e4SLinus Torvalds { 32051da177e4SLinus Torvalds BUG_ON(!list_empty(&bh->b_assoc_buffers)); 32061da177e4SLinus Torvalds kmem_cache_free(bh_cachep, bh); 3207c7b92516SChristoph Lameter preempt_disable(); 3208c7b92516SChristoph Lameter __this_cpu_dec(bh_accounting.nr); 32091da177e4SLinus Torvalds recalc_bh_state(); 3210c7b92516SChristoph Lameter preempt_enable(); 32111da177e4SLinus Torvalds } 32121da177e4SLinus Torvalds EXPORT_SYMBOL(free_buffer_head); 32131da177e4SLinus Torvalds 32141da177e4SLinus Torvalds static void buffer_exit_cpu(int cpu) 32151da177e4SLinus Torvalds { 32161da177e4SLinus Torvalds int i; 32171da177e4SLinus Torvalds struct bh_lru *b = &per_cpu(bh_lrus, cpu); 32181da177e4SLinus Torvalds 32191da177e4SLinus Torvalds for (i = 0; i < BH_LRU_SIZE; i++) { 32201da177e4SLinus Torvalds brelse(b->bhs[i]); 32211da177e4SLinus Torvalds b->bhs[i] = NULL; 32221da177e4SLinus Torvalds } 3223c7b92516SChristoph Lameter this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr); 32248a143426SEric Dumazet per_cpu(bh_accounting, cpu).nr = 0; 32251da177e4SLinus Torvalds } 32261da177e4SLinus Torvalds 32271da177e4SLinus Torvalds static int buffer_cpu_notify(struct notifier_block *self, 32281da177e4SLinus Torvalds unsigned long action, void *hcpu) 32291da177e4SLinus Torvalds { 32308bb78442SRafael J. Wysocki if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) 32311da177e4SLinus Torvalds buffer_exit_cpu((unsigned long)hcpu); 32321da177e4SLinus Torvalds return NOTIFY_OK; 32331da177e4SLinus Torvalds } 32341da177e4SLinus Torvalds 3235389d1b08SAneesh Kumar K.V /** 3236a6b91919SRandy Dunlap * bh_uptodate_or_lock - Test whether the buffer is uptodate 3237389d1b08SAneesh Kumar K.V * @bh: struct buffer_head 3238389d1b08SAneesh Kumar K.V * 3239389d1b08SAneesh Kumar K.V * Return true if the buffer is up-to-date and false, 3240389d1b08SAneesh Kumar K.V * with the buffer locked, if not. 3241389d1b08SAneesh Kumar K.V */ 3242389d1b08SAneesh Kumar K.V int bh_uptodate_or_lock(struct buffer_head *bh) 3243389d1b08SAneesh Kumar K.V { 3244389d1b08SAneesh Kumar K.V if (!buffer_uptodate(bh)) { 3245389d1b08SAneesh Kumar K.V lock_buffer(bh); 3246389d1b08SAneesh Kumar K.V if (!buffer_uptodate(bh)) 3247389d1b08SAneesh Kumar K.V return 0; 3248389d1b08SAneesh Kumar K.V unlock_buffer(bh); 3249389d1b08SAneesh Kumar K.V } 3250389d1b08SAneesh Kumar K.V return 1; 3251389d1b08SAneesh Kumar K.V } 3252389d1b08SAneesh Kumar K.V EXPORT_SYMBOL(bh_uptodate_or_lock); 3253389d1b08SAneesh Kumar K.V 3254389d1b08SAneesh Kumar K.V /** 3255a6b91919SRandy Dunlap * bh_submit_read - Submit a locked buffer for reading 3256389d1b08SAneesh Kumar K.V * @bh: struct buffer_head 3257389d1b08SAneesh Kumar K.V * 3258389d1b08SAneesh Kumar K.V * Returns zero on success and -EIO on error. 3259389d1b08SAneesh Kumar K.V */ 3260389d1b08SAneesh Kumar K.V int bh_submit_read(struct buffer_head *bh) 3261389d1b08SAneesh Kumar K.V { 3262389d1b08SAneesh Kumar K.V BUG_ON(!buffer_locked(bh)); 3263389d1b08SAneesh Kumar K.V 3264389d1b08SAneesh Kumar K.V if (buffer_uptodate(bh)) { 3265389d1b08SAneesh Kumar K.V unlock_buffer(bh); 3266389d1b08SAneesh Kumar K.V return 0; 3267389d1b08SAneesh Kumar K.V } 3268389d1b08SAneesh Kumar K.V 3269389d1b08SAneesh Kumar K.V get_bh(bh); 3270389d1b08SAneesh Kumar K.V bh->b_end_io = end_buffer_read_sync; 3271389d1b08SAneesh Kumar K.V submit_bh(READ, bh); 3272389d1b08SAneesh Kumar K.V wait_on_buffer(bh); 3273389d1b08SAneesh Kumar K.V if (buffer_uptodate(bh)) 3274389d1b08SAneesh Kumar K.V return 0; 3275389d1b08SAneesh Kumar K.V return -EIO; 3276389d1b08SAneesh Kumar K.V } 3277389d1b08SAneesh Kumar K.V EXPORT_SYMBOL(bh_submit_read); 3278389d1b08SAneesh Kumar K.V 32791da177e4SLinus Torvalds void __init buffer_init(void) 32801da177e4SLinus Torvalds { 32811da177e4SLinus Torvalds int nrpages; 32821da177e4SLinus Torvalds 3283b98938c3SChristoph Lameter bh_cachep = kmem_cache_create("buffer_head", 3284b98938c3SChristoph Lameter sizeof(struct buffer_head), 0, 3285b98938c3SChristoph Lameter (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| 3286b98938c3SChristoph Lameter SLAB_MEM_SPREAD), 3287019b4d12SRichard Kennedy NULL); 32881da177e4SLinus Torvalds 32891da177e4SLinus Torvalds /* 32901da177e4SLinus Torvalds * Limit the bh occupancy to 10% of ZONE_NORMAL 32911da177e4SLinus Torvalds */ 32921da177e4SLinus Torvalds nrpages = (nr_free_buffer_pages() * 10) / 100; 32931da177e4SLinus Torvalds max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head)); 32941da177e4SLinus Torvalds hotcpu_notifier(buffer_cpu_notify, 0); 32951da177e4SLinus Torvalds } 3296