11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/fs/buffer.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1991, 1992, 2002 Linus Torvalds 51da177e4SLinus Torvalds */ 61da177e4SLinus Torvalds 71da177e4SLinus Torvalds /* 81da177e4SLinus Torvalds * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * Removed a lot of unnecessary code and simplified things now that 111da177e4SLinus Torvalds * the buffer cache isn't our primary cache - Andrew Tridgell 12/96 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * Speed up hash, lru, and free list operations. Use gfp() for allocating 141da177e4SLinus Torvalds * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * Added 32k buffer block sizes - these are required older ARM systems. - RMK 171da177e4SLinus Torvalds * 181da177e4SLinus Torvalds * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de> 191da177e4SLinus Torvalds */ 201da177e4SLinus Torvalds 211da177e4SLinus Torvalds #include <linux/kernel.h> 221da177e4SLinus Torvalds #include <linux/syscalls.h> 231da177e4SLinus Torvalds #include <linux/fs.h> 241da177e4SLinus Torvalds #include <linux/mm.h> 251da177e4SLinus Torvalds #include <linux/percpu.h> 261da177e4SLinus Torvalds #include <linux/slab.h> 271da177e4SLinus Torvalds #include <linux/smp_lock.h> 2816f7e0feSRandy Dunlap #include <linux/capability.h> 291da177e4SLinus Torvalds #include <linux/blkdev.h> 301da177e4SLinus Torvalds #include <linux/file.h> 311da177e4SLinus Torvalds #include <linux/quotaops.h> 321da177e4SLinus Torvalds #include <linux/highmem.h> 331da177e4SLinus Torvalds #include <linux/module.h> 341da177e4SLinus Torvalds #include <linux/writeback.h> 351da177e4SLinus Torvalds #include <linux/hash.h> 361da177e4SLinus Torvalds #include <linux/suspend.h> 371da177e4SLinus Torvalds #include <linux/buffer_head.h> 381da177e4SLinus Torvalds #include <linux/bio.h> 391da177e4SLinus Torvalds #include <linux/notifier.h> 401da177e4SLinus Torvalds #include <linux/cpu.h> 411da177e4SLinus Torvalds #include <linux/bitops.h> 421da177e4SLinus Torvalds #include <linux/mpage.h> 43fb1c8f93SIngo Molnar #include <linux/bit_spinlock.h> 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); 461da177e4SLinus Torvalds static void invalidate_bh_lrus(void); 471da177e4SLinus Torvalds 481da177e4SLinus Torvalds #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) 491da177e4SLinus Torvalds 501da177e4SLinus Torvalds inline void 511da177e4SLinus Torvalds init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private) 521da177e4SLinus Torvalds { 531da177e4SLinus Torvalds bh->b_end_io = handler; 541da177e4SLinus Torvalds bh->b_private = private; 551da177e4SLinus Torvalds } 561da177e4SLinus Torvalds 571da177e4SLinus Torvalds static int sync_buffer(void *word) 581da177e4SLinus Torvalds { 591da177e4SLinus Torvalds struct block_device *bd; 601da177e4SLinus Torvalds struct buffer_head *bh 611da177e4SLinus Torvalds = container_of(word, struct buffer_head, b_state); 621da177e4SLinus Torvalds 631da177e4SLinus Torvalds smp_mb(); 641da177e4SLinus Torvalds bd = bh->b_bdev; 651da177e4SLinus Torvalds if (bd) 661da177e4SLinus Torvalds blk_run_address_space(bd->bd_inode->i_mapping); 671da177e4SLinus Torvalds io_schedule(); 681da177e4SLinus Torvalds return 0; 691da177e4SLinus Torvalds } 701da177e4SLinus Torvalds 711da177e4SLinus Torvalds void fastcall __lock_buffer(struct buffer_head *bh) 721da177e4SLinus Torvalds { 731da177e4SLinus Torvalds wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer, 741da177e4SLinus Torvalds TASK_UNINTERRUPTIBLE); 751da177e4SLinus Torvalds } 761da177e4SLinus Torvalds EXPORT_SYMBOL(__lock_buffer); 771da177e4SLinus Torvalds 781da177e4SLinus Torvalds void fastcall unlock_buffer(struct buffer_head *bh) 791da177e4SLinus Torvalds { 801da177e4SLinus Torvalds clear_buffer_locked(bh); 811da177e4SLinus Torvalds smp_mb__after_clear_bit(); 821da177e4SLinus Torvalds wake_up_bit(&bh->b_state, BH_Lock); 831da177e4SLinus Torvalds } 841da177e4SLinus Torvalds 851da177e4SLinus Torvalds /* 861da177e4SLinus Torvalds * Block until a buffer comes unlocked. This doesn't stop it 871da177e4SLinus Torvalds * from becoming locked again - you have to lock it yourself 881da177e4SLinus Torvalds * if you want to preserve its state. 891da177e4SLinus Torvalds */ 901da177e4SLinus Torvalds void __wait_on_buffer(struct buffer_head * bh) 911da177e4SLinus Torvalds { 921da177e4SLinus Torvalds wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE); 931da177e4SLinus Torvalds } 941da177e4SLinus Torvalds 951da177e4SLinus Torvalds static void 961da177e4SLinus Torvalds __clear_page_buffers(struct page *page) 971da177e4SLinus Torvalds { 981da177e4SLinus Torvalds ClearPagePrivate(page); 994c21e2f2SHugh Dickins set_page_private(page, 0); 1001da177e4SLinus Torvalds page_cache_release(page); 1011da177e4SLinus Torvalds } 1021da177e4SLinus Torvalds 1031da177e4SLinus Torvalds static void buffer_io_error(struct buffer_head *bh) 1041da177e4SLinus Torvalds { 1051da177e4SLinus Torvalds char b[BDEVNAME_SIZE]; 1061da177e4SLinus Torvalds 1071da177e4SLinus Torvalds printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n", 1081da177e4SLinus Torvalds bdevname(bh->b_bdev, b), 1091da177e4SLinus Torvalds (unsigned long long)bh->b_blocknr); 1101da177e4SLinus Torvalds } 1111da177e4SLinus Torvalds 1121da177e4SLinus Torvalds /* 1131da177e4SLinus Torvalds * Default synchronous end-of-IO handler.. Just mark it up-to-date and 1141da177e4SLinus Torvalds * unlock the buffer. This is what ll_rw_block uses too. 1151da177e4SLinus Torvalds */ 1161da177e4SLinus Torvalds void end_buffer_read_sync(struct buffer_head *bh, int uptodate) 1171da177e4SLinus Torvalds { 1181da177e4SLinus Torvalds if (uptodate) { 1191da177e4SLinus Torvalds set_buffer_uptodate(bh); 1201da177e4SLinus Torvalds } else { 1211da177e4SLinus Torvalds /* This happens, due to failed READA attempts. */ 1221da177e4SLinus Torvalds clear_buffer_uptodate(bh); 1231da177e4SLinus Torvalds } 1241da177e4SLinus Torvalds unlock_buffer(bh); 1251da177e4SLinus Torvalds put_bh(bh); 1261da177e4SLinus Torvalds } 1271da177e4SLinus Torvalds 1281da177e4SLinus Torvalds void end_buffer_write_sync(struct buffer_head *bh, int uptodate) 1291da177e4SLinus Torvalds { 1301da177e4SLinus Torvalds char b[BDEVNAME_SIZE]; 1311da177e4SLinus Torvalds 1321da177e4SLinus Torvalds if (uptodate) { 1331da177e4SLinus Torvalds set_buffer_uptodate(bh); 1341da177e4SLinus Torvalds } else { 1351da177e4SLinus Torvalds if (!buffer_eopnotsupp(bh) && printk_ratelimit()) { 1361da177e4SLinus Torvalds buffer_io_error(bh); 1371da177e4SLinus Torvalds printk(KERN_WARNING "lost page write due to " 1381da177e4SLinus Torvalds "I/O error on %s\n", 1391da177e4SLinus Torvalds bdevname(bh->b_bdev, b)); 1401da177e4SLinus Torvalds } 1411da177e4SLinus Torvalds set_buffer_write_io_error(bh); 1421da177e4SLinus Torvalds clear_buffer_uptodate(bh); 1431da177e4SLinus Torvalds } 1441da177e4SLinus Torvalds unlock_buffer(bh); 1451da177e4SLinus Torvalds put_bh(bh); 1461da177e4SLinus Torvalds } 1471da177e4SLinus Torvalds 1481da177e4SLinus Torvalds /* 1491da177e4SLinus Torvalds * Write out and wait upon all the dirty data associated with a block 1501da177e4SLinus Torvalds * device via its mapping. Does not take the superblock lock. 1511da177e4SLinus Torvalds */ 1521da177e4SLinus Torvalds int sync_blockdev(struct block_device *bdev) 1531da177e4SLinus Torvalds { 1541da177e4SLinus Torvalds int ret = 0; 1551da177e4SLinus Torvalds 15628fd1298SOGAWA Hirofumi if (bdev) 15728fd1298SOGAWA Hirofumi ret = filemap_write_and_wait(bdev->bd_inode->i_mapping); 1581da177e4SLinus Torvalds return ret; 1591da177e4SLinus Torvalds } 1601da177e4SLinus Torvalds EXPORT_SYMBOL(sync_blockdev); 1611da177e4SLinus Torvalds 162d25b9a1fSOGAWA Hirofumi static void __fsync_super(struct super_block *sb) 1631da177e4SLinus Torvalds { 1641da177e4SLinus Torvalds sync_inodes_sb(sb, 0); 1651da177e4SLinus Torvalds DQUOT_SYNC(sb); 1661da177e4SLinus Torvalds lock_super(sb); 1671da177e4SLinus Torvalds if (sb->s_dirt && sb->s_op->write_super) 1681da177e4SLinus Torvalds sb->s_op->write_super(sb); 1691da177e4SLinus Torvalds unlock_super(sb); 1701da177e4SLinus Torvalds if (sb->s_op->sync_fs) 1711da177e4SLinus Torvalds sb->s_op->sync_fs(sb, 1); 1721da177e4SLinus Torvalds sync_blockdev(sb->s_bdev); 1731da177e4SLinus Torvalds sync_inodes_sb(sb, 1); 174d25b9a1fSOGAWA Hirofumi } 1751da177e4SLinus Torvalds 176d25b9a1fSOGAWA Hirofumi /* 177d25b9a1fSOGAWA Hirofumi * Write out and wait upon all dirty data associated with this 178d25b9a1fSOGAWA Hirofumi * superblock. Filesystem data as well as the underlying block 179d25b9a1fSOGAWA Hirofumi * device. Takes the superblock lock. 180d25b9a1fSOGAWA Hirofumi */ 181d25b9a1fSOGAWA Hirofumi int fsync_super(struct super_block *sb) 182d25b9a1fSOGAWA Hirofumi { 183d25b9a1fSOGAWA Hirofumi __fsync_super(sb); 1841da177e4SLinus Torvalds return sync_blockdev(sb->s_bdev); 1851da177e4SLinus Torvalds } 1861da177e4SLinus Torvalds 1871da177e4SLinus Torvalds /* 1881da177e4SLinus Torvalds * Write out and wait upon all dirty data associated with this 1891da177e4SLinus Torvalds * device. Filesystem data as well as the underlying block 1901da177e4SLinus Torvalds * device. Takes the superblock lock. 1911da177e4SLinus Torvalds */ 1921da177e4SLinus Torvalds int fsync_bdev(struct block_device *bdev) 1931da177e4SLinus Torvalds { 1941da177e4SLinus Torvalds struct super_block *sb = get_super(bdev); 1951da177e4SLinus Torvalds if (sb) { 1961da177e4SLinus Torvalds int res = fsync_super(sb); 1971da177e4SLinus Torvalds drop_super(sb); 1981da177e4SLinus Torvalds return res; 1991da177e4SLinus Torvalds } 2001da177e4SLinus Torvalds return sync_blockdev(bdev); 2011da177e4SLinus Torvalds } 2021da177e4SLinus Torvalds 2031da177e4SLinus Torvalds /** 2041da177e4SLinus Torvalds * freeze_bdev -- lock a filesystem and force it into a consistent state 2051da177e4SLinus Torvalds * @bdev: blockdevice to lock 2061da177e4SLinus Torvalds * 207c039e313SArjan van de Ven * This takes the block device bd_mount_mutex to make sure no new mounts 2081da177e4SLinus Torvalds * happen on bdev until thaw_bdev() is called. 2091da177e4SLinus Torvalds * If a superblock is found on this device, we take the s_umount semaphore 2101da177e4SLinus Torvalds * on it to make sure nobody unmounts until the snapshot creation is done. 2111da177e4SLinus Torvalds */ 2121da177e4SLinus Torvalds struct super_block *freeze_bdev(struct block_device *bdev) 2131da177e4SLinus Torvalds { 2141da177e4SLinus Torvalds struct super_block *sb; 2151da177e4SLinus Torvalds 216c039e313SArjan van de Ven mutex_lock(&bdev->bd_mount_mutex); 2171da177e4SLinus Torvalds sb = get_super(bdev); 2181da177e4SLinus Torvalds if (sb && !(sb->s_flags & MS_RDONLY)) { 2191da177e4SLinus Torvalds sb->s_frozen = SB_FREEZE_WRITE; 220d59dd462Sakpm@osdl.org smp_wmb(); 2211da177e4SLinus Torvalds 222d25b9a1fSOGAWA Hirofumi __fsync_super(sb); 2231da177e4SLinus Torvalds 2241da177e4SLinus Torvalds sb->s_frozen = SB_FREEZE_TRANS; 225d59dd462Sakpm@osdl.org smp_wmb(); 2261da177e4SLinus Torvalds 2271da177e4SLinus Torvalds sync_blockdev(sb->s_bdev); 2281da177e4SLinus Torvalds 2291da177e4SLinus Torvalds if (sb->s_op->write_super_lockfs) 2301da177e4SLinus Torvalds sb->s_op->write_super_lockfs(sb); 2311da177e4SLinus Torvalds } 2321da177e4SLinus Torvalds 2331da177e4SLinus Torvalds sync_blockdev(bdev); 2341da177e4SLinus Torvalds return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */ 2351da177e4SLinus Torvalds } 2361da177e4SLinus Torvalds EXPORT_SYMBOL(freeze_bdev); 2371da177e4SLinus Torvalds 2381da177e4SLinus Torvalds /** 2391da177e4SLinus Torvalds * thaw_bdev -- unlock filesystem 2401da177e4SLinus Torvalds * @bdev: blockdevice to unlock 2411da177e4SLinus Torvalds * @sb: associated superblock 2421da177e4SLinus Torvalds * 2431da177e4SLinus Torvalds * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 2441da177e4SLinus Torvalds */ 2451da177e4SLinus Torvalds void thaw_bdev(struct block_device *bdev, struct super_block *sb) 2461da177e4SLinus Torvalds { 2471da177e4SLinus Torvalds if (sb) { 2481da177e4SLinus Torvalds BUG_ON(sb->s_bdev != bdev); 2491da177e4SLinus Torvalds 2501da177e4SLinus Torvalds if (sb->s_op->unlockfs) 2511da177e4SLinus Torvalds sb->s_op->unlockfs(sb); 2521da177e4SLinus Torvalds sb->s_frozen = SB_UNFROZEN; 253d59dd462Sakpm@osdl.org smp_wmb(); 2541da177e4SLinus Torvalds wake_up(&sb->s_wait_unfrozen); 2551da177e4SLinus Torvalds drop_super(sb); 2561da177e4SLinus Torvalds } 2571da177e4SLinus Torvalds 258c039e313SArjan van de Ven mutex_unlock(&bdev->bd_mount_mutex); 2591da177e4SLinus Torvalds } 2601da177e4SLinus Torvalds EXPORT_SYMBOL(thaw_bdev); 2611da177e4SLinus Torvalds 2621da177e4SLinus Torvalds /* 2631da177e4SLinus Torvalds * sync everything. Start out by waking pdflush, because that writes back 2641da177e4SLinus Torvalds * all queues in parallel. 2651da177e4SLinus Torvalds */ 2661da177e4SLinus Torvalds static void do_sync(unsigned long wait) 2671da177e4SLinus Torvalds { 268687a21ceSPekka J Enberg wakeup_pdflush(0); 2691da177e4SLinus Torvalds sync_inodes(0); /* All mappings, inodes and their blockdevs */ 2701da177e4SLinus Torvalds DQUOT_SYNC(NULL); 2711da177e4SLinus Torvalds sync_supers(); /* Write the superblocks */ 2721da177e4SLinus Torvalds sync_filesystems(0); /* Start syncing the filesystems */ 2731da177e4SLinus Torvalds sync_filesystems(wait); /* Waitingly sync the filesystems */ 2741da177e4SLinus Torvalds sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */ 2751da177e4SLinus Torvalds if (!wait) 2761da177e4SLinus Torvalds printk("Emergency Sync complete\n"); 2771da177e4SLinus Torvalds if (unlikely(laptop_mode)) 2781da177e4SLinus Torvalds laptop_sync_completion(); 2791da177e4SLinus Torvalds } 2801da177e4SLinus Torvalds 2811da177e4SLinus Torvalds asmlinkage long sys_sync(void) 2821da177e4SLinus Torvalds { 2831da177e4SLinus Torvalds do_sync(1); 2841da177e4SLinus Torvalds return 0; 2851da177e4SLinus Torvalds } 2861da177e4SLinus Torvalds 2871da177e4SLinus Torvalds void emergency_sync(void) 2881da177e4SLinus Torvalds { 2891da177e4SLinus Torvalds pdflush_operation(do_sync, 0); 2901da177e4SLinus Torvalds } 2911da177e4SLinus Torvalds 2921da177e4SLinus Torvalds /* 2931da177e4SLinus Torvalds * Generic function to fsync a file. 2941da177e4SLinus Torvalds * 2951da177e4SLinus Torvalds * filp may be NULL if called via the msync of a vma. 2961da177e4SLinus Torvalds */ 2971da177e4SLinus Torvalds 2981da177e4SLinus Torvalds int file_fsync(struct file *filp, struct dentry *dentry, int datasync) 2991da177e4SLinus Torvalds { 3001da177e4SLinus Torvalds struct inode * inode = dentry->d_inode; 3011da177e4SLinus Torvalds struct super_block * sb; 3021da177e4SLinus Torvalds int ret, err; 3031da177e4SLinus Torvalds 3041da177e4SLinus Torvalds /* sync the inode to buffers */ 3051da177e4SLinus Torvalds ret = write_inode_now(inode, 0); 3061da177e4SLinus Torvalds 3071da177e4SLinus Torvalds /* sync the superblock to buffers */ 3081da177e4SLinus Torvalds sb = inode->i_sb; 3091da177e4SLinus Torvalds lock_super(sb); 3101da177e4SLinus Torvalds if (sb->s_op->write_super) 3111da177e4SLinus Torvalds sb->s_op->write_super(sb); 3121da177e4SLinus Torvalds unlock_super(sb); 3131da177e4SLinus Torvalds 3141da177e4SLinus Torvalds /* .. finally sync the buffers to disk */ 3151da177e4SLinus Torvalds err = sync_blockdev(sb->s_bdev); 3161da177e4SLinus Torvalds if (!ret) 3171da177e4SLinus Torvalds ret = err; 3181da177e4SLinus Torvalds return ret; 3191da177e4SLinus Torvalds } 3201da177e4SLinus Torvalds 32118e79b40SAndrew Morton long do_fsync(struct file *file, int datasync) 3221da177e4SLinus Torvalds { 32318e79b40SAndrew Morton int ret; 32418e79b40SAndrew Morton int err; 32518e79b40SAndrew Morton struct address_space *mapping = file->f_mapping; 3261da177e4SLinus Torvalds 3271da177e4SLinus Torvalds if (!file->f_op || !file->f_op->fsync) { 3281da177e4SLinus Torvalds /* Why? We can still call filemap_fdatawrite */ 32918e79b40SAndrew Morton ret = -EINVAL; 33018e79b40SAndrew Morton goto out; 3311da177e4SLinus Torvalds } 3321da177e4SLinus Torvalds 3331da177e4SLinus Torvalds ret = filemap_fdatawrite(mapping); 3341da177e4SLinus Torvalds 3351da177e4SLinus Torvalds /* 33618e79b40SAndrew Morton * We need to protect against concurrent writers, which could cause 33718e79b40SAndrew Morton * livelocks in fsync_buffers_list(). 3381da177e4SLinus Torvalds */ 3391b1dcc1bSJes Sorensen mutex_lock(&mapping->host->i_mutex); 340dfb388bfSOleg Nesterov err = file->f_op->fsync(file, file->f_dentry, datasync); 3411da177e4SLinus Torvalds if (!ret) 3421da177e4SLinus Torvalds ret = err; 3431b1dcc1bSJes Sorensen mutex_unlock(&mapping->host->i_mutex); 3441da177e4SLinus Torvalds err = filemap_fdatawait(mapping); 3451da177e4SLinus Torvalds if (!ret) 3461da177e4SLinus Torvalds ret = err; 3471da177e4SLinus Torvalds out: 3481da177e4SLinus Torvalds return ret; 3491da177e4SLinus Torvalds } 3501da177e4SLinus Torvalds 35118e79b40SAndrew Morton static long __do_fsync(unsigned int fd, int datasync) 35218e79b40SAndrew Morton { 35318e79b40SAndrew Morton struct file *file; 35418e79b40SAndrew Morton int ret = -EBADF; 35518e79b40SAndrew Morton 35618e79b40SAndrew Morton file = fget(fd); 35718e79b40SAndrew Morton if (file) { 35818e79b40SAndrew Morton ret = do_fsync(file, datasync); 35918e79b40SAndrew Morton fput(file); 36018e79b40SAndrew Morton } 36118e79b40SAndrew Morton return ret; 36218e79b40SAndrew Morton } 36318e79b40SAndrew Morton 364dfb388bfSOleg Nesterov asmlinkage long sys_fsync(unsigned int fd) 365dfb388bfSOleg Nesterov { 36618e79b40SAndrew Morton return __do_fsync(fd, 0); 367dfb388bfSOleg Nesterov } 368dfb388bfSOleg Nesterov 3691da177e4SLinus Torvalds asmlinkage long sys_fdatasync(unsigned int fd) 3701da177e4SLinus Torvalds { 37118e79b40SAndrew Morton return __do_fsync(fd, 1); 3721da177e4SLinus Torvalds } 3731da177e4SLinus Torvalds 3741da177e4SLinus Torvalds /* 3751da177e4SLinus Torvalds * Various filesystems appear to want __find_get_block to be non-blocking. 3761da177e4SLinus Torvalds * But it's the page lock which protects the buffers. To get around this, 3771da177e4SLinus Torvalds * we get exclusion from try_to_free_buffers with the blockdev mapping's 3781da177e4SLinus Torvalds * private_lock. 3791da177e4SLinus Torvalds * 3801da177e4SLinus Torvalds * Hack idea: for the blockdev mapping, i_bufferlist_lock contention 3811da177e4SLinus Torvalds * may be quite high. This code could TryLock the page, and if that 3821da177e4SLinus Torvalds * succeeds, there is no need to take private_lock. (But if 3831da177e4SLinus Torvalds * private_lock is contended then so is mapping->tree_lock). 3841da177e4SLinus Torvalds */ 3851da177e4SLinus Torvalds static struct buffer_head * 386385fd4c5SCoywolf Qi Hunt __find_get_block_slow(struct block_device *bdev, sector_t block) 3871da177e4SLinus Torvalds { 3881da177e4SLinus Torvalds struct inode *bd_inode = bdev->bd_inode; 3891da177e4SLinus Torvalds struct address_space *bd_mapping = bd_inode->i_mapping; 3901da177e4SLinus Torvalds struct buffer_head *ret = NULL; 3911da177e4SLinus Torvalds pgoff_t index; 3921da177e4SLinus Torvalds struct buffer_head *bh; 3931da177e4SLinus Torvalds struct buffer_head *head; 3941da177e4SLinus Torvalds struct page *page; 3951da177e4SLinus Torvalds int all_mapped = 1; 3961da177e4SLinus Torvalds 3971da177e4SLinus Torvalds index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits); 3981da177e4SLinus Torvalds page = find_get_page(bd_mapping, index); 3991da177e4SLinus Torvalds if (!page) 4001da177e4SLinus Torvalds goto out; 4011da177e4SLinus Torvalds 4021da177e4SLinus Torvalds spin_lock(&bd_mapping->private_lock); 4031da177e4SLinus Torvalds if (!page_has_buffers(page)) 4041da177e4SLinus Torvalds goto out_unlock; 4051da177e4SLinus Torvalds head = page_buffers(page); 4061da177e4SLinus Torvalds bh = head; 4071da177e4SLinus Torvalds do { 4081da177e4SLinus Torvalds if (bh->b_blocknr == block) { 4091da177e4SLinus Torvalds ret = bh; 4101da177e4SLinus Torvalds get_bh(bh); 4111da177e4SLinus Torvalds goto out_unlock; 4121da177e4SLinus Torvalds } 4131da177e4SLinus Torvalds if (!buffer_mapped(bh)) 4141da177e4SLinus Torvalds all_mapped = 0; 4151da177e4SLinus Torvalds bh = bh->b_this_page; 4161da177e4SLinus Torvalds } while (bh != head); 4171da177e4SLinus Torvalds 4181da177e4SLinus Torvalds /* we might be here because some of the buffers on this page are 4191da177e4SLinus Torvalds * not mapped. This is due to various races between 4201da177e4SLinus Torvalds * file io on the block device and getblk. It gets dealt with 4211da177e4SLinus Torvalds * elsewhere, don't buffer_error if we had some unmapped buffers 4221da177e4SLinus Torvalds */ 4231da177e4SLinus Torvalds if (all_mapped) { 4241da177e4SLinus Torvalds printk("__find_get_block_slow() failed. " 4251da177e4SLinus Torvalds "block=%llu, b_blocknr=%llu\n", 426205f87f6SBadari Pulavarty (unsigned long long)block, 427205f87f6SBadari Pulavarty (unsigned long long)bh->b_blocknr); 428205f87f6SBadari Pulavarty printk("b_state=0x%08lx, b_size=%zu\n", 429205f87f6SBadari Pulavarty bh->b_state, bh->b_size); 4301da177e4SLinus Torvalds printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits); 4311da177e4SLinus Torvalds } 4321da177e4SLinus Torvalds out_unlock: 4331da177e4SLinus Torvalds spin_unlock(&bd_mapping->private_lock); 4341da177e4SLinus Torvalds page_cache_release(page); 4351da177e4SLinus Torvalds out: 4361da177e4SLinus Torvalds return ret; 4371da177e4SLinus Torvalds } 4381da177e4SLinus Torvalds 4391da177e4SLinus Torvalds /* If invalidate_buffers() will trash dirty buffers, it means some kind 4401da177e4SLinus Torvalds of fs corruption is going on. Trashing dirty data always imply losing 4411da177e4SLinus Torvalds information that was supposed to be just stored on the physical layer 4421da177e4SLinus Torvalds by the user. 4431da177e4SLinus Torvalds 4441da177e4SLinus Torvalds Thus invalidate_buffers in general usage is not allwowed to trash 4451da177e4SLinus Torvalds dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to 4461da177e4SLinus Torvalds be preserved. These buffers are simply skipped. 4471da177e4SLinus Torvalds 4481da177e4SLinus Torvalds We also skip buffers which are still in use. For example this can 4491da177e4SLinus Torvalds happen if a userspace program is reading the block device. 4501da177e4SLinus Torvalds 4511da177e4SLinus Torvalds NOTE: In the case where the user removed a removable-media-disk even if 4521da177e4SLinus Torvalds there's still dirty data not synced on disk (due a bug in the device driver 4531da177e4SLinus Torvalds or due an error of the user), by not destroying the dirty buffers we could 4541da177e4SLinus Torvalds generate corruption also on the next media inserted, thus a parameter is 4551da177e4SLinus Torvalds necessary to handle this case in the most safe way possible (trying 4561da177e4SLinus Torvalds to not corrupt also the new disk inserted with the data belonging to 4571da177e4SLinus Torvalds the old now corrupted disk). Also for the ramdisk the natural thing 4581da177e4SLinus Torvalds to do in order to release the ramdisk memory is to destroy dirty buffers. 4591da177e4SLinus Torvalds 4601da177e4SLinus Torvalds These are two special cases. Normal usage imply the device driver 4611da177e4SLinus Torvalds to issue a sync on the device (without waiting I/O completion) and 4621da177e4SLinus Torvalds then an invalidate_buffers call that doesn't trash dirty buffers. 4631da177e4SLinus Torvalds 4641da177e4SLinus Torvalds For handling cache coherency with the blkdev pagecache the 'update' case 4651da177e4SLinus Torvalds is been introduced. It is needed to re-read from disk any pinned 4661da177e4SLinus Torvalds buffer. NOTE: re-reading from disk is destructive so we can do it only 4671da177e4SLinus Torvalds when we assume nobody is changing the buffercache under our I/O and when 4681da177e4SLinus Torvalds we think the disk contains more recent information than the buffercache. 4691da177e4SLinus Torvalds The update == 1 pass marks the buffers we need to update, the update == 2 4701da177e4SLinus Torvalds pass does the actual I/O. */ 4711da177e4SLinus Torvalds void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers) 4721da177e4SLinus Torvalds { 473*0e1dfc66SAndrew Morton struct address_space *mapping = bdev->bd_inode->i_mapping; 474*0e1dfc66SAndrew Morton 475*0e1dfc66SAndrew Morton if (mapping->nrpages == 0) 476*0e1dfc66SAndrew Morton return; 477*0e1dfc66SAndrew Morton 4781da177e4SLinus Torvalds invalidate_bh_lrus(); 4791da177e4SLinus Torvalds /* 4801da177e4SLinus Torvalds * FIXME: what about destroy_dirty_buffers? 4811da177e4SLinus Torvalds * We really want to use invalidate_inode_pages2() for 4821da177e4SLinus Torvalds * that, but not until that's cleaned up. 4831da177e4SLinus Torvalds */ 484*0e1dfc66SAndrew Morton invalidate_inode_pages(mapping); 4851da177e4SLinus Torvalds } 4861da177e4SLinus Torvalds 4871da177e4SLinus Torvalds /* 4881da177e4SLinus Torvalds * Kick pdflush then try to free up some ZONE_NORMAL memory. 4891da177e4SLinus Torvalds */ 4901da177e4SLinus Torvalds static void free_more_memory(void) 4911da177e4SLinus Torvalds { 4921da177e4SLinus Torvalds struct zone **zones; 4931da177e4SLinus Torvalds pg_data_t *pgdat; 4941da177e4SLinus Torvalds 495687a21ceSPekka J Enberg wakeup_pdflush(1024); 4961da177e4SLinus Torvalds yield(); 4971da177e4SLinus Torvalds 498ec936fc5SKAMEZAWA Hiroyuki for_each_online_pgdat(pgdat) { 499af4ca457SAl Viro zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones; 5001da177e4SLinus Torvalds if (*zones) 5011ad539b2SDarren Hart try_to_free_pages(zones, GFP_NOFS); 5021da177e4SLinus Torvalds } 5031da177e4SLinus Torvalds } 5041da177e4SLinus Torvalds 5051da177e4SLinus Torvalds /* 5061da177e4SLinus Torvalds * I/O completion handler for block_read_full_page() - pages 5071da177e4SLinus Torvalds * which come unlocked at the end of I/O. 5081da177e4SLinus Torvalds */ 5091da177e4SLinus Torvalds static void end_buffer_async_read(struct buffer_head *bh, int uptodate) 5101da177e4SLinus Torvalds { 5111da177e4SLinus Torvalds unsigned long flags; 512a3972203SNick Piggin struct buffer_head *first; 5131da177e4SLinus Torvalds struct buffer_head *tmp; 5141da177e4SLinus Torvalds struct page *page; 5151da177e4SLinus Torvalds int page_uptodate = 1; 5161da177e4SLinus Torvalds 5171da177e4SLinus Torvalds BUG_ON(!buffer_async_read(bh)); 5181da177e4SLinus Torvalds 5191da177e4SLinus Torvalds page = bh->b_page; 5201da177e4SLinus Torvalds if (uptodate) { 5211da177e4SLinus Torvalds set_buffer_uptodate(bh); 5221da177e4SLinus Torvalds } else { 5231da177e4SLinus Torvalds clear_buffer_uptodate(bh); 5241da177e4SLinus Torvalds if (printk_ratelimit()) 5251da177e4SLinus Torvalds buffer_io_error(bh); 5261da177e4SLinus Torvalds SetPageError(page); 5271da177e4SLinus Torvalds } 5281da177e4SLinus Torvalds 5291da177e4SLinus Torvalds /* 5301da177e4SLinus Torvalds * Be _very_ careful from here on. Bad things can happen if 5311da177e4SLinus Torvalds * two buffer heads end IO at almost the same time and both 5321da177e4SLinus Torvalds * decide that the page is now completely done. 5331da177e4SLinus Torvalds */ 534a3972203SNick Piggin first = page_buffers(page); 535a3972203SNick Piggin local_irq_save(flags); 536a3972203SNick Piggin bit_spin_lock(BH_Uptodate_Lock, &first->b_state); 5371da177e4SLinus Torvalds clear_buffer_async_read(bh); 5381da177e4SLinus Torvalds unlock_buffer(bh); 5391da177e4SLinus Torvalds tmp = bh; 5401da177e4SLinus Torvalds do { 5411da177e4SLinus Torvalds if (!buffer_uptodate(tmp)) 5421da177e4SLinus Torvalds page_uptodate = 0; 5431da177e4SLinus Torvalds if (buffer_async_read(tmp)) { 5441da177e4SLinus Torvalds BUG_ON(!buffer_locked(tmp)); 5451da177e4SLinus Torvalds goto still_busy; 5461da177e4SLinus Torvalds } 5471da177e4SLinus Torvalds tmp = tmp->b_this_page; 5481da177e4SLinus Torvalds } while (tmp != bh); 549a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 550a3972203SNick Piggin local_irq_restore(flags); 5511da177e4SLinus Torvalds 5521da177e4SLinus Torvalds /* 5531da177e4SLinus Torvalds * If none of the buffers had errors and they are all 5541da177e4SLinus Torvalds * uptodate then we can set the page uptodate. 5551da177e4SLinus Torvalds */ 5561da177e4SLinus Torvalds if (page_uptodate && !PageError(page)) 5571da177e4SLinus Torvalds SetPageUptodate(page); 5581da177e4SLinus Torvalds unlock_page(page); 5591da177e4SLinus Torvalds return; 5601da177e4SLinus Torvalds 5611da177e4SLinus Torvalds still_busy: 562a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 563a3972203SNick Piggin local_irq_restore(flags); 5641da177e4SLinus Torvalds return; 5651da177e4SLinus Torvalds } 5661da177e4SLinus Torvalds 5671da177e4SLinus Torvalds /* 5681da177e4SLinus Torvalds * Completion handler for block_write_full_page() - pages which are unlocked 5691da177e4SLinus Torvalds * during I/O, and which have PageWriteback cleared upon I/O completion. 5701da177e4SLinus Torvalds */ 571b6cd0b77SAdrian Bunk static void end_buffer_async_write(struct buffer_head *bh, int uptodate) 5721da177e4SLinus Torvalds { 5731da177e4SLinus Torvalds char b[BDEVNAME_SIZE]; 5741da177e4SLinus Torvalds unsigned long flags; 575a3972203SNick Piggin struct buffer_head *first; 5761da177e4SLinus Torvalds struct buffer_head *tmp; 5771da177e4SLinus Torvalds struct page *page; 5781da177e4SLinus Torvalds 5791da177e4SLinus Torvalds BUG_ON(!buffer_async_write(bh)); 5801da177e4SLinus Torvalds 5811da177e4SLinus Torvalds page = bh->b_page; 5821da177e4SLinus Torvalds if (uptodate) { 5831da177e4SLinus Torvalds set_buffer_uptodate(bh); 5841da177e4SLinus Torvalds } else { 5851da177e4SLinus Torvalds if (printk_ratelimit()) { 5861da177e4SLinus Torvalds buffer_io_error(bh); 5871da177e4SLinus Torvalds printk(KERN_WARNING "lost page write due to " 5881da177e4SLinus Torvalds "I/O error on %s\n", 5891da177e4SLinus Torvalds bdevname(bh->b_bdev, b)); 5901da177e4SLinus Torvalds } 5911da177e4SLinus Torvalds set_bit(AS_EIO, &page->mapping->flags); 5921da177e4SLinus Torvalds clear_buffer_uptodate(bh); 5931da177e4SLinus Torvalds SetPageError(page); 5941da177e4SLinus Torvalds } 5951da177e4SLinus Torvalds 596a3972203SNick Piggin first = page_buffers(page); 597a3972203SNick Piggin local_irq_save(flags); 598a3972203SNick Piggin bit_spin_lock(BH_Uptodate_Lock, &first->b_state); 599a3972203SNick Piggin 6001da177e4SLinus Torvalds clear_buffer_async_write(bh); 6011da177e4SLinus Torvalds unlock_buffer(bh); 6021da177e4SLinus Torvalds tmp = bh->b_this_page; 6031da177e4SLinus Torvalds while (tmp != bh) { 6041da177e4SLinus Torvalds if (buffer_async_write(tmp)) { 6051da177e4SLinus Torvalds BUG_ON(!buffer_locked(tmp)); 6061da177e4SLinus Torvalds goto still_busy; 6071da177e4SLinus Torvalds } 6081da177e4SLinus Torvalds tmp = tmp->b_this_page; 6091da177e4SLinus Torvalds } 610a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 611a3972203SNick Piggin local_irq_restore(flags); 6121da177e4SLinus Torvalds end_page_writeback(page); 6131da177e4SLinus Torvalds return; 6141da177e4SLinus Torvalds 6151da177e4SLinus Torvalds still_busy: 616a3972203SNick Piggin bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 617a3972203SNick Piggin local_irq_restore(flags); 6181da177e4SLinus Torvalds return; 6191da177e4SLinus Torvalds } 6201da177e4SLinus Torvalds 6211da177e4SLinus Torvalds /* 6221da177e4SLinus Torvalds * If a page's buffers are under async readin (end_buffer_async_read 6231da177e4SLinus Torvalds * completion) then there is a possibility that another thread of 6241da177e4SLinus Torvalds * control could lock one of the buffers after it has completed 6251da177e4SLinus Torvalds * but while some of the other buffers have not completed. This 6261da177e4SLinus Torvalds * locked buffer would confuse end_buffer_async_read() into not unlocking 6271da177e4SLinus Torvalds * the page. So the absence of BH_Async_Read tells end_buffer_async_read() 6281da177e4SLinus Torvalds * that this buffer is not under async I/O. 6291da177e4SLinus Torvalds * 6301da177e4SLinus Torvalds * The page comes unlocked when it has no locked buffer_async buffers 6311da177e4SLinus Torvalds * left. 6321da177e4SLinus Torvalds * 6331da177e4SLinus Torvalds * PageLocked prevents anyone starting new async I/O reads any of 6341da177e4SLinus Torvalds * the buffers. 6351da177e4SLinus Torvalds * 6361da177e4SLinus Torvalds * PageWriteback is used to prevent simultaneous writeout of the same 6371da177e4SLinus Torvalds * page. 6381da177e4SLinus Torvalds * 6391da177e4SLinus Torvalds * PageLocked prevents anyone from starting writeback of a page which is 6401da177e4SLinus Torvalds * under read I/O (PageWriteback is only ever set against a locked page). 6411da177e4SLinus Torvalds */ 6421da177e4SLinus Torvalds static void mark_buffer_async_read(struct buffer_head *bh) 6431da177e4SLinus Torvalds { 6441da177e4SLinus Torvalds bh->b_end_io = end_buffer_async_read; 6451da177e4SLinus Torvalds set_buffer_async_read(bh); 6461da177e4SLinus Torvalds } 6471da177e4SLinus Torvalds 6481da177e4SLinus Torvalds void mark_buffer_async_write(struct buffer_head *bh) 6491da177e4SLinus Torvalds { 6501da177e4SLinus Torvalds bh->b_end_io = end_buffer_async_write; 6511da177e4SLinus Torvalds set_buffer_async_write(bh); 6521da177e4SLinus Torvalds } 6531da177e4SLinus Torvalds EXPORT_SYMBOL(mark_buffer_async_write); 6541da177e4SLinus Torvalds 6551da177e4SLinus Torvalds 6561da177e4SLinus Torvalds /* 6571da177e4SLinus Torvalds * fs/buffer.c contains helper functions for buffer-backed address space's 6581da177e4SLinus Torvalds * fsync functions. A common requirement for buffer-based filesystems is 6591da177e4SLinus Torvalds * that certain data from the backing blockdev needs to be written out for 6601da177e4SLinus Torvalds * a successful fsync(). For example, ext2 indirect blocks need to be 6611da177e4SLinus Torvalds * written back and waited upon before fsync() returns. 6621da177e4SLinus Torvalds * 6631da177e4SLinus Torvalds * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(), 6641da177e4SLinus Torvalds * inode_has_buffers() and invalidate_inode_buffers() are provided for the 6651da177e4SLinus Torvalds * management of a list of dependent buffers at ->i_mapping->private_list. 6661da177e4SLinus Torvalds * 6671da177e4SLinus Torvalds * Locking is a little subtle: try_to_free_buffers() will remove buffers 6681da177e4SLinus Torvalds * from their controlling inode's queue when they are being freed. But 6691da177e4SLinus Torvalds * try_to_free_buffers() will be operating against the *blockdev* mapping 6701da177e4SLinus Torvalds * at the time, not against the S_ISREG file which depends on those buffers. 6711da177e4SLinus Torvalds * So the locking for private_list is via the private_lock in the address_space 6721da177e4SLinus Torvalds * which backs the buffers. Which is different from the address_space 6731da177e4SLinus Torvalds * against which the buffers are listed. So for a particular address_space, 6741da177e4SLinus Torvalds * mapping->private_lock does *not* protect mapping->private_list! In fact, 6751da177e4SLinus Torvalds * mapping->private_list will always be protected by the backing blockdev's 6761da177e4SLinus Torvalds * ->private_lock. 6771da177e4SLinus Torvalds * 6781da177e4SLinus Torvalds * Which introduces a requirement: all buffers on an address_space's 6791da177e4SLinus Torvalds * ->private_list must be from the same address_space: the blockdev's. 6801da177e4SLinus Torvalds * 6811da177e4SLinus Torvalds * address_spaces which do not place buffers at ->private_list via these 6821da177e4SLinus Torvalds * utility functions are free to use private_lock and private_list for 6831da177e4SLinus Torvalds * whatever they want. The only requirement is that list_empty(private_list) 6841da177e4SLinus Torvalds * be true at clear_inode() time. 6851da177e4SLinus Torvalds * 6861da177e4SLinus Torvalds * FIXME: clear_inode should not call invalidate_inode_buffers(). The 6871da177e4SLinus Torvalds * filesystems should do that. invalidate_inode_buffers() should just go 6881da177e4SLinus Torvalds * BUG_ON(!list_empty). 6891da177e4SLinus Torvalds * 6901da177e4SLinus Torvalds * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should 6911da177e4SLinus Torvalds * take an address_space, not an inode. And it should be called 6921da177e4SLinus Torvalds * mark_buffer_dirty_fsync() to clearly define why those buffers are being 6931da177e4SLinus Torvalds * queued up. 6941da177e4SLinus Torvalds * 6951da177e4SLinus Torvalds * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the 6961da177e4SLinus Torvalds * list if it is already on a list. Because if the buffer is on a list, 6971da177e4SLinus Torvalds * it *must* already be on the right one. If not, the filesystem is being 6981da177e4SLinus Torvalds * silly. This will save a ton of locking. But first we have to ensure 6991da177e4SLinus Torvalds * that buffers are taken *off* the old inode's list when they are freed 7001da177e4SLinus Torvalds * (presumably in truncate). That requires careful auditing of all 7011da177e4SLinus Torvalds * filesystems (do it inside bforget()). It could also be done by bringing 7021da177e4SLinus Torvalds * b_inode back. 7031da177e4SLinus Torvalds */ 7041da177e4SLinus Torvalds 7051da177e4SLinus Torvalds /* 7061da177e4SLinus Torvalds * The buffer's backing address_space's private_lock must be held 7071da177e4SLinus Torvalds */ 7081da177e4SLinus Torvalds static inline void __remove_assoc_queue(struct buffer_head *bh) 7091da177e4SLinus Torvalds { 7101da177e4SLinus Torvalds list_del_init(&bh->b_assoc_buffers); 7111da177e4SLinus Torvalds } 7121da177e4SLinus Torvalds 7131da177e4SLinus Torvalds int inode_has_buffers(struct inode *inode) 7141da177e4SLinus Torvalds { 7151da177e4SLinus Torvalds return !list_empty(&inode->i_data.private_list); 7161da177e4SLinus Torvalds } 7171da177e4SLinus Torvalds 7181da177e4SLinus Torvalds /* 7191da177e4SLinus Torvalds * osync is designed to support O_SYNC io. It waits synchronously for 7201da177e4SLinus Torvalds * all already-submitted IO to complete, but does not queue any new 7211da177e4SLinus Torvalds * writes to the disk. 7221da177e4SLinus Torvalds * 7231da177e4SLinus Torvalds * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as 7241da177e4SLinus Torvalds * you dirty the buffers, and then use osync_inode_buffers to wait for 7251da177e4SLinus Torvalds * completion. Any other dirty buffers which are not yet queued for 7261da177e4SLinus Torvalds * write will not be flushed to disk by the osync. 7271da177e4SLinus Torvalds */ 7281da177e4SLinus Torvalds static int osync_buffers_list(spinlock_t *lock, struct list_head *list) 7291da177e4SLinus Torvalds { 7301da177e4SLinus Torvalds struct buffer_head *bh; 7311da177e4SLinus Torvalds struct list_head *p; 7321da177e4SLinus Torvalds int err = 0; 7331da177e4SLinus Torvalds 7341da177e4SLinus Torvalds spin_lock(lock); 7351da177e4SLinus Torvalds repeat: 7361da177e4SLinus Torvalds list_for_each_prev(p, list) { 7371da177e4SLinus Torvalds bh = BH_ENTRY(p); 7381da177e4SLinus Torvalds if (buffer_locked(bh)) { 7391da177e4SLinus Torvalds get_bh(bh); 7401da177e4SLinus Torvalds spin_unlock(lock); 7411da177e4SLinus Torvalds wait_on_buffer(bh); 7421da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 7431da177e4SLinus Torvalds err = -EIO; 7441da177e4SLinus Torvalds brelse(bh); 7451da177e4SLinus Torvalds spin_lock(lock); 7461da177e4SLinus Torvalds goto repeat; 7471da177e4SLinus Torvalds } 7481da177e4SLinus Torvalds } 7491da177e4SLinus Torvalds spin_unlock(lock); 7501da177e4SLinus Torvalds return err; 7511da177e4SLinus Torvalds } 7521da177e4SLinus Torvalds 7531da177e4SLinus Torvalds /** 7541da177e4SLinus Torvalds * sync_mapping_buffers - write out and wait upon a mapping's "associated" 7551da177e4SLinus Torvalds * buffers 75667be2dd1SMartin Waitz * @mapping: the mapping which wants those buffers written 7571da177e4SLinus Torvalds * 7581da177e4SLinus Torvalds * Starts I/O against the buffers at mapping->private_list, and waits upon 7591da177e4SLinus Torvalds * that I/O. 7601da177e4SLinus Torvalds * 76167be2dd1SMartin Waitz * Basically, this is a convenience function for fsync(). 76267be2dd1SMartin Waitz * @mapping is a file or directory which needs those buffers to be written for 76367be2dd1SMartin Waitz * a successful fsync(). 7641da177e4SLinus Torvalds */ 7651da177e4SLinus Torvalds int sync_mapping_buffers(struct address_space *mapping) 7661da177e4SLinus Torvalds { 7671da177e4SLinus Torvalds struct address_space *buffer_mapping = mapping->assoc_mapping; 7681da177e4SLinus Torvalds 7691da177e4SLinus Torvalds if (buffer_mapping == NULL || list_empty(&mapping->private_list)) 7701da177e4SLinus Torvalds return 0; 7711da177e4SLinus Torvalds 7721da177e4SLinus Torvalds return fsync_buffers_list(&buffer_mapping->private_lock, 7731da177e4SLinus Torvalds &mapping->private_list); 7741da177e4SLinus Torvalds } 7751da177e4SLinus Torvalds EXPORT_SYMBOL(sync_mapping_buffers); 7761da177e4SLinus Torvalds 7771da177e4SLinus Torvalds /* 7781da177e4SLinus Torvalds * Called when we've recently written block `bblock', and it is known that 7791da177e4SLinus Torvalds * `bblock' was for a buffer_boundary() buffer. This means that the block at 7801da177e4SLinus Torvalds * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's 7811da177e4SLinus Torvalds * dirty, schedule it for IO. So that indirects merge nicely with their data. 7821da177e4SLinus Torvalds */ 7831da177e4SLinus Torvalds void write_boundary_block(struct block_device *bdev, 7841da177e4SLinus Torvalds sector_t bblock, unsigned blocksize) 7851da177e4SLinus Torvalds { 7861da177e4SLinus Torvalds struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize); 7871da177e4SLinus Torvalds if (bh) { 7881da177e4SLinus Torvalds if (buffer_dirty(bh)) 7891da177e4SLinus Torvalds ll_rw_block(WRITE, 1, &bh); 7901da177e4SLinus Torvalds put_bh(bh); 7911da177e4SLinus Torvalds } 7921da177e4SLinus Torvalds } 7931da177e4SLinus Torvalds 7941da177e4SLinus Torvalds void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) 7951da177e4SLinus Torvalds { 7961da177e4SLinus Torvalds struct address_space *mapping = inode->i_mapping; 7971da177e4SLinus Torvalds struct address_space *buffer_mapping = bh->b_page->mapping; 7981da177e4SLinus Torvalds 7991da177e4SLinus Torvalds mark_buffer_dirty(bh); 8001da177e4SLinus Torvalds if (!mapping->assoc_mapping) { 8011da177e4SLinus Torvalds mapping->assoc_mapping = buffer_mapping; 8021da177e4SLinus Torvalds } else { 803e827f923SEric Sesterhenn BUG_ON(mapping->assoc_mapping != buffer_mapping); 8041da177e4SLinus Torvalds } 8051da177e4SLinus Torvalds if (list_empty(&bh->b_assoc_buffers)) { 8061da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 8071da177e4SLinus Torvalds list_move_tail(&bh->b_assoc_buffers, 8081da177e4SLinus Torvalds &mapping->private_list); 8091da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 8101da177e4SLinus Torvalds } 8111da177e4SLinus Torvalds } 8121da177e4SLinus Torvalds EXPORT_SYMBOL(mark_buffer_dirty_inode); 8131da177e4SLinus Torvalds 8141da177e4SLinus Torvalds /* 8151da177e4SLinus Torvalds * Add a page to the dirty page list. 8161da177e4SLinus Torvalds * 8171da177e4SLinus Torvalds * It is a sad fact of life that this function is called from several places 8181da177e4SLinus Torvalds * deeply under spinlocking. It may not sleep. 8191da177e4SLinus Torvalds * 8201da177e4SLinus Torvalds * If the page has buffers, the uptodate buffers are set dirty, to preserve 8211da177e4SLinus Torvalds * dirty-state coherency between the page and the buffers. It the page does 8221da177e4SLinus Torvalds * not have buffers then when they are later attached they will all be set 8231da177e4SLinus Torvalds * dirty. 8241da177e4SLinus Torvalds * 8251da177e4SLinus Torvalds * The buffers are dirtied before the page is dirtied. There's a small race 8261da177e4SLinus Torvalds * window in which a writepage caller may see the page cleanness but not the 8271da177e4SLinus Torvalds * buffer dirtiness. That's fine. If this code were to set the page dirty 8281da177e4SLinus Torvalds * before the buffers, a concurrent writepage caller could clear the page dirty 8291da177e4SLinus Torvalds * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean 8301da177e4SLinus Torvalds * page on the dirty page list. 8311da177e4SLinus Torvalds * 8321da177e4SLinus Torvalds * We use private_lock to lock against try_to_free_buffers while using the 8331da177e4SLinus Torvalds * page's buffer list. Also use this to protect against clean buffers being 8341da177e4SLinus Torvalds * added to the page after it was set dirty. 8351da177e4SLinus Torvalds * 8361da177e4SLinus Torvalds * FIXME: may need to call ->reservepage here as well. That's rather up to the 8371da177e4SLinus Torvalds * address_space though. 8381da177e4SLinus Torvalds */ 8391da177e4SLinus Torvalds int __set_page_dirty_buffers(struct page *page) 8401da177e4SLinus Torvalds { 8411da177e4SLinus Torvalds struct address_space * const mapping = page->mapping; 8421da177e4SLinus Torvalds 8431da177e4SLinus Torvalds spin_lock(&mapping->private_lock); 8441da177e4SLinus Torvalds if (page_has_buffers(page)) { 8451da177e4SLinus Torvalds struct buffer_head *head = page_buffers(page); 8461da177e4SLinus Torvalds struct buffer_head *bh = head; 8471da177e4SLinus Torvalds 8481da177e4SLinus Torvalds do { 8491da177e4SLinus Torvalds set_buffer_dirty(bh); 8501da177e4SLinus Torvalds bh = bh->b_this_page; 8511da177e4SLinus Torvalds } while (bh != head); 8521da177e4SLinus Torvalds } 8531da177e4SLinus Torvalds spin_unlock(&mapping->private_lock); 8541da177e4SLinus Torvalds 8551da177e4SLinus Torvalds if (!TestSetPageDirty(page)) { 8561da177e4SLinus Torvalds write_lock_irq(&mapping->tree_lock); 8571da177e4SLinus Torvalds if (page->mapping) { /* Race with truncate? */ 8581da177e4SLinus Torvalds if (mapping_cap_account_dirty(mapping)) 859b1e7a8fdSChristoph Lameter __inc_zone_page_state(page, NR_FILE_DIRTY); 8601da177e4SLinus Torvalds radix_tree_tag_set(&mapping->page_tree, 8611da177e4SLinus Torvalds page_index(page), 8621da177e4SLinus Torvalds PAGECACHE_TAG_DIRTY); 8631da177e4SLinus Torvalds } 8641da177e4SLinus Torvalds write_unlock_irq(&mapping->tree_lock); 8651da177e4SLinus Torvalds __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 8664741c9fdSAndrew Morton return 1; 8671da177e4SLinus Torvalds } 8681da177e4SLinus Torvalds return 0; 8691da177e4SLinus Torvalds } 8701da177e4SLinus Torvalds EXPORT_SYMBOL(__set_page_dirty_buffers); 8711da177e4SLinus Torvalds 8721da177e4SLinus Torvalds /* 8731da177e4SLinus Torvalds * Write out and wait upon a list of buffers. 8741da177e4SLinus Torvalds * 8751da177e4SLinus Torvalds * We have conflicting pressures: we want to make sure that all 8761da177e4SLinus Torvalds * initially dirty buffers get waited on, but that any subsequently 8771da177e4SLinus Torvalds * dirtied buffers don't. After all, we don't want fsync to last 8781da177e4SLinus Torvalds * forever if somebody is actively writing to the file. 8791da177e4SLinus Torvalds * 8801da177e4SLinus Torvalds * Do this in two main stages: first we copy dirty buffers to a 8811da177e4SLinus Torvalds * temporary inode list, queueing the writes as we go. Then we clean 8821da177e4SLinus Torvalds * up, waiting for those writes to complete. 8831da177e4SLinus Torvalds * 8841da177e4SLinus Torvalds * During this second stage, any subsequent updates to the file may end 8851da177e4SLinus Torvalds * up refiling the buffer on the original inode's dirty list again, so 8861da177e4SLinus Torvalds * there is a chance we will end up with a buffer queued for write but 8871da177e4SLinus Torvalds * not yet completed on that list. So, as a final cleanup we go through 8881da177e4SLinus Torvalds * the osync code to catch these locked, dirty buffers without requeuing 8891da177e4SLinus Torvalds * any newly dirty buffers for write. 8901da177e4SLinus Torvalds */ 8911da177e4SLinus Torvalds static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) 8921da177e4SLinus Torvalds { 8931da177e4SLinus Torvalds struct buffer_head *bh; 8941da177e4SLinus Torvalds struct list_head tmp; 8951da177e4SLinus Torvalds int err = 0, err2; 8961da177e4SLinus Torvalds 8971da177e4SLinus Torvalds INIT_LIST_HEAD(&tmp); 8981da177e4SLinus Torvalds 8991da177e4SLinus Torvalds spin_lock(lock); 9001da177e4SLinus Torvalds while (!list_empty(list)) { 9011da177e4SLinus Torvalds bh = BH_ENTRY(list->next); 9021da177e4SLinus Torvalds list_del_init(&bh->b_assoc_buffers); 9031da177e4SLinus Torvalds if (buffer_dirty(bh) || buffer_locked(bh)) { 9041da177e4SLinus Torvalds list_add(&bh->b_assoc_buffers, &tmp); 9051da177e4SLinus Torvalds if (buffer_dirty(bh)) { 9061da177e4SLinus Torvalds get_bh(bh); 9071da177e4SLinus Torvalds spin_unlock(lock); 9081da177e4SLinus Torvalds /* 9091da177e4SLinus Torvalds * Ensure any pending I/O completes so that 9101da177e4SLinus Torvalds * ll_rw_block() actually writes the current 9111da177e4SLinus Torvalds * contents - it is a noop if I/O is still in 9121da177e4SLinus Torvalds * flight on potentially older contents. 9131da177e4SLinus Torvalds */ 914a7662236SJan Kara ll_rw_block(SWRITE, 1, &bh); 9151da177e4SLinus Torvalds brelse(bh); 9161da177e4SLinus Torvalds spin_lock(lock); 9171da177e4SLinus Torvalds } 9181da177e4SLinus Torvalds } 9191da177e4SLinus Torvalds } 9201da177e4SLinus Torvalds 9211da177e4SLinus Torvalds while (!list_empty(&tmp)) { 9221da177e4SLinus Torvalds bh = BH_ENTRY(tmp.prev); 9231da177e4SLinus Torvalds __remove_assoc_queue(bh); 9241da177e4SLinus Torvalds get_bh(bh); 9251da177e4SLinus Torvalds spin_unlock(lock); 9261da177e4SLinus Torvalds wait_on_buffer(bh); 9271da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 9281da177e4SLinus Torvalds err = -EIO; 9291da177e4SLinus Torvalds brelse(bh); 9301da177e4SLinus Torvalds spin_lock(lock); 9311da177e4SLinus Torvalds } 9321da177e4SLinus Torvalds 9331da177e4SLinus Torvalds spin_unlock(lock); 9341da177e4SLinus Torvalds err2 = osync_buffers_list(lock, list); 9351da177e4SLinus Torvalds if (err) 9361da177e4SLinus Torvalds return err; 9371da177e4SLinus Torvalds else 9381da177e4SLinus Torvalds return err2; 9391da177e4SLinus Torvalds } 9401da177e4SLinus Torvalds 9411da177e4SLinus Torvalds /* 9421da177e4SLinus Torvalds * Invalidate any and all dirty buffers on a given inode. We are 9431da177e4SLinus Torvalds * probably unmounting the fs, but that doesn't mean we have already 9441da177e4SLinus Torvalds * done a sync(). Just drop the buffers from the inode list. 9451da177e4SLinus Torvalds * 9461da177e4SLinus Torvalds * NOTE: we take the inode's blockdev's mapping's private_lock. Which 9471da177e4SLinus Torvalds * assumes that all the buffers are against the blockdev. Not true 9481da177e4SLinus Torvalds * for reiserfs. 9491da177e4SLinus Torvalds */ 9501da177e4SLinus Torvalds void invalidate_inode_buffers(struct inode *inode) 9511da177e4SLinus Torvalds { 9521da177e4SLinus Torvalds if (inode_has_buffers(inode)) { 9531da177e4SLinus Torvalds struct address_space *mapping = &inode->i_data; 9541da177e4SLinus Torvalds struct list_head *list = &mapping->private_list; 9551da177e4SLinus Torvalds struct address_space *buffer_mapping = mapping->assoc_mapping; 9561da177e4SLinus Torvalds 9571da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 9581da177e4SLinus Torvalds while (!list_empty(list)) 9591da177e4SLinus Torvalds __remove_assoc_queue(BH_ENTRY(list->next)); 9601da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 9611da177e4SLinus Torvalds } 9621da177e4SLinus Torvalds } 9631da177e4SLinus Torvalds 9641da177e4SLinus Torvalds /* 9651da177e4SLinus Torvalds * Remove any clean buffers from the inode's buffer list. This is called 9661da177e4SLinus Torvalds * when we're trying to free the inode itself. Those buffers can pin it. 9671da177e4SLinus Torvalds * 9681da177e4SLinus Torvalds * Returns true if all buffers were removed. 9691da177e4SLinus Torvalds */ 9701da177e4SLinus Torvalds int remove_inode_buffers(struct inode *inode) 9711da177e4SLinus Torvalds { 9721da177e4SLinus Torvalds int ret = 1; 9731da177e4SLinus Torvalds 9741da177e4SLinus Torvalds if (inode_has_buffers(inode)) { 9751da177e4SLinus Torvalds struct address_space *mapping = &inode->i_data; 9761da177e4SLinus Torvalds struct list_head *list = &mapping->private_list; 9771da177e4SLinus Torvalds struct address_space *buffer_mapping = mapping->assoc_mapping; 9781da177e4SLinus Torvalds 9791da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 9801da177e4SLinus Torvalds while (!list_empty(list)) { 9811da177e4SLinus Torvalds struct buffer_head *bh = BH_ENTRY(list->next); 9821da177e4SLinus Torvalds if (buffer_dirty(bh)) { 9831da177e4SLinus Torvalds ret = 0; 9841da177e4SLinus Torvalds break; 9851da177e4SLinus Torvalds } 9861da177e4SLinus Torvalds __remove_assoc_queue(bh); 9871da177e4SLinus Torvalds } 9881da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 9891da177e4SLinus Torvalds } 9901da177e4SLinus Torvalds return ret; 9911da177e4SLinus Torvalds } 9921da177e4SLinus Torvalds 9931da177e4SLinus Torvalds /* 9941da177e4SLinus Torvalds * Create the appropriate buffers when given a page for data area and 9951da177e4SLinus Torvalds * the size of each buffer.. Use the bh->b_this_page linked list to 9961da177e4SLinus Torvalds * follow the buffers created. Return NULL if unable to create more 9971da177e4SLinus Torvalds * buffers. 9981da177e4SLinus Torvalds * 9991da177e4SLinus Torvalds * The retry flag is used to differentiate async IO (paging, swapping) 10001da177e4SLinus Torvalds * which may not fail from ordinary buffer allocations. 10011da177e4SLinus Torvalds */ 10021da177e4SLinus Torvalds struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, 10031da177e4SLinus Torvalds int retry) 10041da177e4SLinus Torvalds { 10051da177e4SLinus Torvalds struct buffer_head *bh, *head; 10061da177e4SLinus Torvalds long offset; 10071da177e4SLinus Torvalds 10081da177e4SLinus Torvalds try_again: 10091da177e4SLinus Torvalds head = NULL; 10101da177e4SLinus Torvalds offset = PAGE_SIZE; 10111da177e4SLinus Torvalds while ((offset -= size) >= 0) { 10121da177e4SLinus Torvalds bh = alloc_buffer_head(GFP_NOFS); 10131da177e4SLinus Torvalds if (!bh) 10141da177e4SLinus Torvalds goto no_grow; 10151da177e4SLinus Torvalds 10161da177e4SLinus Torvalds bh->b_bdev = NULL; 10171da177e4SLinus Torvalds bh->b_this_page = head; 10181da177e4SLinus Torvalds bh->b_blocknr = -1; 10191da177e4SLinus Torvalds head = bh; 10201da177e4SLinus Torvalds 10211da177e4SLinus Torvalds bh->b_state = 0; 10221da177e4SLinus Torvalds atomic_set(&bh->b_count, 0); 1023fc5cd582SChris Mason bh->b_private = NULL; 10241da177e4SLinus Torvalds bh->b_size = size; 10251da177e4SLinus Torvalds 10261da177e4SLinus Torvalds /* Link the buffer to its page */ 10271da177e4SLinus Torvalds set_bh_page(bh, page, offset); 10281da177e4SLinus Torvalds 102901ffe339SNathan Scott init_buffer(bh, NULL, NULL); 10301da177e4SLinus Torvalds } 10311da177e4SLinus Torvalds return head; 10321da177e4SLinus Torvalds /* 10331da177e4SLinus Torvalds * In case anything failed, we just free everything we got. 10341da177e4SLinus Torvalds */ 10351da177e4SLinus Torvalds no_grow: 10361da177e4SLinus Torvalds if (head) { 10371da177e4SLinus Torvalds do { 10381da177e4SLinus Torvalds bh = head; 10391da177e4SLinus Torvalds head = head->b_this_page; 10401da177e4SLinus Torvalds free_buffer_head(bh); 10411da177e4SLinus Torvalds } while (head); 10421da177e4SLinus Torvalds } 10431da177e4SLinus Torvalds 10441da177e4SLinus Torvalds /* 10451da177e4SLinus Torvalds * Return failure for non-async IO requests. Async IO requests 10461da177e4SLinus Torvalds * are not allowed to fail, so we have to wait until buffer heads 10471da177e4SLinus Torvalds * become available. But we don't want tasks sleeping with 10481da177e4SLinus Torvalds * partially complete buffers, so all were released above. 10491da177e4SLinus Torvalds */ 10501da177e4SLinus Torvalds if (!retry) 10511da177e4SLinus Torvalds return NULL; 10521da177e4SLinus Torvalds 10531da177e4SLinus Torvalds /* We're _really_ low on memory. Now we just 10541da177e4SLinus Torvalds * wait for old buffer heads to become free due to 10551da177e4SLinus Torvalds * finishing IO. Since this is an async request and 10561da177e4SLinus Torvalds * the reserve list is empty, we're sure there are 10571da177e4SLinus Torvalds * async buffer heads in use. 10581da177e4SLinus Torvalds */ 10591da177e4SLinus Torvalds free_more_memory(); 10601da177e4SLinus Torvalds goto try_again; 10611da177e4SLinus Torvalds } 10621da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(alloc_page_buffers); 10631da177e4SLinus Torvalds 10641da177e4SLinus Torvalds static inline void 10651da177e4SLinus Torvalds link_dev_buffers(struct page *page, struct buffer_head *head) 10661da177e4SLinus Torvalds { 10671da177e4SLinus Torvalds struct buffer_head *bh, *tail; 10681da177e4SLinus Torvalds 10691da177e4SLinus Torvalds bh = head; 10701da177e4SLinus Torvalds do { 10711da177e4SLinus Torvalds tail = bh; 10721da177e4SLinus Torvalds bh = bh->b_this_page; 10731da177e4SLinus Torvalds } while (bh); 10741da177e4SLinus Torvalds tail->b_this_page = head; 10751da177e4SLinus Torvalds attach_page_buffers(page, head); 10761da177e4SLinus Torvalds } 10771da177e4SLinus Torvalds 10781da177e4SLinus Torvalds /* 10791da177e4SLinus Torvalds * Initialise the state of a blockdev page's buffers. 10801da177e4SLinus Torvalds */ 10811da177e4SLinus Torvalds static void 10821da177e4SLinus Torvalds init_page_buffers(struct page *page, struct block_device *bdev, 10831da177e4SLinus Torvalds sector_t block, int size) 10841da177e4SLinus Torvalds { 10851da177e4SLinus Torvalds struct buffer_head *head = page_buffers(page); 10861da177e4SLinus Torvalds struct buffer_head *bh = head; 10871da177e4SLinus Torvalds int uptodate = PageUptodate(page); 10881da177e4SLinus Torvalds 10891da177e4SLinus Torvalds do { 10901da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 10911da177e4SLinus Torvalds init_buffer(bh, NULL, NULL); 10921da177e4SLinus Torvalds bh->b_bdev = bdev; 10931da177e4SLinus Torvalds bh->b_blocknr = block; 10941da177e4SLinus Torvalds if (uptodate) 10951da177e4SLinus Torvalds set_buffer_uptodate(bh); 10961da177e4SLinus Torvalds set_buffer_mapped(bh); 10971da177e4SLinus Torvalds } 10981da177e4SLinus Torvalds block++; 10991da177e4SLinus Torvalds bh = bh->b_this_page; 11001da177e4SLinus Torvalds } while (bh != head); 11011da177e4SLinus Torvalds } 11021da177e4SLinus Torvalds 11031da177e4SLinus Torvalds /* 11041da177e4SLinus Torvalds * Create the page-cache page that contains the requested block. 11051da177e4SLinus Torvalds * 11061da177e4SLinus Torvalds * This is user purely for blockdev mappings. 11071da177e4SLinus Torvalds */ 11081da177e4SLinus Torvalds static struct page * 11091da177e4SLinus Torvalds grow_dev_page(struct block_device *bdev, sector_t block, 11101da177e4SLinus Torvalds pgoff_t index, int size) 11111da177e4SLinus Torvalds { 11121da177e4SLinus Torvalds struct inode *inode = bdev->bd_inode; 11131da177e4SLinus Torvalds struct page *page; 11141da177e4SLinus Torvalds struct buffer_head *bh; 11151da177e4SLinus Torvalds 11161da177e4SLinus Torvalds page = find_or_create_page(inode->i_mapping, index, GFP_NOFS); 11171da177e4SLinus Torvalds if (!page) 11181da177e4SLinus Torvalds return NULL; 11191da177e4SLinus Torvalds 1120e827f923SEric Sesterhenn BUG_ON(!PageLocked(page)); 11211da177e4SLinus Torvalds 11221da177e4SLinus Torvalds if (page_has_buffers(page)) { 11231da177e4SLinus Torvalds bh = page_buffers(page); 11241da177e4SLinus Torvalds if (bh->b_size == size) { 11251da177e4SLinus Torvalds init_page_buffers(page, bdev, block, size); 11261da177e4SLinus Torvalds return page; 11271da177e4SLinus Torvalds } 11281da177e4SLinus Torvalds if (!try_to_free_buffers(page)) 11291da177e4SLinus Torvalds goto failed; 11301da177e4SLinus Torvalds } 11311da177e4SLinus Torvalds 11321da177e4SLinus Torvalds /* 11331da177e4SLinus Torvalds * Allocate some buffers for this page 11341da177e4SLinus Torvalds */ 11351da177e4SLinus Torvalds bh = alloc_page_buffers(page, size, 0); 11361da177e4SLinus Torvalds if (!bh) 11371da177e4SLinus Torvalds goto failed; 11381da177e4SLinus Torvalds 11391da177e4SLinus Torvalds /* 11401da177e4SLinus Torvalds * Link the page to the buffers and initialise them. Take the 11411da177e4SLinus Torvalds * lock to be atomic wrt __find_get_block(), which does not 11421da177e4SLinus Torvalds * run under the page lock. 11431da177e4SLinus Torvalds */ 11441da177e4SLinus Torvalds spin_lock(&inode->i_mapping->private_lock); 11451da177e4SLinus Torvalds link_dev_buffers(page, bh); 11461da177e4SLinus Torvalds init_page_buffers(page, bdev, block, size); 11471da177e4SLinus Torvalds spin_unlock(&inode->i_mapping->private_lock); 11481da177e4SLinus Torvalds return page; 11491da177e4SLinus Torvalds 11501da177e4SLinus Torvalds failed: 11511da177e4SLinus Torvalds BUG(); 11521da177e4SLinus Torvalds unlock_page(page); 11531da177e4SLinus Torvalds page_cache_release(page); 11541da177e4SLinus Torvalds return NULL; 11551da177e4SLinus Torvalds } 11561da177e4SLinus Torvalds 11571da177e4SLinus Torvalds /* 11581da177e4SLinus Torvalds * Create buffers for the specified block device block's page. If 11591da177e4SLinus Torvalds * that page was dirty, the buffers are set dirty also. 11601da177e4SLinus Torvalds * 11611da177e4SLinus Torvalds * Except that's a bug. Attaching dirty buffers to a dirty 11621da177e4SLinus Torvalds * blockdev's page can result in filesystem corruption, because 11631da177e4SLinus Torvalds * some of those buffers may be aliases of filesystem data. 11641da177e4SLinus Torvalds * grow_dev_page() will go BUG() if this happens. 11651da177e4SLinus Torvalds */ 1166858119e1SArjan van de Ven static int 11671da177e4SLinus Torvalds grow_buffers(struct block_device *bdev, sector_t block, int size) 11681da177e4SLinus Torvalds { 11691da177e4SLinus Torvalds struct page *page; 11701da177e4SLinus Torvalds pgoff_t index; 11711da177e4SLinus Torvalds int sizebits; 11721da177e4SLinus Torvalds 11731da177e4SLinus Torvalds sizebits = -1; 11741da177e4SLinus Torvalds do { 11751da177e4SLinus Torvalds sizebits++; 11761da177e4SLinus Torvalds } while ((size << sizebits) < PAGE_SIZE); 11771da177e4SLinus Torvalds 11781da177e4SLinus Torvalds index = block >> sizebits; 11791da177e4SLinus Torvalds block = index << sizebits; 11801da177e4SLinus Torvalds 11811da177e4SLinus Torvalds /* Create a page with the proper size buffers.. */ 11821da177e4SLinus Torvalds page = grow_dev_page(bdev, block, index, size); 11831da177e4SLinus Torvalds if (!page) 11841da177e4SLinus Torvalds return 0; 11851da177e4SLinus Torvalds unlock_page(page); 11861da177e4SLinus Torvalds page_cache_release(page); 11871da177e4SLinus Torvalds return 1; 11881da177e4SLinus Torvalds } 11891da177e4SLinus Torvalds 119075c96f85SAdrian Bunk static struct buffer_head * 11911da177e4SLinus Torvalds __getblk_slow(struct block_device *bdev, sector_t block, int size) 11921da177e4SLinus Torvalds { 11931da177e4SLinus Torvalds /* Size must be multiple of hard sectorsize */ 11941da177e4SLinus Torvalds if (unlikely(size & (bdev_hardsect_size(bdev)-1) || 11951da177e4SLinus Torvalds (size < 512 || size > PAGE_SIZE))) { 11961da177e4SLinus Torvalds printk(KERN_ERR "getblk(): invalid block size %d requested\n", 11971da177e4SLinus Torvalds size); 11981da177e4SLinus Torvalds printk(KERN_ERR "hardsect size: %d\n", 11991da177e4SLinus Torvalds bdev_hardsect_size(bdev)); 12001da177e4SLinus Torvalds 12011da177e4SLinus Torvalds dump_stack(); 12021da177e4SLinus Torvalds return NULL; 12031da177e4SLinus Torvalds } 12041da177e4SLinus Torvalds 12051da177e4SLinus Torvalds for (;;) { 12061da177e4SLinus Torvalds struct buffer_head * bh; 12071da177e4SLinus Torvalds 12081da177e4SLinus Torvalds bh = __find_get_block(bdev, block, size); 12091da177e4SLinus Torvalds if (bh) 12101da177e4SLinus Torvalds return bh; 12111da177e4SLinus Torvalds 12121da177e4SLinus Torvalds if (!grow_buffers(bdev, block, size)) 12131da177e4SLinus Torvalds free_more_memory(); 12141da177e4SLinus Torvalds } 12151da177e4SLinus Torvalds } 12161da177e4SLinus Torvalds 12171da177e4SLinus Torvalds /* 12181da177e4SLinus Torvalds * The relationship between dirty buffers and dirty pages: 12191da177e4SLinus Torvalds * 12201da177e4SLinus Torvalds * Whenever a page has any dirty buffers, the page's dirty bit is set, and 12211da177e4SLinus Torvalds * the page is tagged dirty in its radix tree. 12221da177e4SLinus Torvalds * 12231da177e4SLinus Torvalds * At all times, the dirtiness of the buffers represents the dirtiness of 12241da177e4SLinus Torvalds * subsections of the page. If the page has buffers, the page dirty bit is 12251da177e4SLinus Torvalds * merely a hint about the true dirty state. 12261da177e4SLinus Torvalds * 12271da177e4SLinus Torvalds * When a page is set dirty in its entirety, all its buffers are marked dirty 12281da177e4SLinus Torvalds * (if the page has buffers). 12291da177e4SLinus Torvalds * 12301da177e4SLinus Torvalds * When a buffer is marked dirty, its page is dirtied, but the page's other 12311da177e4SLinus Torvalds * buffers are not. 12321da177e4SLinus Torvalds * 12331da177e4SLinus Torvalds * Also. When blockdev buffers are explicitly read with bread(), they 12341da177e4SLinus Torvalds * individually become uptodate. But their backing page remains not 12351da177e4SLinus Torvalds * uptodate - even if all of its buffers are uptodate. A subsequent 12361da177e4SLinus Torvalds * block_read_full_page() against that page will discover all the uptodate 12371da177e4SLinus Torvalds * buffers, will set the page uptodate and will perform no I/O. 12381da177e4SLinus Torvalds */ 12391da177e4SLinus Torvalds 12401da177e4SLinus Torvalds /** 12411da177e4SLinus Torvalds * mark_buffer_dirty - mark a buffer_head as needing writeout 124267be2dd1SMartin Waitz * @bh: the buffer_head to mark dirty 12431da177e4SLinus Torvalds * 12441da177e4SLinus Torvalds * mark_buffer_dirty() will set the dirty bit against the buffer, then set its 12451da177e4SLinus Torvalds * backing page dirty, then tag the page as dirty in its address_space's radix 12461da177e4SLinus Torvalds * tree and then attach the address_space's inode to its superblock's dirty 12471da177e4SLinus Torvalds * inode list. 12481da177e4SLinus Torvalds * 12491da177e4SLinus Torvalds * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, 12501da177e4SLinus Torvalds * mapping->tree_lock and the global inode_lock. 12511da177e4SLinus Torvalds */ 12521da177e4SLinus Torvalds void fastcall mark_buffer_dirty(struct buffer_head *bh) 12531da177e4SLinus Torvalds { 12541da177e4SLinus Torvalds if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh)) 12551da177e4SLinus Torvalds __set_page_dirty_nobuffers(bh->b_page); 12561da177e4SLinus Torvalds } 12571da177e4SLinus Torvalds 12581da177e4SLinus Torvalds /* 12591da177e4SLinus Torvalds * Decrement a buffer_head's reference count. If all buffers against a page 12601da177e4SLinus Torvalds * have zero reference count, are clean and unlocked, and if the page is clean 12611da177e4SLinus Torvalds * and unlocked then try_to_free_buffers() may strip the buffers from the page 12621da177e4SLinus Torvalds * in preparation for freeing it (sometimes, rarely, buffers are removed from 12631da177e4SLinus Torvalds * a page but it ends up not being freed, and buffers may later be reattached). 12641da177e4SLinus Torvalds */ 12651da177e4SLinus Torvalds void __brelse(struct buffer_head * buf) 12661da177e4SLinus Torvalds { 12671da177e4SLinus Torvalds if (atomic_read(&buf->b_count)) { 12681da177e4SLinus Torvalds put_bh(buf); 12691da177e4SLinus Torvalds return; 12701da177e4SLinus Torvalds } 12711da177e4SLinus Torvalds printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n"); 12721da177e4SLinus Torvalds WARN_ON(1); 12731da177e4SLinus Torvalds } 12741da177e4SLinus Torvalds 12751da177e4SLinus Torvalds /* 12761da177e4SLinus Torvalds * bforget() is like brelse(), except it discards any 12771da177e4SLinus Torvalds * potentially dirty data. 12781da177e4SLinus Torvalds */ 12791da177e4SLinus Torvalds void __bforget(struct buffer_head *bh) 12801da177e4SLinus Torvalds { 12811da177e4SLinus Torvalds clear_buffer_dirty(bh); 12821da177e4SLinus Torvalds if (!list_empty(&bh->b_assoc_buffers)) { 12831da177e4SLinus Torvalds struct address_space *buffer_mapping = bh->b_page->mapping; 12841da177e4SLinus Torvalds 12851da177e4SLinus Torvalds spin_lock(&buffer_mapping->private_lock); 12861da177e4SLinus Torvalds list_del_init(&bh->b_assoc_buffers); 12871da177e4SLinus Torvalds spin_unlock(&buffer_mapping->private_lock); 12881da177e4SLinus Torvalds } 12891da177e4SLinus Torvalds __brelse(bh); 12901da177e4SLinus Torvalds } 12911da177e4SLinus Torvalds 12921da177e4SLinus Torvalds static struct buffer_head *__bread_slow(struct buffer_head *bh) 12931da177e4SLinus Torvalds { 12941da177e4SLinus Torvalds lock_buffer(bh); 12951da177e4SLinus Torvalds if (buffer_uptodate(bh)) { 12961da177e4SLinus Torvalds unlock_buffer(bh); 12971da177e4SLinus Torvalds return bh; 12981da177e4SLinus Torvalds } else { 12991da177e4SLinus Torvalds get_bh(bh); 13001da177e4SLinus Torvalds bh->b_end_io = end_buffer_read_sync; 13011da177e4SLinus Torvalds submit_bh(READ, bh); 13021da177e4SLinus Torvalds wait_on_buffer(bh); 13031da177e4SLinus Torvalds if (buffer_uptodate(bh)) 13041da177e4SLinus Torvalds return bh; 13051da177e4SLinus Torvalds } 13061da177e4SLinus Torvalds brelse(bh); 13071da177e4SLinus Torvalds return NULL; 13081da177e4SLinus Torvalds } 13091da177e4SLinus Torvalds 13101da177e4SLinus Torvalds /* 13111da177e4SLinus Torvalds * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block(). 13121da177e4SLinus Torvalds * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their 13131da177e4SLinus Torvalds * refcount elevated by one when they're in an LRU. A buffer can only appear 13141da177e4SLinus Torvalds * once in a particular CPU's LRU. A single buffer can be present in multiple 13151da177e4SLinus Torvalds * CPU's LRUs at the same time. 13161da177e4SLinus Torvalds * 13171da177e4SLinus Torvalds * This is a transparent caching front-end to sb_bread(), sb_getblk() and 13181da177e4SLinus Torvalds * sb_find_get_block(). 13191da177e4SLinus Torvalds * 13201da177e4SLinus Torvalds * The LRUs themselves only need locking against invalidate_bh_lrus. We use 13211da177e4SLinus Torvalds * a local interrupt disable for that. 13221da177e4SLinus Torvalds */ 13231da177e4SLinus Torvalds 13241da177e4SLinus Torvalds #define BH_LRU_SIZE 8 13251da177e4SLinus Torvalds 13261da177e4SLinus Torvalds struct bh_lru { 13271da177e4SLinus Torvalds struct buffer_head *bhs[BH_LRU_SIZE]; 13281da177e4SLinus Torvalds }; 13291da177e4SLinus Torvalds 13301da177e4SLinus Torvalds static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }}; 13311da177e4SLinus Torvalds 13321da177e4SLinus Torvalds #ifdef CONFIG_SMP 13331da177e4SLinus Torvalds #define bh_lru_lock() local_irq_disable() 13341da177e4SLinus Torvalds #define bh_lru_unlock() local_irq_enable() 13351da177e4SLinus Torvalds #else 13361da177e4SLinus Torvalds #define bh_lru_lock() preempt_disable() 13371da177e4SLinus Torvalds #define bh_lru_unlock() preempt_enable() 13381da177e4SLinus Torvalds #endif 13391da177e4SLinus Torvalds 13401da177e4SLinus Torvalds static inline void check_irqs_on(void) 13411da177e4SLinus Torvalds { 13421da177e4SLinus Torvalds #ifdef irqs_disabled 13431da177e4SLinus Torvalds BUG_ON(irqs_disabled()); 13441da177e4SLinus Torvalds #endif 13451da177e4SLinus Torvalds } 13461da177e4SLinus Torvalds 13471da177e4SLinus Torvalds /* 13481da177e4SLinus Torvalds * The LRU management algorithm is dopey-but-simple. Sorry. 13491da177e4SLinus Torvalds */ 13501da177e4SLinus Torvalds static void bh_lru_install(struct buffer_head *bh) 13511da177e4SLinus Torvalds { 13521da177e4SLinus Torvalds struct buffer_head *evictee = NULL; 13531da177e4SLinus Torvalds struct bh_lru *lru; 13541da177e4SLinus Torvalds 13551da177e4SLinus Torvalds check_irqs_on(); 13561da177e4SLinus Torvalds bh_lru_lock(); 13571da177e4SLinus Torvalds lru = &__get_cpu_var(bh_lrus); 13581da177e4SLinus Torvalds if (lru->bhs[0] != bh) { 13591da177e4SLinus Torvalds struct buffer_head *bhs[BH_LRU_SIZE]; 13601da177e4SLinus Torvalds int in; 13611da177e4SLinus Torvalds int out = 0; 13621da177e4SLinus Torvalds 13631da177e4SLinus Torvalds get_bh(bh); 13641da177e4SLinus Torvalds bhs[out++] = bh; 13651da177e4SLinus Torvalds for (in = 0; in < BH_LRU_SIZE; in++) { 13661da177e4SLinus Torvalds struct buffer_head *bh2 = lru->bhs[in]; 13671da177e4SLinus Torvalds 13681da177e4SLinus Torvalds if (bh2 == bh) { 13691da177e4SLinus Torvalds __brelse(bh2); 13701da177e4SLinus Torvalds } else { 13711da177e4SLinus Torvalds if (out >= BH_LRU_SIZE) { 13721da177e4SLinus Torvalds BUG_ON(evictee != NULL); 13731da177e4SLinus Torvalds evictee = bh2; 13741da177e4SLinus Torvalds } else { 13751da177e4SLinus Torvalds bhs[out++] = bh2; 13761da177e4SLinus Torvalds } 13771da177e4SLinus Torvalds } 13781da177e4SLinus Torvalds } 13791da177e4SLinus Torvalds while (out < BH_LRU_SIZE) 13801da177e4SLinus Torvalds bhs[out++] = NULL; 13811da177e4SLinus Torvalds memcpy(lru->bhs, bhs, sizeof(bhs)); 13821da177e4SLinus Torvalds } 13831da177e4SLinus Torvalds bh_lru_unlock(); 13841da177e4SLinus Torvalds 13851da177e4SLinus Torvalds if (evictee) 13861da177e4SLinus Torvalds __brelse(evictee); 13871da177e4SLinus Torvalds } 13881da177e4SLinus Torvalds 13891da177e4SLinus Torvalds /* 13901da177e4SLinus Torvalds * Look up the bh in this cpu's LRU. If it's there, move it to the head. 13911da177e4SLinus Torvalds */ 1392858119e1SArjan van de Ven static struct buffer_head * 13931da177e4SLinus Torvalds lookup_bh_lru(struct block_device *bdev, sector_t block, int size) 13941da177e4SLinus Torvalds { 13951da177e4SLinus Torvalds struct buffer_head *ret = NULL; 13961da177e4SLinus Torvalds struct bh_lru *lru; 13971da177e4SLinus Torvalds int i; 13981da177e4SLinus Torvalds 13991da177e4SLinus Torvalds check_irqs_on(); 14001da177e4SLinus Torvalds bh_lru_lock(); 14011da177e4SLinus Torvalds lru = &__get_cpu_var(bh_lrus); 14021da177e4SLinus Torvalds for (i = 0; i < BH_LRU_SIZE; i++) { 14031da177e4SLinus Torvalds struct buffer_head *bh = lru->bhs[i]; 14041da177e4SLinus Torvalds 14051da177e4SLinus Torvalds if (bh && bh->b_bdev == bdev && 14061da177e4SLinus Torvalds bh->b_blocknr == block && bh->b_size == size) { 14071da177e4SLinus Torvalds if (i) { 14081da177e4SLinus Torvalds while (i) { 14091da177e4SLinus Torvalds lru->bhs[i] = lru->bhs[i - 1]; 14101da177e4SLinus Torvalds i--; 14111da177e4SLinus Torvalds } 14121da177e4SLinus Torvalds lru->bhs[0] = bh; 14131da177e4SLinus Torvalds } 14141da177e4SLinus Torvalds get_bh(bh); 14151da177e4SLinus Torvalds ret = bh; 14161da177e4SLinus Torvalds break; 14171da177e4SLinus Torvalds } 14181da177e4SLinus Torvalds } 14191da177e4SLinus Torvalds bh_lru_unlock(); 14201da177e4SLinus Torvalds return ret; 14211da177e4SLinus Torvalds } 14221da177e4SLinus Torvalds 14231da177e4SLinus Torvalds /* 14241da177e4SLinus Torvalds * Perform a pagecache lookup for the matching buffer. If it's there, refresh 14251da177e4SLinus Torvalds * it in the LRU and mark it as accessed. If it is not present then return 14261da177e4SLinus Torvalds * NULL 14271da177e4SLinus Torvalds */ 14281da177e4SLinus Torvalds struct buffer_head * 14291da177e4SLinus Torvalds __find_get_block(struct block_device *bdev, sector_t block, int size) 14301da177e4SLinus Torvalds { 14311da177e4SLinus Torvalds struct buffer_head *bh = lookup_bh_lru(bdev, block, size); 14321da177e4SLinus Torvalds 14331da177e4SLinus Torvalds if (bh == NULL) { 1434385fd4c5SCoywolf Qi Hunt bh = __find_get_block_slow(bdev, block); 14351da177e4SLinus Torvalds if (bh) 14361da177e4SLinus Torvalds bh_lru_install(bh); 14371da177e4SLinus Torvalds } 14381da177e4SLinus Torvalds if (bh) 14391da177e4SLinus Torvalds touch_buffer(bh); 14401da177e4SLinus Torvalds return bh; 14411da177e4SLinus Torvalds } 14421da177e4SLinus Torvalds EXPORT_SYMBOL(__find_get_block); 14431da177e4SLinus Torvalds 14441da177e4SLinus Torvalds /* 14451da177e4SLinus Torvalds * __getblk will locate (and, if necessary, create) the buffer_head 14461da177e4SLinus Torvalds * which corresponds to the passed block_device, block and size. The 14471da177e4SLinus Torvalds * returned buffer has its reference count incremented. 14481da177e4SLinus Torvalds * 14491da177e4SLinus Torvalds * __getblk() cannot fail - it just keeps trying. If you pass it an 14501da177e4SLinus Torvalds * illegal block number, __getblk() will happily return a buffer_head 14511da177e4SLinus Torvalds * which represents the non-existent block. Very weird. 14521da177e4SLinus Torvalds * 14531da177e4SLinus Torvalds * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers() 14541da177e4SLinus Torvalds * attempt is failing. FIXME, perhaps? 14551da177e4SLinus Torvalds */ 14561da177e4SLinus Torvalds struct buffer_head * 14571da177e4SLinus Torvalds __getblk(struct block_device *bdev, sector_t block, int size) 14581da177e4SLinus Torvalds { 14591da177e4SLinus Torvalds struct buffer_head *bh = __find_get_block(bdev, block, size); 14601da177e4SLinus Torvalds 14611da177e4SLinus Torvalds might_sleep(); 14621da177e4SLinus Torvalds if (bh == NULL) 14631da177e4SLinus Torvalds bh = __getblk_slow(bdev, block, size); 14641da177e4SLinus Torvalds return bh; 14651da177e4SLinus Torvalds } 14661da177e4SLinus Torvalds EXPORT_SYMBOL(__getblk); 14671da177e4SLinus Torvalds 14681da177e4SLinus Torvalds /* 14691da177e4SLinus Torvalds * Do async read-ahead on a buffer.. 14701da177e4SLinus Torvalds */ 14711da177e4SLinus Torvalds void __breadahead(struct block_device *bdev, sector_t block, int size) 14721da177e4SLinus Torvalds { 14731da177e4SLinus Torvalds struct buffer_head *bh = __getblk(bdev, block, size); 1474a3e713b5SAndrew Morton if (likely(bh)) { 14751da177e4SLinus Torvalds ll_rw_block(READA, 1, &bh); 14761da177e4SLinus Torvalds brelse(bh); 14771da177e4SLinus Torvalds } 1478a3e713b5SAndrew Morton } 14791da177e4SLinus Torvalds EXPORT_SYMBOL(__breadahead); 14801da177e4SLinus Torvalds 14811da177e4SLinus Torvalds /** 14821da177e4SLinus Torvalds * __bread() - reads a specified block and returns the bh 148367be2dd1SMartin Waitz * @bdev: the block_device to read from 14841da177e4SLinus Torvalds * @block: number of block 14851da177e4SLinus Torvalds * @size: size (in bytes) to read 14861da177e4SLinus Torvalds * 14871da177e4SLinus Torvalds * Reads a specified block, and returns buffer head that contains it. 14881da177e4SLinus Torvalds * It returns NULL if the block was unreadable. 14891da177e4SLinus Torvalds */ 14901da177e4SLinus Torvalds struct buffer_head * 14911da177e4SLinus Torvalds __bread(struct block_device *bdev, sector_t block, int size) 14921da177e4SLinus Torvalds { 14931da177e4SLinus Torvalds struct buffer_head *bh = __getblk(bdev, block, size); 14941da177e4SLinus Torvalds 1495a3e713b5SAndrew Morton if (likely(bh) && !buffer_uptodate(bh)) 14961da177e4SLinus Torvalds bh = __bread_slow(bh); 14971da177e4SLinus Torvalds return bh; 14981da177e4SLinus Torvalds } 14991da177e4SLinus Torvalds EXPORT_SYMBOL(__bread); 15001da177e4SLinus Torvalds 15011da177e4SLinus Torvalds /* 15021da177e4SLinus Torvalds * invalidate_bh_lrus() is called rarely - but not only at unmount. 15031da177e4SLinus Torvalds * This doesn't race because it runs in each cpu either in irq 15041da177e4SLinus Torvalds * or with preempt disabled. 15051da177e4SLinus Torvalds */ 15061da177e4SLinus Torvalds static void invalidate_bh_lru(void *arg) 15071da177e4SLinus Torvalds { 15081da177e4SLinus Torvalds struct bh_lru *b = &get_cpu_var(bh_lrus); 15091da177e4SLinus Torvalds int i; 15101da177e4SLinus Torvalds 15111da177e4SLinus Torvalds for (i = 0; i < BH_LRU_SIZE; i++) { 15121da177e4SLinus Torvalds brelse(b->bhs[i]); 15131da177e4SLinus Torvalds b->bhs[i] = NULL; 15141da177e4SLinus Torvalds } 15151da177e4SLinus Torvalds put_cpu_var(bh_lrus); 15161da177e4SLinus Torvalds } 15171da177e4SLinus Torvalds 15181da177e4SLinus Torvalds static void invalidate_bh_lrus(void) 15191da177e4SLinus Torvalds { 15201da177e4SLinus Torvalds on_each_cpu(invalidate_bh_lru, NULL, 1, 1); 15211da177e4SLinus Torvalds } 15221da177e4SLinus Torvalds 15231da177e4SLinus Torvalds void set_bh_page(struct buffer_head *bh, 15241da177e4SLinus Torvalds struct page *page, unsigned long offset) 15251da177e4SLinus Torvalds { 15261da177e4SLinus Torvalds bh->b_page = page; 1527e827f923SEric Sesterhenn BUG_ON(offset >= PAGE_SIZE); 15281da177e4SLinus Torvalds if (PageHighMem(page)) 15291da177e4SLinus Torvalds /* 15301da177e4SLinus Torvalds * This catches illegal uses and preserves the offset: 15311da177e4SLinus Torvalds */ 15321da177e4SLinus Torvalds bh->b_data = (char *)(0 + offset); 15331da177e4SLinus Torvalds else 15341da177e4SLinus Torvalds bh->b_data = page_address(page) + offset; 15351da177e4SLinus Torvalds } 15361da177e4SLinus Torvalds EXPORT_SYMBOL(set_bh_page); 15371da177e4SLinus Torvalds 15381da177e4SLinus Torvalds /* 15391da177e4SLinus Torvalds * Called when truncating a buffer on a page completely. 15401da177e4SLinus Torvalds */ 1541858119e1SArjan van de Ven static void discard_buffer(struct buffer_head * bh) 15421da177e4SLinus Torvalds { 15431da177e4SLinus Torvalds lock_buffer(bh); 15441da177e4SLinus Torvalds clear_buffer_dirty(bh); 15451da177e4SLinus Torvalds bh->b_bdev = NULL; 15461da177e4SLinus Torvalds clear_buffer_mapped(bh); 15471da177e4SLinus Torvalds clear_buffer_req(bh); 15481da177e4SLinus Torvalds clear_buffer_new(bh); 15491da177e4SLinus Torvalds clear_buffer_delay(bh); 15501da177e4SLinus Torvalds unlock_buffer(bh); 15511da177e4SLinus Torvalds } 15521da177e4SLinus Torvalds 15531da177e4SLinus Torvalds /** 15541da177e4SLinus Torvalds * try_to_release_page() - release old fs-specific metadata on a page 15551da177e4SLinus Torvalds * 15561da177e4SLinus Torvalds * @page: the page which the kernel is trying to free 15571da177e4SLinus Torvalds * @gfp_mask: memory allocation flags (and I/O mode) 15581da177e4SLinus Torvalds * 15591da177e4SLinus Torvalds * The address_space is to try to release any data against the page 15601da177e4SLinus Torvalds * (presumably at page->private). If the release was successful, return `1'. 15611da177e4SLinus Torvalds * Otherwise return zero. 15621da177e4SLinus Torvalds * 15631da177e4SLinus Torvalds * The @gfp_mask argument specifies whether I/O may be performed to release 15641da177e4SLinus Torvalds * this page (__GFP_IO), and whether the call may block (__GFP_WAIT). 15651da177e4SLinus Torvalds * 15661da177e4SLinus Torvalds * NOTE: @gfp_mask may go away, and this function may become non-blocking. 15671da177e4SLinus Torvalds */ 156827496a8cSAl Viro int try_to_release_page(struct page *page, gfp_t gfp_mask) 15691da177e4SLinus Torvalds { 15701da177e4SLinus Torvalds struct address_space * const mapping = page->mapping; 15711da177e4SLinus Torvalds 15721da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 15731da177e4SLinus Torvalds if (PageWriteback(page)) 15741da177e4SLinus Torvalds return 0; 15751da177e4SLinus Torvalds 15761da177e4SLinus Torvalds if (mapping && mapping->a_ops->releasepage) 15771da177e4SLinus Torvalds return mapping->a_ops->releasepage(page, gfp_mask); 15781da177e4SLinus Torvalds return try_to_free_buffers(page); 15791da177e4SLinus Torvalds } 15801da177e4SLinus Torvalds EXPORT_SYMBOL(try_to_release_page); 15811da177e4SLinus Torvalds 15821da177e4SLinus Torvalds /** 15831da177e4SLinus Torvalds * block_invalidatepage - invalidate part of all of a buffer-backed page 15841da177e4SLinus Torvalds * 15851da177e4SLinus Torvalds * @page: the page which is affected 15861da177e4SLinus Torvalds * @offset: the index of the truncation point 15871da177e4SLinus Torvalds * 15881da177e4SLinus Torvalds * block_invalidatepage() is called when all or part of the page has become 15891da177e4SLinus Torvalds * invalidatedby a truncate operation. 15901da177e4SLinus Torvalds * 15911da177e4SLinus Torvalds * block_invalidatepage() does not have to release all buffers, but it must 15921da177e4SLinus Torvalds * ensure that no dirty buffer is left outside @offset and that no I/O 15931da177e4SLinus Torvalds * is underway against any of the blocks which are outside the truncation 15941da177e4SLinus Torvalds * point. Because the caller is about to free (and possibly reuse) those 15951da177e4SLinus Torvalds * blocks on-disk. 15961da177e4SLinus Torvalds */ 15972ff28e22SNeilBrown void block_invalidatepage(struct page *page, unsigned long offset) 15981da177e4SLinus Torvalds { 15991da177e4SLinus Torvalds struct buffer_head *head, *bh, *next; 16001da177e4SLinus Torvalds unsigned int curr_off = 0; 16011da177e4SLinus Torvalds 16021da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 16031da177e4SLinus Torvalds if (!page_has_buffers(page)) 16041da177e4SLinus Torvalds goto out; 16051da177e4SLinus Torvalds 16061da177e4SLinus Torvalds head = page_buffers(page); 16071da177e4SLinus Torvalds bh = head; 16081da177e4SLinus Torvalds do { 16091da177e4SLinus Torvalds unsigned int next_off = curr_off + bh->b_size; 16101da177e4SLinus Torvalds next = bh->b_this_page; 16111da177e4SLinus Torvalds 16121da177e4SLinus Torvalds /* 16131da177e4SLinus Torvalds * is this block fully invalidated? 16141da177e4SLinus Torvalds */ 16151da177e4SLinus Torvalds if (offset <= curr_off) 16161da177e4SLinus Torvalds discard_buffer(bh); 16171da177e4SLinus Torvalds curr_off = next_off; 16181da177e4SLinus Torvalds bh = next; 16191da177e4SLinus Torvalds } while (bh != head); 16201da177e4SLinus Torvalds 16211da177e4SLinus Torvalds /* 16221da177e4SLinus Torvalds * We release buffers only if the entire page is being invalidated. 16231da177e4SLinus Torvalds * The get_block cached value has been unconditionally invalidated, 16241da177e4SLinus Torvalds * so real IO is not possible anymore. 16251da177e4SLinus Torvalds */ 16261da177e4SLinus Torvalds if (offset == 0) 16272ff28e22SNeilBrown try_to_release_page(page, 0); 16281da177e4SLinus Torvalds out: 16292ff28e22SNeilBrown return; 16301da177e4SLinus Torvalds } 16311da177e4SLinus Torvalds EXPORT_SYMBOL(block_invalidatepage); 16321da177e4SLinus Torvalds 16332ff28e22SNeilBrown void do_invalidatepage(struct page *page, unsigned long offset) 1634aaa4059bSJan Kara { 16352ff28e22SNeilBrown void (*invalidatepage)(struct page *, unsigned long); 16362ff28e22SNeilBrown invalidatepage = page->mapping->a_ops->invalidatepage ? : 16372ff28e22SNeilBrown block_invalidatepage; 16382ff28e22SNeilBrown (*invalidatepage)(page, offset); 1639aaa4059bSJan Kara } 1640aaa4059bSJan Kara 16411da177e4SLinus Torvalds /* 16421da177e4SLinus Torvalds * We attach and possibly dirty the buffers atomically wrt 16431da177e4SLinus Torvalds * __set_page_dirty_buffers() via private_lock. try_to_free_buffers 16441da177e4SLinus Torvalds * is already excluded via the page lock. 16451da177e4SLinus Torvalds */ 16461da177e4SLinus Torvalds void create_empty_buffers(struct page *page, 16471da177e4SLinus Torvalds unsigned long blocksize, unsigned long b_state) 16481da177e4SLinus Torvalds { 16491da177e4SLinus Torvalds struct buffer_head *bh, *head, *tail; 16501da177e4SLinus Torvalds 16511da177e4SLinus Torvalds head = alloc_page_buffers(page, blocksize, 1); 16521da177e4SLinus Torvalds bh = head; 16531da177e4SLinus Torvalds do { 16541da177e4SLinus Torvalds bh->b_state |= b_state; 16551da177e4SLinus Torvalds tail = bh; 16561da177e4SLinus Torvalds bh = bh->b_this_page; 16571da177e4SLinus Torvalds } while (bh); 16581da177e4SLinus Torvalds tail->b_this_page = head; 16591da177e4SLinus Torvalds 16601da177e4SLinus Torvalds spin_lock(&page->mapping->private_lock); 16611da177e4SLinus Torvalds if (PageUptodate(page) || PageDirty(page)) { 16621da177e4SLinus Torvalds bh = head; 16631da177e4SLinus Torvalds do { 16641da177e4SLinus Torvalds if (PageDirty(page)) 16651da177e4SLinus Torvalds set_buffer_dirty(bh); 16661da177e4SLinus Torvalds if (PageUptodate(page)) 16671da177e4SLinus Torvalds set_buffer_uptodate(bh); 16681da177e4SLinus Torvalds bh = bh->b_this_page; 16691da177e4SLinus Torvalds } while (bh != head); 16701da177e4SLinus Torvalds } 16711da177e4SLinus Torvalds attach_page_buffers(page, head); 16721da177e4SLinus Torvalds spin_unlock(&page->mapping->private_lock); 16731da177e4SLinus Torvalds } 16741da177e4SLinus Torvalds EXPORT_SYMBOL(create_empty_buffers); 16751da177e4SLinus Torvalds 16761da177e4SLinus Torvalds /* 16771da177e4SLinus Torvalds * We are taking a block for data and we don't want any output from any 16781da177e4SLinus Torvalds * buffer-cache aliases starting from return from that function and 16791da177e4SLinus Torvalds * until the moment when something will explicitly mark the buffer 16801da177e4SLinus Torvalds * dirty (hopefully that will not happen until we will free that block ;-) 16811da177e4SLinus Torvalds * We don't even need to mark it not-uptodate - nobody can expect 16821da177e4SLinus Torvalds * anything from a newly allocated buffer anyway. We used to used 16831da177e4SLinus Torvalds * unmap_buffer() for such invalidation, but that was wrong. We definitely 16841da177e4SLinus Torvalds * don't want to mark the alias unmapped, for example - it would confuse 16851da177e4SLinus Torvalds * anyone who might pick it with bread() afterwards... 16861da177e4SLinus Torvalds * 16871da177e4SLinus Torvalds * Also.. Note that bforget() doesn't lock the buffer. So there can 16881da177e4SLinus Torvalds * be writeout I/O going on against recently-freed buffers. We don't 16891da177e4SLinus Torvalds * wait on that I/O in bforget() - it's more efficient to wait on the I/O 16901da177e4SLinus Torvalds * only if we really need to. That happens here. 16911da177e4SLinus Torvalds */ 16921da177e4SLinus Torvalds void unmap_underlying_metadata(struct block_device *bdev, sector_t block) 16931da177e4SLinus Torvalds { 16941da177e4SLinus Torvalds struct buffer_head *old_bh; 16951da177e4SLinus Torvalds 16961da177e4SLinus Torvalds might_sleep(); 16971da177e4SLinus Torvalds 1698385fd4c5SCoywolf Qi Hunt old_bh = __find_get_block_slow(bdev, block); 16991da177e4SLinus Torvalds if (old_bh) { 17001da177e4SLinus Torvalds clear_buffer_dirty(old_bh); 17011da177e4SLinus Torvalds wait_on_buffer(old_bh); 17021da177e4SLinus Torvalds clear_buffer_req(old_bh); 17031da177e4SLinus Torvalds __brelse(old_bh); 17041da177e4SLinus Torvalds } 17051da177e4SLinus Torvalds } 17061da177e4SLinus Torvalds EXPORT_SYMBOL(unmap_underlying_metadata); 17071da177e4SLinus Torvalds 17081da177e4SLinus Torvalds /* 17091da177e4SLinus Torvalds * NOTE! All mapped/uptodate combinations are valid: 17101da177e4SLinus Torvalds * 17111da177e4SLinus Torvalds * Mapped Uptodate Meaning 17121da177e4SLinus Torvalds * 17131da177e4SLinus Torvalds * No No "unknown" - must do get_block() 17141da177e4SLinus Torvalds * No Yes "hole" - zero-filled 17151da177e4SLinus Torvalds * Yes No "allocated" - allocated on disk, not read in 17161da177e4SLinus Torvalds * Yes Yes "valid" - allocated and up-to-date in memory. 17171da177e4SLinus Torvalds * 17181da177e4SLinus Torvalds * "Dirty" is valid only with the last case (mapped+uptodate). 17191da177e4SLinus Torvalds */ 17201da177e4SLinus Torvalds 17211da177e4SLinus Torvalds /* 17221da177e4SLinus Torvalds * While block_write_full_page is writing back the dirty buffers under 17231da177e4SLinus Torvalds * the page lock, whoever dirtied the buffers may decide to clean them 17241da177e4SLinus Torvalds * again at any time. We handle that by only looking at the buffer 17251da177e4SLinus Torvalds * state inside lock_buffer(). 17261da177e4SLinus Torvalds * 17271da177e4SLinus Torvalds * If block_write_full_page() is called for regular writeback 17281da177e4SLinus Torvalds * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a 17291da177e4SLinus Torvalds * locked buffer. This only can happen if someone has written the buffer 17301da177e4SLinus Torvalds * directly, with submit_bh(). At the address_space level PageWriteback 17311da177e4SLinus Torvalds * prevents this contention from occurring. 17321da177e4SLinus Torvalds */ 17331da177e4SLinus Torvalds static int __block_write_full_page(struct inode *inode, struct page *page, 17341da177e4SLinus Torvalds get_block_t *get_block, struct writeback_control *wbc) 17351da177e4SLinus Torvalds { 17361da177e4SLinus Torvalds int err; 17371da177e4SLinus Torvalds sector_t block; 17381da177e4SLinus Torvalds sector_t last_block; 1739f0fbd5fcSAndrew Morton struct buffer_head *bh, *head; 1740b0cf2321SBadari Pulavarty const unsigned blocksize = 1 << inode->i_blkbits; 17411da177e4SLinus Torvalds int nr_underway = 0; 17421da177e4SLinus Torvalds 17431da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 17441da177e4SLinus Torvalds 17451da177e4SLinus Torvalds last_block = (i_size_read(inode) - 1) >> inode->i_blkbits; 17461da177e4SLinus Torvalds 17471da177e4SLinus Torvalds if (!page_has_buffers(page)) { 1748b0cf2321SBadari Pulavarty create_empty_buffers(page, blocksize, 17491da177e4SLinus Torvalds (1 << BH_Dirty)|(1 << BH_Uptodate)); 17501da177e4SLinus Torvalds } 17511da177e4SLinus Torvalds 17521da177e4SLinus Torvalds /* 17531da177e4SLinus Torvalds * Be very careful. We have no exclusion from __set_page_dirty_buffers 17541da177e4SLinus Torvalds * here, and the (potentially unmapped) buffers may become dirty at 17551da177e4SLinus Torvalds * any time. If a buffer becomes dirty here after we've inspected it 17561da177e4SLinus Torvalds * then we just miss that fact, and the page stays dirty. 17571da177e4SLinus Torvalds * 17581da177e4SLinus Torvalds * Buffers outside i_size may be dirtied by __set_page_dirty_buffers; 17591da177e4SLinus Torvalds * handle that here by just cleaning them. 17601da177e4SLinus Torvalds */ 17611da177e4SLinus Torvalds 176254b21a79SAndrew Morton block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 17631da177e4SLinus Torvalds head = page_buffers(page); 17641da177e4SLinus Torvalds bh = head; 17651da177e4SLinus Torvalds 17661da177e4SLinus Torvalds /* 17671da177e4SLinus Torvalds * Get all the dirty buffers mapped to disk addresses and 17681da177e4SLinus Torvalds * handle any aliases from the underlying blockdev's mapping. 17691da177e4SLinus Torvalds */ 17701da177e4SLinus Torvalds do { 17711da177e4SLinus Torvalds if (block > last_block) { 17721da177e4SLinus Torvalds /* 17731da177e4SLinus Torvalds * mapped buffers outside i_size will occur, because 17741da177e4SLinus Torvalds * this page can be outside i_size when there is a 17751da177e4SLinus Torvalds * truncate in progress. 17761da177e4SLinus Torvalds */ 17771da177e4SLinus Torvalds /* 17781da177e4SLinus Torvalds * The buffer was zeroed by block_write_full_page() 17791da177e4SLinus Torvalds */ 17801da177e4SLinus Torvalds clear_buffer_dirty(bh); 17811da177e4SLinus Torvalds set_buffer_uptodate(bh); 17821da177e4SLinus Torvalds } else if (!buffer_mapped(bh) && buffer_dirty(bh)) { 1783b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 17841da177e4SLinus Torvalds err = get_block(inode, block, bh, 1); 17851da177e4SLinus Torvalds if (err) 17861da177e4SLinus Torvalds goto recover; 17871da177e4SLinus Torvalds if (buffer_new(bh)) { 17881da177e4SLinus Torvalds /* blockdev mappings never come here */ 17891da177e4SLinus Torvalds clear_buffer_new(bh); 17901da177e4SLinus Torvalds unmap_underlying_metadata(bh->b_bdev, 17911da177e4SLinus Torvalds bh->b_blocknr); 17921da177e4SLinus Torvalds } 17931da177e4SLinus Torvalds } 17941da177e4SLinus Torvalds bh = bh->b_this_page; 17951da177e4SLinus Torvalds block++; 17961da177e4SLinus Torvalds } while (bh != head); 17971da177e4SLinus Torvalds 17981da177e4SLinus Torvalds do { 17991da177e4SLinus Torvalds if (!buffer_mapped(bh)) 18001da177e4SLinus Torvalds continue; 18011da177e4SLinus Torvalds /* 18021da177e4SLinus Torvalds * If it's a fully non-blocking write attempt and we cannot 18031da177e4SLinus Torvalds * lock the buffer then redirty the page. Note that this can 18041da177e4SLinus Torvalds * potentially cause a busy-wait loop from pdflush and kswapd 18051da177e4SLinus Torvalds * activity, but those code paths have their own higher-level 18061da177e4SLinus Torvalds * throttling. 18071da177e4SLinus Torvalds */ 18081da177e4SLinus Torvalds if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) { 18091da177e4SLinus Torvalds lock_buffer(bh); 18101da177e4SLinus Torvalds } else if (test_set_buffer_locked(bh)) { 18111da177e4SLinus Torvalds redirty_page_for_writepage(wbc, page); 18121da177e4SLinus Torvalds continue; 18131da177e4SLinus Torvalds } 18141da177e4SLinus Torvalds if (test_clear_buffer_dirty(bh)) { 18151da177e4SLinus Torvalds mark_buffer_async_write(bh); 18161da177e4SLinus Torvalds } else { 18171da177e4SLinus Torvalds unlock_buffer(bh); 18181da177e4SLinus Torvalds } 18191da177e4SLinus Torvalds } while ((bh = bh->b_this_page) != head); 18201da177e4SLinus Torvalds 18211da177e4SLinus Torvalds /* 18221da177e4SLinus Torvalds * The page and its buffers are protected by PageWriteback(), so we can 18231da177e4SLinus Torvalds * drop the bh refcounts early. 18241da177e4SLinus Torvalds */ 18251da177e4SLinus Torvalds BUG_ON(PageWriteback(page)); 18261da177e4SLinus Torvalds set_page_writeback(page); 18271da177e4SLinus Torvalds 18281da177e4SLinus Torvalds do { 18291da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 18301da177e4SLinus Torvalds if (buffer_async_write(bh)) { 18311da177e4SLinus Torvalds submit_bh(WRITE, bh); 18321da177e4SLinus Torvalds nr_underway++; 1833ad576e63SNick Piggin } 18341da177e4SLinus Torvalds bh = next; 18351da177e4SLinus Torvalds } while (bh != head); 183605937baaSAndrew Morton unlock_page(page); 18371da177e4SLinus Torvalds 18381da177e4SLinus Torvalds err = 0; 18391da177e4SLinus Torvalds done: 18401da177e4SLinus Torvalds if (nr_underway == 0) { 18411da177e4SLinus Torvalds /* 18421da177e4SLinus Torvalds * The page was marked dirty, but the buffers were 18431da177e4SLinus Torvalds * clean. Someone wrote them back by hand with 18441da177e4SLinus Torvalds * ll_rw_block/submit_bh. A rare case. 18451da177e4SLinus Torvalds */ 18461da177e4SLinus Torvalds int uptodate = 1; 18471da177e4SLinus Torvalds do { 18481da177e4SLinus Torvalds if (!buffer_uptodate(bh)) { 18491da177e4SLinus Torvalds uptodate = 0; 18501da177e4SLinus Torvalds break; 18511da177e4SLinus Torvalds } 18521da177e4SLinus Torvalds bh = bh->b_this_page; 18531da177e4SLinus Torvalds } while (bh != head); 18541da177e4SLinus Torvalds if (uptodate) 18551da177e4SLinus Torvalds SetPageUptodate(page); 18561da177e4SLinus Torvalds end_page_writeback(page); 18571da177e4SLinus Torvalds /* 18581da177e4SLinus Torvalds * The page and buffer_heads can be released at any time from 18591da177e4SLinus Torvalds * here on. 18601da177e4SLinus Torvalds */ 18611da177e4SLinus Torvalds wbc->pages_skipped++; /* We didn't write this page */ 18621da177e4SLinus Torvalds } 18631da177e4SLinus Torvalds return err; 18641da177e4SLinus Torvalds 18651da177e4SLinus Torvalds recover: 18661da177e4SLinus Torvalds /* 18671da177e4SLinus Torvalds * ENOSPC, or some other error. We may already have added some 18681da177e4SLinus Torvalds * blocks to the file, so we need to write these out to avoid 18691da177e4SLinus Torvalds * exposing stale data. 18701da177e4SLinus Torvalds * The page is currently locked and not marked for writeback 18711da177e4SLinus Torvalds */ 18721da177e4SLinus Torvalds bh = head; 18731da177e4SLinus Torvalds /* Recovery: lock and submit the mapped buffers */ 18741da177e4SLinus Torvalds do { 18751da177e4SLinus Torvalds if (buffer_mapped(bh) && buffer_dirty(bh)) { 18761da177e4SLinus Torvalds lock_buffer(bh); 18771da177e4SLinus Torvalds mark_buffer_async_write(bh); 18781da177e4SLinus Torvalds } else { 18791da177e4SLinus Torvalds /* 18801da177e4SLinus Torvalds * The buffer may have been set dirty during 18811da177e4SLinus Torvalds * attachment to a dirty page. 18821da177e4SLinus Torvalds */ 18831da177e4SLinus Torvalds clear_buffer_dirty(bh); 18841da177e4SLinus Torvalds } 18851da177e4SLinus Torvalds } while ((bh = bh->b_this_page) != head); 18861da177e4SLinus Torvalds SetPageError(page); 18871da177e4SLinus Torvalds BUG_ON(PageWriteback(page)); 18881da177e4SLinus Torvalds set_page_writeback(page); 18891da177e4SLinus Torvalds unlock_page(page); 18901da177e4SLinus Torvalds do { 18911da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 18921da177e4SLinus Torvalds if (buffer_async_write(bh)) { 18931da177e4SLinus Torvalds clear_buffer_dirty(bh); 18941da177e4SLinus Torvalds submit_bh(WRITE, bh); 18951da177e4SLinus Torvalds nr_underway++; 1896ad576e63SNick Piggin } 18971da177e4SLinus Torvalds bh = next; 18981da177e4SLinus Torvalds } while (bh != head); 18991da177e4SLinus Torvalds goto done; 19001da177e4SLinus Torvalds } 19011da177e4SLinus Torvalds 19021da177e4SLinus Torvalds static int __block_prepare_write(struct inode *inode, struct page *page, 19031da177e4SLinus Torvalds unsigned from, unsigned to, get_block_t *get_block) 19041da177e4SLinus Torvalds { 19051da177e4SLinus Torvalds unsigned block_start, block_end; 19061da177e4SLinus Torvalds sector_t block; 19071da177e4SLinus Torvalds int err = 0; 19081da177e4SLinus Torvalds unsigned blocksize, bbits; 19091da177e4SLinus Torvalds struct buffer_head *bh, *head, *wait[2], **wait_bh=wait; 19101da177e4SLinus Torvalds 19111da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 19121da177e4SLinus Torvalds BUG_ON(from > PAGE_CACHE_SIZE); 19131da177e4SLinus Torvalds BUG_ON(to > PAGE_CACHE_SIZE); 19141da177e4SLinus Torvalds BUG_ON(from > to); 19151da177e4SLinus Torvalds 19161da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 19171da177e4SLinus Torvalds if (!page_has_buffers(page)) 19181da177e4SLinus Torvalds create_empty_buffers(page, blocksize, 0); 19191da177e4SLinus Torvalds head = page_buffers(page); 19201da177e4SLinus Torvalds 19211da177e4SLinus Torvalds bbits = inode->i_blkbits; 19221da177e4SLinus Torvalds block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits); 19231da177e4SLinus Torvalds 19241da177e4SLinus Torvalds for(bh = head, block_start = 0; bh != head || !block_start; 19251da177e4SLinus Torvalds block++, block_start=block_end, bh = bh->b_this_page) { 19261da177e4SLinus Torvalds block_end = block_start + blocksize; 19271da177e4SLinus Torvalds if (block_end <= from || block_start >= to) { 19281da177e4SLinus Torvalds if (PageUptodate(page)) { 19291da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 19301da177e4SLinus Torvalds set_buffer_uptodate(bh); 19311da177e4SLinus Torvalds } 19321da177e4SLinus Torvalds continue; 19331da177e4SLinus Torvalds } 19341da177e4SLinus Torvalds if (buffer_new(bh)) 19351da177e4SLinus Torvalds clear_buffer_new(bh); 19361da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 1937b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 19381da177e4SLinus Torvalds err = get_block(inode, block, bh, 1); 19391da177e4SLinus Torvalds if (err) 1940f3ddbdc6SNick Piggin break; 19411da177e4SLinus Torvalds if (buffer_new(bh)) { 19421da177e4SLinus Torvalds unmap_underlying_metadata(bh->b_bdev, 19431da177e4SLinus Torvalds bh->b_blocknr); 19441da177e4SLinus Torvalds if (PageUptodate(page)) { 19451da177e4SLinus Torvalds set_buffer_uptodate(bh); 19461da177e4SLinus Torvalds continue; 19471da177e4SLinus Torvalds } 19481da177e4SLinus Torvalds if (block_end > to || block_start < from) { 19491da177e4SLinus Torvalds void *kaddr; 19501da177e4SLinus Torvalds 19511da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 19521da177e4SLinus Torvalds if (block_end > to) 19531da177e4SLinus Torvalds memset(kaddr+to, 0, 19541da177e4SLinus Torvalds block_end-to); 19551da177e4SLinus Torvalds if (block_start < from) 19561da177e4SLinus Torvalds memset(kaddr+block_start, 19571da177e4SLinus Torvalds 0, from-block_start); 19581da177e4SLinus Torvalds flush_dcache_page(page); 19591da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 19601da177e4SLinus Torvalds } 19611da177e4SLinus Torvalds continue; 19621da177e4SLinus Torvalds } 19631da177e4SLinus Torvalds } 19641da177e4SLinus Torvalds if (PageUptodate(page)) { 19651da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 19661da177e4SLinus Torvalds set_buffer_uptodate(bh); 19671da177e4SLinus Torvalds continue; 19681da177e4SLinus Torvalds } 19691da177e4SLinus Torvalds if (!buffer_uptodate(bh) && !buffer_delay(bh) && 19701da177e4SLinus Torvalds (block_start < from || block_end > to)) { 19711da177e4SLinus Torvalds ll_rw_block(READ, 1, &bh); 19721da177e4SLinus Torvalds *wait_bh++=bh; 19731da177e4SLinus Torvalds } 19741da177e4SLinus Torvalds } 19751da177e4SLinus Torvalds /* 19761da177e4SLinus Torvalds * If we issued read requests - let them complete. 19771da177e4SLinus Torvalds */ 19781da177e4SLinus Torvalds while(wait_bh > wait) { 19791da177e4SLinus Torvalds wait_on_buffer(*--wait_bh); 19801da177e4SLinus Torvalds if (!buffer_uptodate(*wait_bh)) 1981f3ddbdc6SNick Piggin err = -EIO; 19821da177e4SLinus Torvalds } 1983152becd2SAnton Altaparmakov if (!err) { 1984152becd2SAnton Altaparmakov bh = head; 1985152becd2SAnton Altaparmakov do { 1986152becd2SAnton Altaparmakov if (buffer_new(bh)) 1987152becd2SAnton Altaparmakov clear_buffer_new(bh); 1988152becd2SAnton Altaparmakov } while ((bh = bh->b_this_page) != head); 1989152becd2SAnton Altaparmakov return 0; 1990152becd2SAnton Altaparmakov } 1991f3ddbdc6SNick Piggin /* Error case: */ 19921da177e4SLinus Torvalds /* 19931da177e4SLinus Torvalds * Zero out any newly allocated blocks to avoid exposing stale 19941da177e4SLinus Torvalds * data. If BH_New is set, we know that the block was newly 19951da177e4SLinus Torvalds * allocated in the above loop. 19961da177e4SLinus Torvalds */ 19971da177e4SLinus Torvalds bh = head; 19981da177e4SLinus Torvalds block_start = 0; 19991da177e4SLinus Torvalds do { 20001da177e4SLinus Torvalds block_end = block_start+blocksize; 20011da177e4SLinus Torvalds if (block_end <= from) 20021da177e4SLinus Torvalds goto next_bh; 20031da177e4SLinus Torvalds if (block_start >= to) 20041da177e4SLinus Torvalds break; 20051da177e4SLinus Torvalds if (buffer_new(bh)) { 20061da177e4SLinus Torvalds void *kaddr; 20071da177e4SLinus Torvalds 20081da177e4SLinus Torvalds clear_buffer_new(bh); 20091da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 20101da177e4SLinus Torvalds memset(kaddr+block_start, 0, bh->b_size); 20111da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 20121da177e4SLinus Torvalds set_buffer_uptodate(bh); 20131da177e4SLinus Torvalds mark_buffer_dirty(bh); 20141da177e4SLinus Torvalds } 20151da177e4SLinus Torvalds next_bh: 20161da177e4SLinus Torvalds block_start = block_end; 20171da177e4SLinus Torvalds bh = bh->b_this_page; 20181da177e4SLinus Torvalds } while (bh != head); 20191da177e4SLinus Torvalds return err; 20201da177e4SLinus Torvalds } 20211da177e4SLinus Torvalds 20221da177e4SLinus Torvalds static int __block_commit_write(struct inode *inode, struct page *page, 20231da177e4SLinus Torvalds unsigned from, unsigned to) 20241da177e4SLinus Torvalds { 20251da177e4SLinus Torvalds unsigned block_start, block_end; 20261da177e4SLinus Torvalds int partial = 0; 20271da177e4SLinus Torvalds unsigned blocksize; 20281da177e4SLinus Torvalds struct buffer_head *bh, *head; 20291da177e4SLinus Torvalds 20301da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 20311da177e4SLinus Torvalds 20321da177e4SLinus Torvalds for(bh = head = page_buffers(page), block_start = 0; 20331da177e4SLinus Torvalds bh != head || !block_start; 20341da177e4SLinus Torvalds block_start=block_end, bh = bh->b_this_page) { 20351da177e4SLinus Torvalds block_end = block_start + blocksize; 20361da177e4SLinus Torvalds if (block_end <= from || block_start >= to) { 20371da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 20381da177e4SLinus Torvalds partial = 1; 20391da177e4SLinus Torvalds } else { 20401da177e4SLinus Torvalds set_buffer_uptodate(bh); 20411da177e4SLinus Torvalds mark_buffer_dirty(bh); 20421da177e4SLinus Torvalds } 20431da177e4SLinus Torvalds } 20441da177e4SLinus Torvalds 20451da177e4SLinus Torvalds /* 20461da177e4SLinus Torvalds * If this is a partial write which happened to make all buffers 20471da177e4SLinus Torvalds * uptodate then we can optimize away a bogus readpage() for 20481da177e4SLinus Torvalds * the next read(). Here we 'discover' whether the page went 20491da177e4SLinus Torvalds * uptodate as a result of this (potentially partial) write. 20501da177e4SLinus Torvalds */ 20511da177e4SLinus Torvalds if (!partial) 20521da177e4SLinus Torvalds SetPageUptodate(page); 20531da177e4SLinus Torvalds return 0; 20541da177e4SLinus Torvalds } 20551da177e4SLinus Torvalds 20561da177e4SLinus Torvalds /* 20571da177e4SLinus Torvalds * Generic "read page" function for block devices that have the normal 20581da177e4SLinus Torvalds * get_block functionality. This is most of the block device filesystems. 20591da177e4SLinus Torvalds * Reads the page asynchronously --- the unlock_buffer() and 20601da177e4SLinus Torvalds * set/clear_buffer_uptodate() functions propagate buffer state into the 20611da177e4SLinus Torvalds * page struct once IO has completed. 20621da177e4SLinus Torvalds */ 20631da177e4SLinus Torvalds int block_read_full_page(struct page *page, get_block_t *get_block) 20641da177e4SLinus Torvalds { 20651da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 20661da177e4SLinus Torvalds sector_t iblock, lblock; 20671da177e4SLinus Torvalds struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE]; 20681da177e4SLinus Torvalds unsigned int blocksize; 20691da177e4SLinus Torvalds int nr, i; 20701da177e4SLinus Torvalds int fully_mapped = 1; 20711da177e4SLinus Torvalds 2072cd7619d6SMatt Mackall BUG_ON(!PageLocked(page)); 20731da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 20741da177e4SLinus Torvalds if (!page_has_buffers(page)) 20751da177e4SLinus Torvalds create_empty_buffers(page, blocksize, 0); 20761da177e4SLinus Torvalds head = page_buffers(page); 20771da177e4SLinus Torvalds 20781da177e4SLinus Torvalds iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 20791da177e4SLinus Torvalds lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits; 20801da177e4SLinus Torvalds bh = head; 20811da177e4SLinus Torvalds nr = 0; 20821da177e4SLinus Torvalds i = 0; 20831da177e4SLinus Torvalds 20841da177e4SLinus Torvalds do { 20851da177e4SLinus Torvalds if (buffer_uptodate(bh)) 20861da177e4SLinus Torvalds continue; 20871da177e4SLinus Torvalds 20881da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 2089c64610baSAndrew Morton int err = 0; 2090c64610baSAndrew Morton 20911da177e4SLinus Torvalds fully_mapped = 0; 20921da177e4SLinus Torvalds if (iblock < lblock) { 2093b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 2094c64610baSAndrew Morton err = get_block(inode, iblock, bh, 0); 2095c64610baSAndrew Morton if (err) 20961da177e4SLinus Torvalds SetPageError(page); 20971da177e4SLinus Torvalds } 20981da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 20991da177e4SLinus Torvalds void *kaddr = kmap_atomic(page, KM_USER0); 21001da177e4SLinus Torvalds memset(kaddr + i * blocksize, 0, blocksize); 21011da177e4SLinus Torvalds flush_dcache_page(page); 21021da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 2103c64610baSAndrew Morton if (!err) 21041da177e4SLinus Torvalds set_buffer_uptodate(bh); 21051da177e4SLinus Torvalds continue; 21061da177e4SLinus Torvalds } 21071da177e4SLinus Torvalds /* 21081da177e4SLinus Torvalds * get_block() might have updated the buffer 21091da177e4SLinus Torvalds * synchronously 21101da177e4SLinus Torvalds */ 21111da177e4SLinus Torvalds if (buffer_uptodate(bh)) 21121da177e4SLinus Torvalds continue; 21131da177e4SLinus Torvalds } 21141da177e4SLinus Torvalds arr[nr++] = bh; 21151da177e4SLinus Torvalds } while (i++, iblock++, (bh = bh->b_this_page) != head); 21161da177e4SLinus Torvalds 21171da177e4SLinus Torvalds if (fully_mapped) 21181da177e4SLinus Torvalds SetPageMappedToDisk(page); 21191da177e4SLinus Torvalds 21201da177e4SLinus Torvalds if (!nr) { 21211da177e4SLinus Torvalds /* 21221da177e4SLinus Torvalds * All buffers are uptodate - we can set the page uptodate 21231da177e4SLinus Torvalds * as well. But not if get_block() returned an error. 21241da177e4SLinus Torvalds */ 21251da177e4SLinus Torvalds if (!PageError(page)) 21261da177e4SLinus Torvalds SetPageUptodate(page); 21271da177e4SLinus Torvalds unlock_page(page); 21281da177e4SLinus Torvalds return 0; 21291da177e4SLinus Torvalds } 21301da177e4SLinus Torvalds 21311da177e4SLinus Torvalds /* Stage two: lock the buffers */ 21321da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 21331da177e4SLinus Torvalds bh = arr[i]; 21341da177e4SLinus Torvalds lock_buffer(bh); 21351da177e4SLinus Torvalds mark_buffer_async_read(bh); 21361da177e4SLinus Torvalds } 21371da177e4SLinus Torvalds 21381da177e4SLinus Torvalds /* 21391da177e4SLinus Torvalds * Stage 3: start the IO. Check for uptodateness 21401da177e4SLinus Torvalds * inside the buffer lock in case another process reading 21411da177e4SLinus Torvalds * the underlying blockdev brought it uptodate (the sct fix). 21421da177e4SLinus Torvalds */ 21431da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 21441da177e4SLinus Torvalds bh = arr[i]; 21451da177e4SLinus Torvalds if (buffer_uptodate(bh)) 21461da177e4SLinus Torvalds end_buffer_async_read(bh, 1); 21471da177e4SLinus Torvalds else 21481da177e4SLinus Torvalds submit_bh(READ, bh); 21491da177e4SLinus Torvalds } 21501da177e4SLinus Torvalds return 0; 21511da177e4SLinus Torvalds } 21521da177e4SLinus Torvalds 21531da177e4SLinus Torvalds /* utility function for filesystems that need to do work on expanding 21541da177e4SLinus Torvalds * truncates. Uses prepare/commit_write to allow the filesystem to 21551da177e4SLinus Torvalds * deal with the hole. 21561da177e4SLinus Torvalds */ 215705eb0b51SOGAWA Hirofumi static int __generic_cont_expand(struct inode *inode, loff_t size, 215805eb0b51SOGAWA Hirofumi pgoff_t index, unsigned int offset) 21591da177e4SLinus Torvalds { 21601da177e4SLinus Torvalds struct address_space *mapping = inode->i_mapping; 21611da177e4SLinus Torvalds struct page *page; 216205eb0b51SOGAWA Hirofumi unsigned long limit; 21631da177e4SLinus Torvalds int err; 21641da177e4SLinus Torvalds 21651da177e4SLinus Torvalds err = -EFBIG; 21661da177e4SLinus Torvalds limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; 21671da177e4SLinus Torvalds if (limit != RLIM_INFINITY && size > (loff_t)limit) { 21681da177e4SLinus Torvalds send_sig(SIGXFSZ, current, 0); 21691da177e4SLinus Torvalds goto out; 21701da177e4SLinus Torvalds } 21711da177e4SLinus Torvalds if (size > inode->i_sb->s_maxbytes) 21721da177e4SLinus Torvalds goto out; 21731da177e4SLinus Torvalds 217405eb0b51SOGAWA Hirofumi err = -ENOMEM; 217505eb0b51SOGAWA Hirofumi page = grab_cache_page(mapping, index); 217605eb0b51SOGAWA Hirofumi if (!page) 217705eb0b51SOGAWA Hirofumi goto out; 217805eb0b51SOGAWA Hirofumi err = mapping->a_ops->prepare_write(NULL, page, offset, offset); 217905eb0b51SOGAWA Hirofumi if (err) { 218005eb0b51SOGAWA Hirofumi /* 218105eb0b51SOGAWA Hirofumi * ->prepare_write() may have instantiated a few blocks 218205eb0b51SOGAWA Hirofumi * outside i_size. Trim these off again. 218305eb0b51SOGAWA Hirofumi */ 218405eb0b51SOGAWA Hirofumi unlock_page(page); 218505eb0b51SOGAWA Hirofumi page_cache_release(page); 218605eb0b51SOGAWA Hirofumi vmtruncate(inode, inode->i_size); 218705eb0b51SOGAWA Hirofumi goto out; 218805eb0b51SOGAWA Hirofumi } 218905eb0b51SOGAWA Hirofumi 219005eb0b51SOGAWA Hirofumi err = mapping->a_ops->commit_write(NULL, page, offset, offset); 219105eb0b51SOGAWA Hirofumi 219205eb0b51SOGAWA Hirofumi unlock_page(page); 219305eb0b51SOGAWA Hirofumi page_cache_release(page); 219405eb0b51SOGAWA Hirofumi if (err > 0) 219505eb0b51SOGAWA Hirofumi err = 0; 219605eb0b51SOGAWA Hirofumi out: 219705eb0b51SOGAWA Hirofumi return err; 219805eb0b51SOGAWA Hirofumi } 219905eb0b51SOGAWA Hirofumi 220005eb0b51SOGAWA Hirofumi int generic_cont_expand(struct inode *inode, loff_t size) 220105eb0b51SOGAWA Hirofumi { 220205eb0b51SOGAWA Hirofumi pgoff_t index; 220305eb0b51SOGAWA Hirofumi unsigned int offset; 220405eb0b51SOGAWA Hirofumi 22051da177e4SLinus Torvalds offset = (size & (PAGE_CACHE_SIZE - 1)); /* Within page */ 22061da177e4SLinus Torvalds 22071da177e4SLinus Torvalds /* ugh. in prepare/commit_write, if from==to==start of block, we 22081da177e4SLinus Torvalds ** skip the prepare. make sure we never send an offset for the start 22091da177e4SLinus Torvalds ** of a block 22101da177e4SLinus Torvalds */ 22111da177e4SLinus Torvalds if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) { 221205eb0b51SOGAWA Hirofumi /* caller must handle this extra byte. */ 22131da177e4SLinus Torvalds offset++; 22141da177e4SLinus Torvalds } 22151da177e4SLinus Torvalds index = size >> PAGE_CACHE_SHIFT; 221605eb0b51SOGAWA Hirofumi 221705eb0b51SOGAWA Hirofumi return __generic_cont_expand(inode, size, index, offset); 22181da177e4SLinus Torvalds } 221905eb0b51SOGAWA Hirofumi 222005eb0b51SOGAWA Hirofumi int generic_cont_expand_simple(struct inode *inode, loff_t size) 222105eb0b51SOGAWA Hirofumi { 222205eb0b51SOGAWA Hirofumi loff_t pos = size - 1; 222305eb0b51SOGAWA Hirofumi pgoff_t index = pos >> PAGE_CACHE_SHIFT; 222405eb0b51SOGAWA Hirofumi unsigned int offset = (pos & (PAGE_CACHE_SIZE - 1)) + 1; 222505eb0b51SOGAWA Hirofumi 222605eb0b51SOGAWA Hirofumi /* prepare/commit_write can handle even if from==to==start of block. */ 222705eb0b51SOGAWA Hirofumi return __generic_cont_expand(inode, size, index, offset); 22281da177e4SLinus Torvalds } 22291da177e4SLinus Torvalds 22301da177e4SLinus Torvalds /* 22311da177e4SLinus Torvalds * For moronic filesystems that do not allow holes in file. 22321da177e4SLinus Torvalds * We may have to extend the file. 22331da177e4SLinus Torvalds */ 22341da177e4SLinus Torvalds 22351da177e4SLinus Torvalds int cont_prepare_write(struct page *page, unsigned offset, 22361da177e4SLinus Torvalds unsigned to, get_block_t *get_block, loff_t *bytes) 22371da177e4SLinus Torvalds { 22381da177e4SLinus Torvalds struct address_space *mapping = page->mapping; 22391da177e4SLinus Torvalds struct inode *inode = mapping->host; 22401da177e4SLinus Torvalds struct page *new_page; 22411da177e4SLinus Torvalds pgoff_t pgpos; 22421da177e4SLinus Torvalds long status; 22431da177e4SLinus Torvalds unsigned zerofrom; 22441da177e4SLinus Torvalds unsigned blocksize = 1 << inode->i_blkbits; 22451da177e4SLinus Torvalds void *kaddr; 22461da177e4SLinus Torvalds 22471da177e4SLinus Torvalds while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) { 22481da177e4SLinus Torvalds status = -ENOMEM; 22491da177e4SLinus Torvalds new_page = grab_cache_page(mapping, pgpos); 22501da177e4SLinus Torvalds if (!new_page) 22511da177e4SLinus Torvalds goto out; 22521da177e4SLinus Torvalds /* we might sleep */ 22531da177e4SLinus Torvalds if (*bytes>>PAGE_CACHE_SHIFT != pgpos) { 22541da177e4SLinus Torvalds unlock_page(new_page); 22551da177e4SLinus Torvalds page_cache_release(new_page); 22561da177e4SLinus Torvalds continue; 22571da177e4SLinus Torvalds } 22581da177e4SLinus Torvalds zerofrom = *bytes & ~PAGE_CACHE_MASK; 22591da177e4SLinus Torvalds if (zerofrom & (blocksize-1)) { 22601da177e4SLinus Torvalds *bytes |= (blocksize-1); 22611da177e4SLinus Torvalds (*bytes)++; 22621da177e4SLinus Torvalds } 22631da177e4SLinus Torvalds status = __block_prepare_write(inode, new_page, zerofrom, 22641da177e4SLinus Torvalds PAGE_CACHE_SIZE, get_block); 22651da177e4SLinus Torvalds if (status) 22661da177e4SLinus Torvalds goto out_unmap; 22671da177e4SLinus Torvalds kaddr = kmap_atomic(new_page, KM_USER0); 22681da177e4SLinus Torvalds memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom); 22691da177e4SLinus Torvalds flush_dcache_page(new_page); 22701da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 22711da177e4SLinus Torvalds generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE); 22721da177e4SLinus Torvalds unlock_page(new_page); 22731da177e4SLinus Torvalds page_cache_release(new_page); 22741da177e4SLinus Torvalds } 22751da177e4SLinus Torvalds 22761da177e4SLinus Torvalds if (page->index < pgpos) { 22771da177e4SLinus Torvalds /* completely inside the area */ 22781da177e4SLinus Torvalds zerofrom = offset; 22791da177e4SLinus Torvalds } else { 22801da177e4SLinus Torvalds /* page covers the boundary, find the boundary offset */ 22811da177e4SLinus Torvalds zerofrom = *bytes & ~PAGE_CACHE_MASK; 22821da177e4SLinus Torvalds 22831da177e4SLinus Torvalds /* if we will expand the thing last block will be filled */ 22841da177e4SLinus Torvalds if (to > zerofrom && (zerofrom & (blocksize-1))) { 22851da177e4SLinus Torvalds *bytes |= (blocksize-1); 22861da177e4SLinus Torvalds (*bytes)++; 22871da177e4SLinus Torvalds } 22881da177e4SLinus Torvalds 22891da177e4SLinus Torvalds /* starting below the boundary? Nothing to zero out */ 22901da177e4SLinus Torvalds if (offset <= zerofrom) 22911da177e4SLinus Torvalds zerofrom = offset; 22921da177e4SLinus Torvalds } 22931da177e4SLinus Torvalds status = __block_prepare_write(inode, page, zerofrom, to, get_block); 22941da177e4SLinus Torvalds if (status) 22951da177e4SLinus Torvalds goto out1; 22961da177e4SLinus Torvalds if (zerofrom < offset) { 22971da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 22981da177e4SLinus Torvalds memset(kaddr+zerofrom, 0, offset-zerofrom); 22991da177e4SLinus Torvalds flush_dcache_page(page); 23001da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 23011da177e4SLinus Torvalds __block_commit_write(inode, page, zerofrom, offset); 23021da177e4SLinus Torvalds } 23031da177e4SLinus Torvalds return 0; 23041da177e4SLinus Torvalds out1: 23051da177e4SLinus Torvalds ClearPageUptodate(page); 23061da177e4SLinus Torvalds return status; 23071da177e4SLinus Torvalds 23081da177e4SLinus Torvalds out_unmap: 23091da177e4SLinus Torvalds ClearPageUptodate(new_page); 23101da177e4SLinus Torvalds unlock_page(new_page); 23111da177e4SLinus Torvalds page_cache_release(new_page); 23121da177e4SLinus Torvalds out: 23131da177e4SLinus Torvalds return status; 23141da177e4SLinus Torvalds } 23151da177e4SLinus Torvalds 23161da177e4SLinus Torvalds int block_prepare_write(struct page *page, unsigned from, unsigned to, 23171da177e4SLinus Torvalds get_block_t *get_block) 23181da177e4SLinus Torvalds { 23191da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 23201da177e4SLinus Torvalds int err = __block_prepare_write(inode, page, from, to, get_block); 23211da177e4SLinus Torvalds if (err) 23221da177e4SLinus Torvalds ClearPageUptodate(page); 23231da177e4SLinus Torvalds return err; 23241da177e4SLinus Torvalds } 23251da177e4SLinus Torvalds 23261da177e4SLinus Torvalds int block_commit_write(struct page *page, unsigned from, unsigned to) 23271da177e4SLinus Torvalds { 23281da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 23291da177e4SLinus Torvalds __block_commit_write(inode,page,from,to); 23301da177e4SLinus Torvalds return 0; 23311da177e4SLinus Torvalds } 23321da177e4SLinus Torvalds 23331da177e4SLinus Torvalds int generic_commit_write(struct file *file, struct page *page, 23341da177e4SLinus Torvalds unsigned from, unsigned to) 23351da177e4SLinus Torvalds { 23361da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 23371da177e4SLinus Torvalds loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to; 23381da177e4SLinus Torvalds __block_commit_write(inode,page,from,to); 23391da177e4SLinus Torvalds /* 23401da177e4SLinus Torvalds * No need to use i_size_read() here, the i_size 23411b1dcc1bSJes Sorensen * cannot change under us because we hold i_mutex. 23421da177e4SLinus Torvalds */ 23431da177e4SLinus Torvalds if (pos > inode->i_size) { 23441da177e4SLinus Torvalds i_size_write(inode, pos); 23451da177e4SLinus Torvalds mark_inode_dirty(inode); 23461da177e4SLinus Torvalds } 23471da177e4SLinus Torvalds return 0; 23481da177e4SLinus Torvalds } 23491da177e4SLinus Torvalds 23501da177e4SLinus Torvalds 23511da177e4SLinus Torvalds /* 23521da177e4SLinus Torvalds * nobh_prepare_write()'s prereads are special: the buffer_heads are freed 23531da177e4SLinus Torvalds * immediately, while under the page lock. So it needs a special end_io 23541da177e4SLinus Torvalds * handler which does not touch the bh after unlocking it. 23551da177e4SLinus Torvalds * 23561da177e4SLinus Torvalds * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but 23571da177e4SLinus Torvalds * a race there is benign: unlock_buffer() only use the bh's address for 23581da177e4SLinus Torvalds * hashing after unlocking the buffer, so it doesn't actually touch the bh 23591da177e4SLinus Torvalds * itself. 23601da177e4SLinus Torvalds */ 23611da177e4SLinus Torvalds static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate) 23621da177e4SLinus Torvalds { 23631da177e4SLinus Torvalds if (uptodate) { 23641da177e4SLinus Torvalds set_buffer_uptodate(bh); 23651da177e4SLinus Torvalds } else { 23661da177e4SLinus Torvalds /* This happens, due to failed READA attempts. */ 23671da177e4SLinus Torvalds clear_buffer_uptodate(bh); 23681da177e4SLinus Torvalds } 23691da177e4SLinus Torvalds unlock_buffer(bh); 23701da177e4SLinus Torvalds } 23711da177e4SLinus Torvalds 23721da177e4SLinus Torvalds /* 23731da177e4SLinus Torvalds * On entry, the page is fully not uptodate. 23741da177e4SLinus Torvalds * On exit the page is fully uptodate in the areas outside (from,to) 23751da177e4SLinus Torvalds */ 23761da177e4SLinus Torvalds int nobh_prepare_write(struct page *page, unsigned from, unsigned to, 23771da177e4SLinus Torvalds get_block_t *get_block) 23781da177e4SLinus Torvalds { 23791da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 23801da177e4SLinus Torvalds const unsigned blkbits = inode->i_blkbits; 23811da177e4SLinus Torvalds const unsigned blocksize = 1 << blkbits; 23821da177e4SLinus Torvalds struct buffer_head map_bh; 23831da177e4SLinus Torvalds struct buffer_head *read_bh[MAX_BUF_PER_PAGE]; 23841da177e4SLinus Torvalds unsigned block_in_page; 23851da177e4SLinus Torvalds unsigned block_start; 23861da177e4SLinus Torvalds sector_t block_in_file; 23871da177e4SLinus Torvalds char *kaddr; 23881da177e4SLinus Torvalds int nr_reads = 0; 23891da177e4SLinus Torvalds int i; 23901da177e4SLinus Torvalds int ret = 0; 23911da177e4SLinus Torvalds int is_mapped_to_disk = 1; 23921da177e4SLinus Torvalds int dirtied_it = 0; 23931da177e4SLinus Torvalds 23941da177e4SLinus Torvalds if (PageMappedToDisk(page)) 23951da177e4SLinus Torvalds return 0; 23961da177e4SLinus Torvalds 23971da177e4SLinus Torvalds block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits); 23981da177e4SLinus Torvalds map_bh.b_page = page; 23991da177e4SLinus Torvalds 24001da177e4SLinus Torvalds /* 24011da177e4SLinus Torvalds * We loop across all blocks in the page, whether or not they are 24021da177e4SLinus Torvalds * part of the affected region. This is so we can discover if the 24031da177e4SLinus Torvalds * page is fully mapped-to-disk. 24041da177e4SLinus Torvalds */ 24051da177e4SLinus Torvalds for (block_start = 0, block_in_page = 0; 24061da177e4SLinus Torvalds block_start < PAGE_CACHE_SIZE; 24071da177e4SLinus Torvalds block_in_page++, block_start += blocksize) { 24081da177e4SLinus Torvalds unsigned block_end = block_start + blocksize; 24091da177e4SLinus Torvalds int create; 24101da177e4SLinus Torvalds 24111da177e4SLinus Torvalds map_bh.b_state = 0; 24121da177e4SLinus Torvalds create = 1; 24131da177e4SLinus Torvalds if (block_start >= to) 24141da177e4SLinus Torvalds create = 0; 2415b0cf2321SBadari Pulavarty map_bh.b_size = blocksize; 24161da177e4SLinus Torvalds ret = get_block(inode, block_in_file + block_in_page, 24171da177e4SLinus Torvalds &map_bh, create); 24181da177e4SLinus Torvalds if (ret) 24191da177e4SLinus Torvalds goto failed; 24201da177e4SLinus Torvalds if (!buffer_mapped(&map_bh)) 24211da177e4SLinus Torvalds is_mapped_to_disk = 0; 24221da177e4SLinus Torvalds if (buffer_new(&map_bh)) 24231da177e4SLinus Torvalds unmap_underlying_metadata(map_bh.b_bdev, 24241da177e4SLinus Torvalds map_bh.b_blocknr); 24251da177e4SLinus Torvalds if (PageUptodate(page)) 24261da177e4SLinus Torvalds continue; 24271da177e4SLinus Torvalds if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) { 24281da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 24291da177e4SLinus Torvalds if (block_start < from) { 24301da177e4SLinus Torvalds memset(kaddr+block_start, 0, from-block_start); 24311da177e4SLinus Torvalds dirtied_it = 1; 24321da177e4SLinus Torvalds } 24331da177e4SLinus Torvalds if (block_end > to) { 24341da177e4SLinus Torvalds memset(kaddr + to, 0, block_end - to); 24351da177e4SLinus Torvalds dirtied_it = 1; 24361da177e4SLinus Torvalds } 24371da177e4SLinus Torvalds flush_dcache_page(page); 24381da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 24391da177e4SLinus Torvalds continue; 24401da177e4SLinus Torvalds } 24411da177e4SLinus Torvalds if (buffer_uptodate(&map_bh)) 24421da177e4SLinus Torvalds continue; /* reiserfs does this */ 24431da177e4SLinus Torvalds if (block_start < from || block_end > to) { 24441da177e4SLinus Torvalds struct buffer_head *bh = alloc_buffer_head(GFP_NOFS); 24451da177e4SLinus Torvalds 24461da177e4SLinus Torvalds if (!bh) { 24471da177e4SLinus Torvalds ret = -ENOMEM; 24481da177e4SLinus Torvalds goto failed; 24491da177e4SLinus Torvalds } 24501da177e4SLinus Torvalds bh->b_state = map_bh.b_state; 24511da177e4SLinus Torvalds atomic_set(&bh->b_count, 0); 24521da177e4SLinus Torvalds bh->b_this_page = NULL; 24531da177e4SLinus Torvalds bh->b_page = page; 24541da177e4SLinus Torvalds bh->b_blocknr = map_bh.b_blocknr; 24551da177e4SLinus Torvalds bh->b_size = blocksize; 24561da177e4SLinus Torvalds bh->b_data = (char *)(long)block_start; 24571da177e4SLinus Torvalds bh->b_bdev = map_bh.b_bdev; 24581da177e4SLinus Torvalds bh->b_private = NULL; 24591da177e4SLinus Torvalds read_bh[nr_reads++] = bh; 24601da177e4SLinus Torvalds } 24611da177e4SLinus Torvalds } 24621da177e4SLinus Torvalds 24631da177e4SLinus Torvalds if (nr_reads) { 24641da177e4SLinus Torvalds struct buffer_head *bh; 24651da177e4SLinus Torvalds 24661da177e4SLinus Torvalds /* 24671da177e4SLinus Torvalds * The page is locked, so these buffers are protected from 24681da177e4SLinus Torvalds * any VM or truncate activity. Hence we don't need to care 24691da177e4SLinus Torvalds * for the buffer_head refcounts. 24701da177e4SLinus Torvalds */ 24711da177e4SLinus Torvalds for (i = 0; i < nr_reads; i++) { 24721da177e4SLinus Torvalds bh = read_bh[i]; 24731da177e4SLinus Torvalds lock_buffer(bh); 24741da177e4SLinus Torvalds bh->b_end_io = end_buffer_read_nobh; 24751da177e4SLinus Torvalds submit_bh(READ, bh); 24761da177e4SLinus Torvalds } 24771da177e4SLinus Torvalds for (i = 0; i < nr_reads; i++) { 24781da177e4SLinus Torvalds bh = read_bh[i]; 24791da177e4SLinus Torvalds wait_on_buffer(bh); 24801da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 24811da177e4SLinus Torvalds ret = -EIO; 24821da177e4SLinus Torvalds free_buffer_head(bh); 24831da177e4SLinus Torvalds read_bh[i] = NULL; 24841da177e4SLinus Torvalds } 24851da177e4SLinus Torvalds if (ret) 24861da177e4SLinus Torvalds goto failed; 24871da177e4SLinus Torvalds } 24881da177e4SLinus Torvalds 24891da177e4SLinus Torvalds if (is_mapped_to_disk) 24901da177e4SLinus Torvalds SetPageMappedToDisk(page); 24911da177e4SLinus Torvalds SetPageUptodate(page); 24921da177e4SLinus Torvalds 24931da177e4SLinus Torvalds /* 24941da177e4SLinus Torvalds * Setting the page dirty here isn't necessary for the prepare_write 24951da177e4SLinus Torvalds * function - commit_write will do that. But if/when this function is 24961da177e4SLinus Torvalds * used within the pagefault handler to ensure that all mmapped pages 24971da177e4SLinus Torvalds * have backing space in the filesystem, we will need to dirty the page 24981da177e4SLinus Torvalds * if its contents were altered. 24991da177e4SLinus Torvalds */ 25001da177e4SLinus Torvalds if (dirtied_it) 25011da177e4SLinus Torvalds set_page_dirty(page); 25021da177e4SLinus Torvalds 25031da177e4SLinus Torvalds return 0; 25041da177e4SLinus Torvalds 25051da177e4SLinus Torvalds failed: 25061da177e4SLinus Torvalds for (i = 0; i < nr_reads; i++) { 25071da177e4SLinus Torvalds if (read_bh[i]) 25081da177e4SLinus Torvalds free_buffer_head(read_bh[i]); 25091da177e4SLinus Torvalds } 25101da177e4SLinus Torvalds 25111da177e4SLinus Torvalds /* 25121da177e4SLinus Torvalds * Error recovery is pretty slack. Clear the page and mark it dirty 25131da177e4SLinus Torvalds * so we'll later zero out any blocks which _were_ allocated. 25141da177e4SLinus Torvalds */ 25151da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 25161da177e4SLinus Torvalds memset(kaddr, 0, PAGE_CACHE_SIZE); 25171da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 25181da177e4SLinus Torvalds SetPageUptodate(page); 25191da177e4SLinus Torvalds set_page_dirty(page); 25201da177e4SLinus Torvalds return ret; 25211da177e4SLinus Torvalds } 25221da177e4SLinus Torvalds EXPORT_SYMBOL(nobh_prepare_write); 25231da177e4SLinus Torvalds 25241da177e4SLinus Torvalds int nobh_commit_write(struct file *file, struct page *page, 25251da177e4SLinus Torvalds unsigned from, unsigned to) 25261da177e4SLinus Torvalds { 25271da177e4SLinus Torvalds struct inode *inode = page->mapping->host; 25281da177e4SLinus Torvalds loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to; 25291da177e4SLinus Torvalds 25301da177e4SLinus Torvalds set_page_dirty(page); 25311da177e4SLinus Torvalds if (pos > inode->i_size) { 25321da177e4SLinus Torvalds i_size_write(inode, pos); 25331da177e4SLinus Torvalds mark_inode_dirty(inode); 25341da177e4SLinus Torvalds } 25351da177e4SLinus Torvalds return 0; 25361da177e4SLinus Torvalds } 25371da177e4SLinus Torvalds EXPORT_SYMBOL(nobh_commit_write); 25381da177e4SLinus Torvalds 25391da177e4SLinus Torvalds /* 25401da177e4SLinus Torvalds * nobh_writepage() - based on block_full_write_page() except 25411da177e4SLinus Torvalds * that it tries to operate without attaching bufferheads to 25421da177e4SLinus Torvalds * the page. 25431da177e4SLinus Torvalds */ 25441da177e4SLinus Torvalds int nobh_writepage(struct page *page, get_block_t *get_block, 25451da177e4SLinus Torvalds struct writeback_control *wbc) 25461da177e4SLinus Torvalds { 25471da177e4SLinus Torvalds struct inode * const inode = page->mapping->host; 25481da177e4SLinus Torvalds loff_t i_size = i_size_read(inode); 25491da177e4SLinus Torvalds const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; 25501da177e4SLinus Torvalds unsigned offset; 25511da177e4SLinus Torvalds void *kaddr; 25521da177e4SLinus Torvalds int ret; 25531da177e4SLinus Torvalds 25541da177e4SLinus Torvalds /* Is the page fully inside i_size? */ 25551da177e4SLinus Torvalds if (page->index < end_index) 25561da177e4SLinus Torvalds goto out; 25571da177e4SLinus Torvalds 25581da177e4SLinus Torvalds /* Is the page fully outside i_size? (truncate in progress) */ 25591da177e4SLinus Torvalds offset = i_size & (PAGE_CACHE_SIZE-1); 25601da177e4SLinus Torvalds if (page->index >= end_index+1 || !offset) { 25611da177e4SLinus Torvalds /* 25621da177e4SLinus Torvalds * The page may have dirty, unmapped buffers. For example, 25631da177e4SLinus Torvalds * they may have been added in ext3_writepage(). Make them 25641da177e4SLinus Torvalds * freeable here, so the page does not leak. 25651da177e4SLinus Torvalds */ 25661da177e4SLinus Torvalds #if 0 25671da177e4SLinus Torvalds /* Not really sure about this - do we need this ? */ 25681da177e4SLinus Torvalds if (page->mapping->a_ops->invalidatepage) 25691da177e4SLinus Torvalds page->mapping->a_ops->invalidatepage(page, offset); 25701da177e4SLinus Torvalds #endif 25711da177e4SLinus Torvalds unlock_page(page); 25721da177e4SLinus Torvalds return 0; /* don't care */ 25731da177e4SLinus Torvalds } 25741da177e4SLinus Torvalds 25751da177e4SLinus Torvalds /* 25761da177e4SLinus Torvalds * The page straddles i_size. It must be zeroed out on each and every 25771da177e4SLinus Torvalds * writepage invocation because it may be mmapped. "A file is mapped 25781da177e4SLinus Torvalds * in multiples of the page size. For a file that is not a multiple of 25791da177e4SLinus Torvalds * the page size, the remaining memory is zeroed when mapped, and 25801da177e4SLinus Torvalds * writes to that region are not written out to the file." 25811da177e4SLinus Torvalds */ 25821da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 25831da177e4SLinus Torvalds memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset); 25841da177e4SLinus Torvalds flush_dcache_page(page); 25851da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 25861da177e4SLinus Torvalds out: 25871da177e4SLinus Torvalds ret = mpage_writepage(page, get_block, wbc); 25881da177e4SLinus Torvalds if (ret == -EAGAIN) 25891da177e4SLinus Torvalds ret = __block_write_full_page(inode, page, get_block, wbc); 25901da177e4SLinus Torvalds return ret; 25911da177e4SLinus Torvalds } 25921da177e4SLinus Torvalds EXPORT_SYMBOL(nobh_writepage); 25931da177e4SLinus Torvalds 25941da177e4SLinus Torvalds /* 25951da177e4SLinus Torvalds * This function assumes that ->prepare_write() uses nobh_prepare_write(). 25961da177e4SLinus Torvalds */ 25971da177e4SLinus Torvalds int nobh_truncate_page(struct address_space *mapping, loff_t from) 25981da177e4SLinus Torvalds { 25991da177e4SLinus Torvalds struct inode *inode = mapping->host; 26001da177e4SLinus Torvalds unsigned blocksize = 1 << inode->i_blkbits; 26011da177e4SLinus Torvalds pgoff_t index = from >> PAGE_CACHE_SHIFT; 26021da177e4SLinus Torvalds unsigned offset = from & (PAGE_CACHE_SIZE-1); 26031da177e4SLinus Torvalds unsigned to; 26041da177e4SLinus Torvalds struct page *page; 2605f5e54d6eSChristoph Hellwig const struct address_space_operations *a_ops = mapping->a_ops; 26061da177e4SLinus Torvalds char *kaddr; 26071da177e4SLinus Torvalds int ret = 0; 26081da177e4SLinus Torvalds 26091da177e4SLinus Torvalds if ((offset & (blocksize - 1)) == 0) 26101da177e4SLinus Torvalds goto out; 26111da177e4SLinus Torvalds 26121da177e4SLinus Torvalds ret = -ENOMEM; 26131da177e4SLinus Torvalds page = grab_cache_page(mapping, index); 26141da177e4SLinus Torvalds if (!page) 26151da177e4SLinus Torvalds goto out; 26161da177e4SLinus Torvalds 26171da177e4SLinus Torvalds to = (offset + blocksize) & ~(blocksize - 1); 26181da177e4SLinus Torvalds ret = a_ops->prepare_write(NULL, page, offset, to); 26191da177e4SLinus Torvalds if (ret == 0) { 26201da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 26211da177e4SLinus Torvalds memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset); 26221da177e4SLinus Torvalds flush_dcache_page(page); 26231da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 26241da177e4SLinus Torvalds set_page_dirty(page); 26251da177e4SLinus Torvalds } 26261da177e4SLinus Torvalds unlock_page(page); 26271da177e4SLinus Torvalds page_cache_release(page); 26281da177e4SLinus Torvalds out: 26291da177e4SLinus Torvalds return ret; 26301da177e4SLinus Torvalds } 26311da177e4SLinus Torvalds EXPORT_SYMBOL(nobh_truncate_page); 26321da177e4SLinus Torvalds 26331da177e4SLinus Torvalds int block_truncate_page(struct address_space *mapping, 26341da177e4SLinus Torvalds loff_t from, get_block_t *get_block) 26351da177e4SLinus Torvalds { 26361da177e4SLinus Torvalds pgoff_t index = from >> PAGE_CACHE_SHIFT; 26371da177e4SLinus Torvalds unsigned offset = from & (PAGE_CACHE_SIZE-1); 26381da177e4SLinus Torvalds unsigned blocksize; 263954b21a79SAndrew Morton sector_t iblock; 26401da177e4SLinus Torvalds unsigned length, pos; 26411da177e4SLinus Torvalds struct inode *inode = mapping->host; 26421da177e4SLinus Torvalds struct page *page; 26431da177e4SLinus Torvalds struct buffer_head *bh; 26441da177e4SLinus Torvalds void *kaddr; 26451da177e4SLinus Torvalds int err; 26461da177e4SLinus Torvalds 26471da177e4SLinus Torvalds blocksize = 1 << inode->i_blkbits; 26481da177e4SLinus Torvalds length = offset & (blocksize - 1); 26491da177e4SLinus Torvalds 26501da177e4SLinus Torvalds /* Block boundary? Nothing to do */ 26511da177e4SLinus Torvalds if (!length) 26521da177e4SLinus Torvalds return 0; 26531da177e4SLinus Torvalds 26541da177e4SLinus Torvalds length = blocksize - length; 265554b21a79SAndrew Morton iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 26561da177e4SLinus Torvalds 26571da177e4SLinus Torvalds page = grab_cache_page(mapping, index); 26581da177e4SLinus Torvalds err = -ENOMEM; 26591da177e4SLinus Torvalds if (!page) 26601da177e4SLinus Torvalds goto out; 26611da177e4SLinus Torvalds 26621da177e4SLinus Torvalds if (!page_has_buffers(page)) 26631da177e4SLinus Torvalds create_empty_buffers(page, blocksize, 0); 26641da177e4SLinus Torvalds 26651da177e4SLinus Torvalds /* Find the buffer that contains "offset" */ 26661da177e4SLinus Torvalds bh = page_buffers(page); 26671da177e4SLinus Torvalds pos = blocksize; 26681da177e4SLinus Torvalds while (offset >= pos) { 26691da177e4SLinus Torvalds bh = bh->b_this_page; 26701da177e4SLinus Torvalds iblock++; 26711da177e4SLinus Torvalds pos += blocksize; 26721da177e4SLinus Torvalds } 26731da177e4SLinus Torvalds 26741da177e4SLinus Torvalds err = 0; 26751da177e4SLinus Torvalds if (!buffer_mapped(bh)) { 2676b0cf2321SBadari Pulavarty WARN_ON(bh->b_size != blocksize); 26771da177e4SLinus Torvalds err = get_block(inode, iblock, bh, 0); 26781da177e4SLinus Torvalds if (err) 26791da177e4SLinus Torvalds goto unlock; 26801da177e4SLinus Torvalds /* unmapped? It's a hole - nothing to do */ 26811da177e4SLinus Torvalds if (!buffer_mapped(bh)) 26821da177e4SLinus Torvalds goto unlock; 26831da177e4SLinus Torvalds } 26841da177e4SLinus Torvalds 26851da177e4SLinus Torvalds /* Ok, it's mapped. Make sure it's up-to-date */ 26861da177e4SLinus Torvalds if (PageUptodate(page)) 26871da177e4SLinus Torvalds set_buffer_uptodate(bh); 26881da177e4SLinus Torvalds 26891da177e4SLinus Torvalds if (!buffer_uptodate(bh) && !buffer_delay(bh)) { 26901da177e4SLinus Torvalds err = -EIO; 26911da177e4SLinus Torvalds ll_rw_block(READ, 1, &bh); 26921da177e4SLinus Torvalds wait_on_buffer(bh); 26931da177e4SLinus Torvalds /* Uhhuh. Read error. Complain and punt. */ 26941da177e4SLinus Torvalds if (!buffer_uptodate(bh)) 26951da177e4SLinus Torvalds goto unlock; 26961da177e4SLinus Torvalds } 26971da177e4SLinus Torvalds 26981da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 26991da177e4SLinus Torvalds memset(kaddr + offset, 0, length); 27001da177e4SLinus Torvalds flush_dcache_page(page); 27011da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 27021da177e4SLinus Torvalds 27031da177e4SLinus Torvalds mark_buffer_dirty(bh); 27041da177e4SLinus Torvalds err = 0; 27051da177e4SLinus Torvalds 27061da177e4SLinus Torvalds unlock: 27071da177e4SLinus Torvalds unlock_page(page); 27081da177e4SLinus Torvalds page_cache_release(page); 27091da177e4SLinus Torvalds out: 27101da177e4SLinus Torvalds return err; 27111da177e4SLinus Torvalds } 27121da177e4SLinus Torvalds 27131da177e4SLinus Torvalds /* 27141da177e4SLinus Torvalds * The generic ->writepage function for buffer-backed address_spaces 27151da177e4SLinus Torvalds */ 27161da177e4SLinus Torvalds int block_write_full_page(struct page *page, get_block_t *get_block, 27171da177e4SLinus Torvalds struct writeback_control *wbc) 27181da177e4SLinus Torvalds { 27191da177e4SLinus Torvalds struct inode * const inode = page->mapping->host; 27201da177e4SLinus Torvalds loff_t i_size = i_size_read(inode); 27211da177e4SLinus Torvalds const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; 27221da177e4SLinus Torvalds unsigned offset; 27231da177e4SLinus Torvalds void *kaddr; 27241da177e4SLinus Torvalds 27251da177e4SLinus Torvalds /* Is the page fully inside i_size? */ 27261da177e4SLinus Torvalds if (page->index < end_index) 27271da177e4SLinus Torvalds return __block_write_full_page(inode, page, get_block, wbc); 27281da177e4SLinus Torvalds 27291da177e4SLinus Torvalds /* Is the page fully outside i_size? (truncate in progress) */ 27301da177e4SLinus Torvalds offset = i_size & (PAGE_CACHE_SIZE-1); 27311da177e4SLinus Torvalds if (page->index >= end_index+1 || !offset) { 27321da177e4SLinus Torvalds /* 27331da177e4SLinus Torvalds * The page may have dirty, unmapped buffers. For example, 27341da177e4SLinus Torvalds * they may have been added in ext3_writepage(). Make them 27351da177e4SLinus Torvalds * freeable here, so the page does not leak. 27361da177e4SLinus Torvalds */ 2737aaa4059bSJan Kara do_invalidatepage(page, 0); 27381da177e4SLinus Torvalds unlock_page(page); 27391da177e4SLinus Torvalds return 0; /* don't care */ 27401da177e4SLinus Torvalds } 27411da177e4SLinus Torvalds 27421da177e4SLinus Torvalds /* 27431da177e4SLinus Torvalds * The page straddles i_size. It must be zeroed out on each and every 27441da177e4SLinus Torvalds * writepage invokation because it may be mmapped. "A file is mapped 27451da177e4SLinus Torvalds * in multiples of the page size. For a file that is not a multiple of 27461da177e4SLinus Torvalds * the page size, the remaining memory is zeroed when mapped, and 27471da177e4SLinus Torvalds * writes to that region are not written out to the file." 27481da177e4SLinus Torvalds */ 27491da177e4SLinus Torvalds kaddr = kmap_atomic(page, KM_USER0); 27501da177e4SLinus Torvalds memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset); 27511da177e4SLinus Torvalds flush_dcache_page(page); 27521da177e4SLinus Torvalds kunmap_atomic(kaddr, KM_USER0); 27531da177e4SLinus Torvalds return __block_write_full_page(inode, page, get_block, wbc); 27541da177e4SLinus Torvalds } 27551da177e4SLinus Torvalds 27561da177e4SLinus Torvalds sector_t generic_block_bmap(struct address_space *mapping, sector_t block, 27571da177e4SLinus Torvalds get_block_t *get_block) 27581da177e4SLinus Torvalds { 27591da177e4SLinus Torvalds struct buffer_head tmp; 27601da177e4SLinus Torvalds struct inode *inode = mapping->host; 27611da177e4SLinus Torvalds tmp.b_state = 0; 27621da177e4SLinus Torvalds tmp.b_blocknr = 0; 2763b0cf2321SBadari Pulavarty tmp.b_size = 1 << inode->i_blkbits; 27641da177e4SLinus Torvalds get_block(inode, block, &tmp, 0); 27651da177e4SLinus Torvalds return tmp.b_blocknr; 27661da177e4SLinus Torvalds } 27671da177e4SLinus Torvalds 27681da177e4SLinus Torvalds static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err) 27691da177e4SLinus Torvalds { 27701da177e4SLinus Torvalds struct buffer_head *bh = bio->bi_private; 27711da177e4SLinus Torvalds 27721da177e4SLinus Torvalds if (bio->bi_size) 27731da177e4SLinus Torvalds return 1; 27741da177e4SLinus Torvalds 27751da177e4SLinus Torvalds if (err == -EOPNOTSUPP) { 27761da177e4SLinus Torvalds set_bit(BIO_EOPNOTSUPP, &bio->bi_flags); 27771da177e4SLinus Torvalds set_bit(BH_Eopnotsupp, &bh->b_state); 27781da177e4SLinus Torvalds } 27791da177e4SLinus Torvalds 27801da177e4SLinus Torvalds bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags)); 27811da177e4SLinus Torvalds bio_put(bio); 27821da177e4SLinus Torvalds return 0; 27831da177e4SLinus Torvalds } 27841da177e4SLinus Torvalds 27851da177e4SLinus Torvalds int submit_bh(int rw, struct buffer_head * bh) 27861da177e4SLinus Torvalds { 27871da177e4SLinus Torvalds struct bio *bio; 27881da177e4SLinus Torvalds int ret = 0; 27891da177e4SLinus Torvalds 27901da177e4SLinus Torvalds BUG_ON(!buffer_locked(bh)); 27911da177e4SLinus Torvalds BUG_ON(!buffer_mapped(bh)); 27921da177e4SLinus Torvalds BUG_ON(!bh->b_end_io); 27931da177e4SLinus Torvalds 27941da177e4SLinus Torvalds if (buffer_ordered(bh) && (rw == WRITE)) 27951da177e4SLinus Torvalds rw = WRITE_BARRIER; 27961da177e4SLinus Torvalds 27971da177e4SLinus Torvalds /* 27981da177e4SLinus Torvalds * Only clear out a write error when rewriting, should this 27991da177e4SLinus Torvalds * include WRITE_SYNC as well? 28001da177e4SLinus Torvalds */ 28011da177e4SLinus Torvalds if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER)) 28021da177e4SLinus Torvalds clear_buffer_write_io_error(bh); 28031da177e4SLinus Torvalds 28041da177e4SLinus Torvalds /* 28051da177e4SLinus Torvalds * from here on down, it's all bio -- do the initial mapping, 28061da177e4SLinus Torvalds * submit_bio -> generic_make_request may further map this bio around 28071da177e4SLinus Torvalds */ 28081da177e4SLinus Torvalds bio = bio_alloc(GFP_NOIO, 1); 28091da177e4SLinus Torvalds 28101da177e4SLinus Torvalds bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 28111da177e4SLinus Torvalds bio->bi_bdev = bh->b_bdev; 28121da177e4SLinus Torvalds bio->bi_io_vec[0].bv_page = bh->b_page; 28131da177e4SLinus Torvalds bio->bi_io_vec[0].bv_len = bh->b_size; 28141da177e4SLinus Torvalds bio->bi_io_vec[0].bv_offset = bh_offset(bh); 28151da177e4SLinus Torvalds 28161da177e4SLinus Torvalds bio->bi_vcnt = 1; 28171da177e4SLinus Torvalds bio->bi_idx = 0; 28181da177e4SLinus Torvalds bio->bi_size = bh->b_size; 28191da177e4SLinus Torvalds 28201da177e4SLinus Torvalds bio->bi_end_io = end_bio_bh_io_sync; 28211da177e4SLinus Torvalds bio->bi_private = bh; 28221da177e4SLinus Torvalds 28231da177e4SLinus Torvalds bio_get(bio); 28241da177e4SLinus Torvalds submit_bio(rw, bio); 28251da177e4SLinus Torvalds 28261da177e4SLinus Torvalds if (bio_flagged(bio, BIO_EOPNOTSUPP)) 28271da177e4SLinus Torvalds ret = -EOPNOTSUPP; 28281da177e4SLinus Torvalds 28291da177e4SLinus Torvalds bio_put(bio); 28301da177e4SLinus Torvalds return ret; 28311da177e4SLinus Torvalds } 28321da177e4SLinus Torvalds 28331da177e4SLinus Torvalds /** 28341da177e4SLinus Torvalds * ll_rw_block: low-level access to block devices (DEPRECATED) 2835a7662236SJan Kara * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead) 28361da177e4SLinus Torvalds * @nr: number of &struct buffer_heads in the array 28371da177e4SLinus Torvalds * @bhs: array of pointers to &struct buffer_head 28381da177e4SLinus Torvalds * 2839a7662236SJan Kara * ll_rw_block() takes an array of pointers to &struct buffer_heads, and 2840a7662236SJan Kara * requests an I/O operation on them, either a %READ or a %WRITE. The third 2841a7662236SJan Kara * %SWRITE is like %WRITE only we make sure that the *current* data in buffers 2842a7662236SJan Kara * are sent to disk. The fourth %READA option is described in the documentation 2843a7662236SJan Kara * for generic_make_request() which ll_rw_block() calls. 28441da177e4SLinus Torvalds * 28451da177e4SLinus Torvalds * This function drops any buffer that it cannot get a lock on (with the 2846a7662236SJan Kara * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be 2847a7662236SJan Kara * clean when doing a write request, and any buffer that appears to be 2848a7662236SJan Kara * up-to-date when doing read request. Further it marks as clean buffers that 2849a7662236SJan Kara * are processed for writing (the buffer cache won't assume that they are 2850a7662236SJan Kara * actually clean until the buffer gets unlocked). 28511da177e4SLinus Torvalds * 28521da177e4SLinus Torvalds * ll_rw_block sets b_end_io to simple completion handler that marks 28531da177e4SLinus Torvalds * the buffer up-to-date (if approriate), unlocks the buffer and wakes 28541da177e4SLinus Torvalds * any waiters. 28551da177e4SLinus Torvalds * 28561da177e4SLinus Torvalds * All of the buffers must be for the same device, and must also be a 28571da177e4SLinus Torvalds * multiple of the current approved size for the device. 28581da177e4SLinus Torvalds */ 28591da177e4SLinus Torvalds void ll_rw_block(int rw, int nr, struct buffer_head *bhs[]) 28601da177e4SLinus Torvalds { 28611da177e4SLinus Torvalds int i; 28621da177e4SLinus Torvalds 28631da177e4SLinus Torvalds for (i = 0; i < nr; i++) { 28641da177e4SLinus Torvalds struct buffer_head *bh = bhs[i]; 28651da177e4SLinus Torvalds 2866a7662236SJan Kara if (rw == SWRITE) 2867a7662236SJan Kara lock_buffer(bh); 2868a7662236SJan Kara else if (test_set_buffer_locked(bh)) 28691da177e4SLinus Torvalds continue; 28701da177e4SLinus Torvalds 2871a7662236SJan Kara if (rw == WRITE || rw == SWRITE) { 28721da177e4SLinus Torvalds if (test_clear_buffer_dirty(bh)) { 287376c3073aSakpm@osdl.org bh->b_end_io = end_buffer_write_sync; 2874e60e5c50SOGAWA Hirofumi get_bh(bh); 28751da177e4SLinus Torvalds submit_bh(WRITE, bh); 28761da177e4SLinus Torvalds continue; 28771da177e4SLinus Torvalds } 28781da177e4SLinus Torvalds } else { 28791da177e4SLinus Torvalds if (!buffer_uptodate(bh)) { 288076c3073aSakpm@osdl.org bh->b_end_io = end_buffer_read_sync; 2881e60e5c50SOGAWA Hirofumi get_bh(bh); 28821da177e4SLinus Torvalds submit_bh(rw, bh); 28831da177e4SLinus Torvalds continue; 28841da177e4SLinus Torvalds } 28851da177e4SLinus Torvalds } 28861da177e4SLinus Torvalds unlock_buffer(bh); 28871da177e4SLinus Torvalds } 28881da177e4SLinus Torvalds } 28891da177e4SLinus Torvalds 28901da177e4SLinus Torvalds /* 28911da177e4SLinus Torvalds * For a data-integrity writeout, we need to wait upon any in-progress I/O 28921da177e4SLinus Torvalds * and then start new I/O and then wait upon it. The caller must have a ref on 28931da177e4SLinus Torvalds * the buffer_head. 28941da177e4SLinus Torvalds */ 28951da177e4SLinus Torvalds int sync_dirty_buffer(struct buffer_head *bh) 28961da177e4SLinus Torvalds { 28971da177e4SLinus Torvalds int ret = 0; 28981da177e4SLinus Torvalds 28991da177e4SLinus Torvalds WARN_ON(atomic_read(&bh->b_count) < 1); 29001da177e4SLinus Torvalds lock_buffer(bh); 29011da177e4SLinus Torvalds if (test_clear_buffer_dirty(bh)) { 29021da177e4SLinus Torvalds get_bh(bh); 29031da177e4SLinus Torvalds bh->b_end_io = end_buffer_write_sync; 29041da177e4SLinus Torvalds ret = submit_bh(WRITE, bh); 29051da177e4SLinus Torvalds wait_on_buffer(bh); 29061da177e4SLinus Torvalds if (buffer_eopnotsupp(bh)) { 29071da177e4SLinus Torvalds clear_buffer_eopnotsupp(bh); 29081da177e4SLinus Torvalds ret = -EOPNOTSUPP; 29091da177e4SLinus Torvalds } 29101da177e4SLinus Torvalds if (!ret && !buffer_uptodate(bh)) 29111da177e4SLinus Torvalds ret = -EIO; 29121da177e4SLinus Torvalds } else { 29131da177e4SLinus Torvalds unlock_buffer(bh); 29141da177e4SLinus Torvalds } 29151da177e4SLinus Torvalds return ret; 29161da177e4SLinus Torvalds } 29171da177e4SLinus Torvalds 29181da177e4SLinus Torvalds /* 29191da177e4SLinus Torvalds * try_to_free_buffers() checks if all the buffers on this particular page 29201da177e4SLinus Torvalds * are unused, and releases them if so. 29211da177e4SLinus Torvalds * 29221da177e4SLinus Torvalds * Exclusion against try_to_free_buffers may be obtained by either 29231da177e4SLinus Torvalds * locking the page or by holding its mapping's private_lock. 29241da177e4SLinus Torvalds * 29251da177e4SLinus Torvalds * If the page is dirty but all the buffers are clean then we need to 29261da177e4SLinus Torvalds * be sure to mark the page clean as well. This is because the page 29271da177e4SLinus Torvalds * may be against a block device, and a later reattachment of buffers 29281da177e4SLinus Torvalds * to a dirty page will set *all* buffers dirty. Which would corrupt 29291da177e4SLinus Torvalds * filesystem data on the same device. 29301da177e4SLinus Torvalds * 29311da177e4SLinus Torvalds * The same applies to regular filesystem pages: if all the buffers are 29321da177e4SLinus Torvalds * clean then we set the page clean and proceed. To do that, we require 29331da177e4SLinus Torvalds * total exclusion from __set_page_dirty_buffers(). That is obtained with 29341da177e4SLinus Torvalds * private_lock. 29351da177e4SLinus Torvalds * 29361da177e4SLinus Torvalds * try_to_free_buffers() is non-blocking. 29371da177e4SLinus Torvalds */ 29381da177e4SLinus Torvalds static inline int buffer_busy(struct buffer_head *bh) 29391da177e4SLinus Torvalds { 29401da177e4SLinus Torvalds return atomic_read(&bh->b_count) | 29411da177e4SLinus Torvalds (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock))); 29421da177e4SLinus Torvalds } 29431da177e4SLinus Torvalds 29441da177e4SLinus Torvalds static int 29451da177e4SLinus Torvalds drop_buffers(struct page *page, struct buffer_head **buffers_to_free) 29461da177e4SLinus Torvalds { 29471da177e4SLinus Torvalds struct buffer_head *head = page_buffers(page); 29481da177e4SLinus Torvalds struct buffer_head *bh; 29491da177e4SLinus Torvalds 29501da177e4SLinus Torvalds bh = head; 29511da177e4SLinus Torvalds do { 2952de7d5a3bSakpm@osdl.org if (buffer_write_io_error(bh) && page->mapping) 29531da177e4SLinus Torvalds set_bit(AS_EIO, &page->mapping->flags); 29541da177e4SLinus Torvalds if (buffer_busy(bh)) 29551da177e4SLinus Torvalds goto failed; 29561da177e4SLinus Torvalds bh = bh->b_this_page; 29571da177e4SLinus Torvalds } while (bh != head); 29581da177e4SLinus Torvalds 29591da177e4SLinus Torvalds do { 29601da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 29611da177e4SLinus Torvalds 29621da177e4SLinus Torvalds if (!list_empty(&bh->b_assoc_buffers)) 29631da177e4SLinus Torvalds __remove_assoc_queue(bh); 29641da177e4SLinus Torvalds bh = next; 29651da177e4SLinus Torvalds } while (bh != head); 29661da177e4SLinus Torvalds *buffers_to_free = head; 29671da177e4SLinus Torvalds __clear_page_buffers(page); 29681da177e4SLinus Torvalds return 1; 29691da177e4SLinus Torvalds failed: 29701da177e4SLinus Torvalds return 0; 29711da177e4SLinus Torvalds } 29721da177e4SLinus Torvalds 29731da177e4SLinus Torvalds int try_to_free_buffers(struct page *page) 29741da177e4SLinus Torvalds { 29751da177e4SLinus Torvalds struct address_space * const mapping = page->mapping; 29761da177e4SLinus Torvalds struct buffer_head *buffers_to_free = NULL; 29771da177e4SLinus Torvalds int ret = 0; 29781da177e4SLinus Torvalds 29791da177e4SLinus Torvalds BUG_ON(!PageLocked(page)); 29801da177e4SLinus Torvalds if (PageWriteback(page)) 29811da177e4SLinus Torvalds return 0; 29821da177e4SLinus Torvalds 29831da177e4SLinus Torvalds if (mapping == NULL) { /* can this still happen? */ 29841da177e4SLinus Torvalds ret = drop_buffers(page, &buffers_to_free); 29851da177e4SLinus Torvalds goto out; 29861da177e4SLinus Torvalds } 29871da177e4SLinus Torvalds 29881da177e4SLinus Torvalds spin_lock(&mapping->private_lock); 29891da177e4SLinus Torvalds ret = drop_buffers(page, &buffers_to_free); 29901da177e4SLinus Torvalds if (ret) { 29911da177e4SLinus Torvalds /* 29921da177e4SLinus Torvalds * If the filesystem writes its buffers by hand (eg ext3) 29931da177e4SLinus Torvalds * then we can have clean buffers against a dirty page. We 29941da177e4SLinus Torvalds * clean the page here; otherwise later reattachment of buffers 29951da177e4SLinus Torvalds * could encounter a non-uptodate page, which is unresolvable. 29961da177e4SLinus Torvalds * This only applies in the rare case where try_to_free_buffers 29971da177e4SLinus Torvalds * succeeds but the page is not freed. 29981da177e4SLinus Torvalds */ 29991da177e4SLinus Torvalds clear_page_dirty(page); 30001da177e4SLinus Torvalds } 30011da177e4SLinus Torvalds spin_unlock(&mapping->private_lock); 30021da177e4SLinus Torvalds out: 30031da177e4SLinus Torvalds if (buffers_to_free) { 30041da177e4SLinus Torvalds struct buffer_head *bh = buffers_to_free; 30051da177e4SLinus Torvalds 30061da177e4SLinus Torvalds do { 30071da177e4SLinus Torvalds struct buffer_head *next = bh->b_this_page; 30081da177e4SLinus Torvalds free_buffer_head(bh); 30091da177e4SLinus Torvalds bh = next; 30101da177e4SLinus Torvalds } while (bh != buffers_to_free); 30111da177e4SLinus Torvalds } 30121da177e4SLinus Torvalds return ret; 30131da177e4SLinus Torvalds } 30141da177e4SLinus Torvalds EXPORT_SYMBOL(try_to_free_buffers); 30151da177e4SLinus Torvalds 30163978d717SNeilBrown void block_sync_page(struct page *page) 30171da177e4SLinus Torvalds { 30181da177e4SLinus Torvalds struct address_space *mapping; 30191da177e4SLinus Torvalds 30201da177e4SLinus Torvalds smp_mb(); 30211da177e4SLinus Torvalds mapping = page_mapping(page); 30221da177e4SLinus Torvalds if (mapping) 30231da177e4SLinus Torvalds blk_run_backing_dev(mapping->backing_dev_info, page); 30241da177e4SLinus Torvalds } 30251da177e4SLinus Torvalds 30261da177e4SLinus Torvalds /* 30271da177e4SLinus Torvalds * There are no bdflush tunables left. But distributions are 30281da177e4SLinus Torvalds * still running obsolete flush daemons, so we terminate them here. 30291da177e4SLinus Torvalds * 30301da177e4SLinus Torvalds * Use of bdflush() is deprecated and will be removed in a future kernel. 30311da177e4SLinus Torvalds * The `pdflush' kernel threads fully replace bdflush daemons and this call. 30321da177e4SLinus Torvalds */ 30331da177e4SLinus Torvalds asmlinkage long sys_bdflush(int func, long data) 30341da177e4SLinus Torvalds { 30351da177e4SLinus Torvalds static int msg_count; 30361da177e4SLinus Torvalds 30371da177e4SLinus Torvalds if (!capable(CAP_SYS_ADMIN)) 30381da177e4SLinus Torvalds return -EPERM; 30391da177e4SLinus Torvalds 30401da177e4SLinus Torvalds if (msg_count < 5) { 30411da177e4SLinus Torvalds msg_count++; 30421da177e4SLinus Torvalds printk(KERN_INFO 30431da177e4SLinus Torvalds "warning: process `%s' used the obsolete bdflush" 30441da177e4SLinus Torvalds " system call\n", current->comm); 30451da177e4SLinus Torvalds printk(KERN_INFO "Fix your initscripts?\n"); 30461da177e4SLinus Torvalds } 30471da177e4SLinus Torvalds 30481da177e4SLinus Torvalds if (func == 1) 30491da177e4SLinus Torvalds do_exit(0); 30501da177e4SLinus Torvalds return 0; 30511da177e4SLinus Torvalds } 30521da177e4SLinus Torvalds 30531da177e4SLinus Torvalds /* 30541da177e4SLinus Torvalds * Buffer-head allocation 30551da177e4SLinus Torvalds */ 30561da177e4SLinus Torvalds static kmem_cache_t *bh_cachep; 30571da177e4SLinus Torvalds 30581da177e4SLinus Torvalds /* 30591da177e4SLinus Torvalds * Once the number of bh's in the machine exceeds this level, we start 30601da177e4SLinus Torvalds * stripping them in writeback. 30611da177e4SLinus Torvalds */ 30621da177e4SLinus Torvalds static int max_buffer_heads; 30631da177e4SLinus Torvalds 30641da177e4SLinus Torvalds int buffer_heads_over_limit; 30651da177e4SLinus Torvalds 30661da177e4SLinus Torvalds struct bh_accounting { 30671da177e4SLinus Torvalds int nr; /* Number of live bh's */ 30681da177e4SLinus Torvalds int ratelimit; /* Limit cacheline bouncing */ 30691da177e4SLinus Torvalds }; 30701da177e4SLinus Torvalds 30711da177e4SLinus Torvalds static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0}; 30721da177e4SLinus Torvalds 30731da177e4SLinus Torvalds static void recalc_bh_state(void) 30741da177e4SLinus Torvalds { 30751da177e4SLinus Torvalds int i; 30761da177e4SLinus Torvalds int tot = 0; 30771da177e4SLinus Torvalds 30781da177e4SLinus Torvalds if (__get_cpu_var(bh_accounting).ratelimit++ < 4096) 30791da177e4SLinus Torvalds return; 30801da177e4SLinus Torvalds __get_cpu_var(bh_accounting).ratelimit = 0; 30818a143426SEric Dumazet for_each_online_cpu(i) 30821da177e4SLinus Torvalds tot += per_cpu(bh_accounting, i).nr; 30831da177e4SLinus Torvalds buffer_heads_over_limit = (tot > max_buffer_heads); 30841da177e4SLinus Torvalds } 30851da177e4SLinus Torvalds 3086dd0fc66fSAl Viro struct buffer_head *alloc_buffer_head(gfp_t gfp_flags) 30871da177e4SLinus Torvalds { 30881da177e4SLinus Torvalds struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags); 30891da177e4SLinus Torvalds if (ret) { 3090736c7b80SCoywolf Qi Hunt get_cpu_var(bh_accounting).nr++; 30911da177e4SLinus Torvalds recalc_bh_state(); 3092736c7b80SCoywolf Qi Hunt put_cpu_var(bh_accounting); 30931da177e4SLinus Torvalds } 30941da177e4SLinus Torvalds return ret; 30951da177e4SLinus Torvalds } 30961da177e4SLinus Torvalds EXPORT_SYMBOL(alloc_buffer_head); 30971da177e4SLinus Torvalds 30981da177e4SLinus Torvalds void free_buffer_head(struct buffer_head *bh) 30991da177e4SLinus Torvalds { 31001da177e4SLinus Torvalds BUG_ON(!list_empty(&bh->b_assoc_buffers)); 31011da177e4SLinus Torvalds kmem_cache_free(bh_cachep, bh); 3102736c7b80SCoywolf Qi Hunt get_cpu_var(bh_accounting).nr--; 31031da177e4SLinus Torvalds recalc_bh_state(); 3104736c7b80SCoywolf Qi Hunt put_cpu_var(bh_accounting); 31051da177e4SLinus Torvalds } 31061da177e4SLinus Torvalds EXPORT_SYMBOL(free_buffer_head); 31071da177e4SLinus Torvalds 31081da177e4SLinus Torvalds static void 31091da177e4SLinus Torvalds init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags) 31101da177e4SLinus Torvalds { 31111da177e4SLinus Torvalds if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == 31121da177e4SLinus Torvalds SLAB_CTOR_CONSTRUCTOR) { 31131da177e4SLinus Torvalds struct buffer_head * bh = (struct buffer_head *)data; 31141da177e4SLinus Torvalds 31151da177e4SLinus Torvalds memset(bh, 0, sizeof(*bh)); 31161da177e4SLinus Torvalds INIT_LIST_HEAD(&bh->b_assoc_buffers); 31171da177e4SLinus Torvalds } 31181da177e4SLinus Torvalds } 31191da177e4SLinus Torvalds 31201da177e4SLinus Torvalds #ifdef CONFIG_HOTPLUG_CPU 31211da177e4SLinus Torvalds static void buffer_exit_cpu(int cpu) 31221da177e4SLinus Torvalds { 31231da177e4SLinus Torvalds int i; 31241da177e4SLinus Torvalds struct bh_lru *b = &per_cpu(bh_lrus, cpu); 31251da177e4SLinus Torvalds 31261da177e4SLinus Torvalds for (i = 0; i < BH_LRU_SIZE; i++) { 31271da177e4SLinus Torvalds brelse(b->bhs[i]); 31281da177e4SLinus Torvalds b->bhs[i] = NULL; 31291da177e4SLinus Torvalds } 31308a143426SEric Dumazet get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr; 31318a143426SEric Dumazet per_cpu(bh_accounting, cpu).nr = 0; 31328a143426SEric Dumazet put_cpu_var(bh_accounting); 31331da177e4SLinus Torvalds } 31341da177e4SLinus Torvalds 31351da177e4SLinus Torvalds static int buffer_cpu_notify(struct notifier_block *self, 31361da177e4SLinus Torvalds unsigned long action, void *hcpu) 31371da177e4SLinus Torvalds { 31381da177e4SLinus Torvalds if (action == CPU_DEAD) 31391da177e4SLinus Torvalds buffer_exit_cpu((unsigned long)hcpu); 31401da177e4SLinus Torvalds return NOTIFY_OK; 31411da177e4SLinus Torvalds } 31421da177e4SLinus Torvalds #endif /* CONFIG_HOTPLUG_CPU */ 31431da177e4SLinus Torvalds 31441da177e4SLinus Torvalds void __init buffer_init(void) 31451da177e4SLinus Torvalds { 31461da177e4SLinus Torvalds int nrpages; 31471da177e4SLinus Torvalds 31481da177e4SLinus Torvalds bh_cachep = kmem_cache_create("buffer_head", 31491da177e4SLinus Torvalds sizeof(struct buffer_head), 0, 3150b0196009SPaul Jackson (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| 3151b0196009SPaul Jackson SLAB_MEM_SPREAD), 3152b0196009SPaul Jackson init_buffer_head, 3153b0196009SPaul Jackson NULL); 31541da177e4SLinus Torvalds 31551da177e4SLinus Torvalds /* 31561da177e4SLinus Torvalds * Limit the bh occupancy to 10% of ZONE_NORMAL 31571da177e4SLinus Torvalds */ 31581da177e4SLinus Torvalds nrpages = (nr_free_buffer_pages() * 10) / 100; 31591da177e4SLinus Torvalds max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head)); 31601da177e4SLinus Torvalds hotcpu_notifier(buffer_cpu_notify, 0); 31611da177e4SLinus Torvalds } 31621da177e4SLinus Torvalds 31631da177e4SLinus Torvalds EXPORT_SYMBOL(__bforget); 31641da177e4SLinus Torvalds EXPORT_SYMBOL(__brelse); 31651da177e4SLinus Torvalds EXPORT_SYMBOL(__wait_on_buffer); 31661da177e4SLinus Torvalds EXPORT_SYMBOL(block_commit_write); 31671da177e4SLinus Torvalds EXPORT_SYMBOL(block_prepare_write); 31681da177e4SLinus Torvalds EXPORT_SYMBOL(block_read_full_page); 31691da177e4SLinus Torvalds EXPORT_SYMBOL(block_sync_page); 31701da177e4SLinus Torvalds EXPORT_SYMBOL(block_truncate_page); 31711da177e4SLinus Torvalds EXPORT_SYMBOL(block_write_full_page); 31721da177e4SLinus Torvalds EXPORT_SYMBOL(cont_prepare_write); 31731da177e4SLinus Torvalds EXPORT_SYMBOL(end_buffer_read_sync); 31741da177e4SLinus Torvalds EXPORT_SYMBOL(end_buffer_write_sync); 31751da177e4SLinus Torvalds EXPORT_SYMBOL(file_fsync); 31761da177e4SLinus Torvalds EXPORT_SYMBOL(fsync_bdev); 31771da177e4SLinus Torvalds EXPORT_SYMBOL(generic_block_bmap); 31781da177e4SLinus Torvalds EXPORT_SYMBOL(generic_commit_write); 31791da177e4SLinus Torvalds EXPORT_SYMBOL(generic_cont_expand); 318005eb0b51SOGAWA Hirofumi EXPORT_SYMBOL(generic_cont_expand_simple); 31811da177e4SLinus Torvalds EXPORT_SYMBOL(init_buffer); 31821da177e4SLinus Torvalds EXPORT_SYMBOL(invalidate_bdev); 31831da177e4SLinus Torvalds EXPORT_SYMBOL(ll_rw_block); 31841da177e4SLinus Torvalds EXPORT_SYMBOL(mark_buffer_dirty); 31851da177e4SLinus Torvalds EXPORT_SYMBOL(submit_bh); 31861da177e4SLinus Torvalds EXPORT_SYMBOL(sync_dirty_buffer); 31871da177e4SLinus Torvalds EXPORT_SYMBOL(unlock_buffer); 3188