1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 5 * Copyright (C) 2016 - 2020 Christoph Hellwig 6 */ 7 8 #include <linux/init.h> 9 #include <linux/mm.h> 10 #include <linux/slab.h> 11 #include <linux/kmod.h> 12 #include <linux/major.h> 13 #include <linux/device_cgroup.h> 14 #include <linux/blkdev.h> 15 #include <linux/blk-integrity.h> 16 #include <linux/backing-dev.h> 17 #include <linux/module.h> 18 #include <linux/blkpg.h> 19 #include <linux/magic.h> 20 #include <linux/buffer_head.h> 21 #include <linux/swap.h> 22 #include <linux/writeback.h> 23 #include <linux/mount.h> 24 #include <linux/pseudo_fs.h> 25 #include <linux/uio.h> 26 #include <linux/namei.h> 27 #include <linux/cleancache.h> 28 #include <linux/part_stat.h> 29 #include <linux/uaccess.h> 30 #include "../fs/internal.h" 31 #include "blk.h" 32 33 struct bdev_inode { 34 struct block_device bdev; 35 struct inode vfs_inode; 36 }; 37 38 static inline struct bdev_inode *BDEV_I(struct inode *inode) 39 { 40 return container_of(inode, struct bdev_inode, vfs_inode); 41 } 42 43 struct block_device *I_BDEV(struct inode *inode) 44 { 45 return &BDEV_I(inode)->bdev; 46 } 47 EXPORT_SYMBOL(I_BDEV); 48 49 static void bdev_write_inode(struct block_device *bdev) 50 { 51 struct inode *inode = bdev->bd_inode; 52 int ret; 53 54 spin_lock(&inode->i_lock); 55 while (inode->i_state & I_DIRTY) { 56 spin_unlock(&inode->i_lock); 57 ret = write_inode_now(inode, true); 58 if (ret) { 59 char name[BDEVNAME_SIZE]; 60 pr_warn_ratelimited("VFS: Dirty inode writeback failed " 61 "for block device %s (err=%d).\n", 62 bdevname(bdev, name), ret); 63 } 64 spin_lock(&inode->i_lock); 65 } 66 spin_unlock(&inode->i_lock); 67 } 68 69 /* Kill _all_ buffers and pagecache , dirty or not.. */ 70 static void kill_bdev(struct block_device *bdev) 71 { 72 struct address_space *mapping = bdev->bd_inode->i_mapping; 73 74 if (mapping_empty(mapping)) 75 return; 76 77 invalidate_bh_lrus(); 78 truncate_inode_pages(mapping, 0); 79 } 80 81 /* Invalidate clean unused buffers and pagecache. */ 82 void invalidate_bdev(struct block_device *bdev) 83 { 84 struct address_space *mapping = bdev->bd_inode->i_mapping; 85 86 if (mapping->nrpages) { 87 invalidate_bh_lrus(); 88 lru_add_drain_all(); /* make sure all lru add caches are flushed */ 89 invalidate_mapping_pages(mapping, 0, -1); 90 } 91 /* 99% of the time, we don't need to flush the cleancache on the bdev. 92 * But, for the strange corners, lets be cautious 93 */ 94 cleancache_invalidate_inode(mapping); 95 } 96 EXPORT_SYMBOL(invalidate_bdev); 97 98 /* 99 * Drop all buffers & page cache for given bdev range. This function bails 100 * with error if bdev has other exclusive owner (such as filesystem). 101 */ 102 int truncate_bdev_range(struct block_device *bdev, fmode_t mode, 103 loff_t lstart, loff_t lend) 104 { 105 /* 106 * If we don't hold exclusive handle for the device, upgrade to it 107 * while we discard the buffer cache to avoid discarding buffers 108 * under live filesystem. 109 */ 110 if (!(mode & FMODE_EXCL)) { 111 int err = bd_prepare_to_claim(bdev, truncate_bdev_range); 112 if (err) 113 goto invalidate; 114 } 115 116 truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend); 117 if (!(mode & FMODE_EXCL)) 118 bd_abort_claiming(bdev, truncate_bdev_range); 119 return 0; 120 121 invalidate: 122 /* 123 * Someone else has handle exclusively open. Try invalidating instead. 124 * The 'end' argument is inclusive so the rounding is safe. 125 */ 126 return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, 127 lstart >> PAGE_SHIFT, 128 lend >> PAGE_SHIFT); 129 } 130 131 static void set_init_blocksize(struct block_device *bdev) 132 { 133 unsigned int bsize = bdev_logical_block_size(bdev); 134 loff_t size = i_size_read(bdev->bd_inode); 135 136 while (bsize < PAGE_SIZE) { 137 if (size & bsize) 138 break; 139 bsize <<= 1; 140 } 141 bdev->bd_inode->i_blkbits = blksize_bits(bsize); 142 } 143 144 int set_blocksize(struct block_device *bdev, int size) 145 { 146 /* Size must be a power of two, and between 512 and PAGE_SIZE */ 147 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 148 return -EINVAL; 149 150 /* Size cannot be smaller than the size supported by the device */ 151 if (size < bdev_logical_block_size(bdev)) 152 return -EINVAL; 153 154 /* Don't change the size if it is same as current */ 155 if (bdev->bd_inode->i_blkbits != blksize_bits(size)) { 156 sync_blockdev(bdev); 157 bdev->bd_inode->i_blkbits = blksize_bits(size); 158 kill_bdev(bdev); 159 } 160 return 0; 161 } 162 163 EXPORT_SYMBOL(set_blocksize); 164 165 int sb_set_blocksize(struct super_block *sb, int size) 166 { 167 if (set_blocksize(sb->s_bdev, size)) 168 return 0; 169 /* If we get here, we know size is power of two 170 * and it's value is between 512 and PAGE_SIZE */ 171 sb->s_blocksize = size; 172 sb->s_blocksize_bits = blksize_bits(size); 173 return sb->s_blocksize; 174 } 175 176 EXPORT_SYMBOL(sb_set_blocksize); 177 178 int sb_min_blocksize(struct super_block *sb, int size) 179 { 180 int minsize = bdev_logical_block_size(sb->s_bdev); 181 if (size < minsize) 182 size = minsize; 183 return sb_set_blocksize(sb, size); 184 } 185 186 EXPORT_SYMBOL(sb_min_blocksize); 187 188 int sync_blockdev_nowait(struct block_device *bdev) 189 { 190 if (!bdev) 191 return 0; 192 return filemap_flush(bdev->bd_inode->i_mapping); 193 } 194 EXPORT_SYMBOL_GPL(sync_blockdev_nowait); 195 196 /* 197 * Write out and wait upon all the dirty data associated with a block 198 * device via its mapping. Does not take the superblock lock. 199 */ 200 int sync_blockdev(struct block_device *bdev) 201 { 202 if (!bdev) 203 return 0; 204 return filemap_write_and_wait(bdev->bd_inode->i_mapping); 205 } 206 EXPORT_SYMBOL(sync_blockdev); 207 208 /* 209 * Write out and wait upon all dirty data associated with this 210 * device. Filesystem data as well as the underlying block 211 * device. Takes the superblock lock. 212 */ 213 int fsync_bdev(struct block_device *bdev) 214 { 215 struct super_block *sb = get_super(bdev); 216 if (sb) { 217 int res = sync_filesystem(sb); 218 drop_super(sb); 219 return res; 220 } 221 return sync_blockdev(bdev); 222 } 223 EXPORT_SYMBOL(fsync_bdev); 224 225 /** 226 * freeze_bdev -- lock a filesystem and force it into a consistent state 227 * @bdev: blockdevice to lock 228 * 229 * If a superblock is found on this device, we take the s_umount semaphore 230 * on it to make sure nobody unmounts until the snapshot creation is done. 231 * The reference counter (bd_fsfreeze_count) guarantees that only the last 232 * unfreeze process can unfreeze the frozen filesystem actually when multiple 233 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and 234 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze 235 * actually. 236 */ 237 int freeze_bdev(struct block_device *bdev) 238 { 239 struct super_block *sb; 240 int error = 0; 241 242 mutex_lock(&bdev->bd_fsfreeze_mutex); 243 if (++bdev->bd_fsfreeze_count > 1) 244 goto done; 245 246 sb = get_active_super(bdev); 247 if (!sb) 248 goto sync; 249 if (sb->s_op->freeze_super) 250 error = sb->s_op->freeze_super(sb); 251 else 252 error = freeze_super(sb); 253 deactivate_super(sb); 254 255 if (error) { 256 bdev->bd_fsfreeze_count--; 257 goto done; 258 } 259 bdev->bd_fsfreeze_sb = sb; 260 261 sync: 262 sync_blockdev(bdev); 263 done: 264 mutex_unlock(&bdev->bd_fsfreeze_mutex); 265 return error; 266 } 267 EXPORT_SYMBOL(freeze_bdev); 268 269 /** 270 * thaw_bdev -- unlock filesystem 271 * @bdev: blockdevice to unlock 272 * 273 * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 274 */ 275 int thaw_bdev(struct block_device *bdev) 276 { 277 struct super_block *sb; 278 int error = -EINVAL; 279 280 mutex_lock(&bdev->bd_fsfreeze_mutex); 281 if (!bdev->bd_fsfreeze_count) 282 goto out; 283 284 error = 0; 285 if (--bdev->bd_fsfreeze_count > 0) 286 goto out; 287 288 sb = bdev->bd_fsfreeze_sb; 289 if (!sb) 290 goto out; 291 292 if (sb->s_op->thaw_super) 293 error = sb->s_op->thaw_super(sb); 294 else 295 error = thaw_super(sb); 296 if (error) 297 bdev->bd_fsfreeze_count++; 298 else 299 bdev->bd_fsfreeze_sb = NULL; 300 out: 301 mutex_unlock(&bdev->bd_fsfreeze_mutex); 302 return error; 303 } 304 EXPORT_SYMBOL(thaw_bdev); 305 306 /** 307 * bdev_read_page() - Start reading a page from a block device 308 * @bdev: The device to read the page from 309 * @sector: The offset on the device to read the page to (need not be aligned) 310 * @page: The page to read 311 * 312 * On entry, the page should be locked. It will be unlocked when the page 313 * has been read. If the block driver implements rw_page synchronously, 314 * that will be true on exit from this function, but it need not be. 315 * 316 * Errors returned by this function are usually "soft", eg out of memory, or 317 * queue full; callers should try a different route to read this page rather 318 * than propagate an error back up the stack. 319 * 320 * Return: negative errno if an error occurs, 0 if submission was successful. 321 */ 322 int bdev_read_page(struct block_device *bdev, sector_t sector, 323 struct page *page) 324 { 325 const struct block_device_operations *ops = bdev->bd_disk->fops; 326 int result = -EOPNOTSUPP; 327 328 if (!ops->rw_page || bdev_get_integrity(bdev)) 329 return result; 330 331 result = blk_queue_enter(bdev_get_queue(bdev), 0); 332 if (result) 333 return result; 334 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, 335 REQ_OP_READ); 336 blk_queue_exit(bdev_get_queue(bdev)); 337 return result; 338 } 339 340 /** 341 * bdev_write_page() - Start writing a page to a block device 342 * @bdev: The device to write the page to 343 * @sector: The offset on the device to write the page to (need not be aligned) 344 * @page: The page to write 345 * @wbc: The writeback_control for the write 346 * 347 * On entry, the page should be locked and not currently under writeback. 348 * On exit, if the write started successfully, the page will be unlocked and 349 * under writeback. If the write failed already (eg the driver failed to 350 * queue the page to the device), the page will still be locked. If the 351 * caller is a ->writepage implementation, it will need to unlock the page. 352 * 353 * Errors returned by this function are usually "soft", eg out of memory, or 354 * queue full; callers should try a different route to write this page rather 355 * than propagate an error back up the stack. 356 * 357 * Return: negative errno if an error occurs, 0 if submission was successful. 358 */ 359 int bdev_write_page(struct block_device *bdev, sector_t sector, 360 struct page *page, struct writeback_control *wbc) 361 { 362 int result; 363 const struct block_device_operations *ops = bdev->bd_disk->fops; 364 365 if (!ops->rw_page || bdev_get_integrity(bdev)) 366 return -EOPNOTSUPP; 367 result = blk_queue_enter(bdev_get_queue(bdev), 0); 368 if (result) 369 return result; 370 371 set_page_writeback(page); 372 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, 373 REQ_OP_WRITE); 374 if (result) { 375 end_page_writeback(page); 376 } else { 377 clean_page_buffers(page); 378 unlock_page(page); 379 } 380 blk_queue_exit(bdev_get_queue(bdev)); 381 return result; 382 } 383 384 /* 385 * pseudo-fs 386 */ 387 388 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock); 389 static struct kmem_cache * bdev_cachep __read_mostly; 390 391 static struct inode *bdev_alloc_inode(struct super_block *sb) 392 { 393 struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL); 394 395 if (!ei) 396 return NULL; 397 memset(&ei->bdev, 0, sizeof(ei->bdev)); 398 return &ei->vfs_inode; 399 } 400 401 static void bdev_free_inode(struct inode *inode) 402 { 403 struct block_device *bdev = I_BDEV(inode); 404 405 free_percpu(bdev->bd_stats); 406 kfree(bdev->bd_meta_info); 407 408 if (!bdev_is_partition(bdev)) { 409 if (bdev->bd_disk && bdev->bd_disk->bdi) 410 bdi_put(bdev->bd_disk->bdi); 411 kfree(bdev->bd_disk); 412 } 413 414 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 415 blk_free_ext_minor(MINOR(bdev->bd_dev)); 416 417 kmem_cache_free(bdev_cachep, BDEV_I(inode)); 418 } 419 420 static void init_once(void *data) 421 { 422 struct bdev_inode *ei = data; 423 424 inode_init_once(&ei->vfs_inode); 425 } 426 427 static void bdev_evict_inode(struct inode *inode) 428 { 429 truncate_inode_pages_final(&inode->i_data); 430 invalidate_inode_buffers(inode); /* is it needed here? */ 431 clear_inode(inode); 432 } 433 434 static const struct super_operations bdev_sops = { 435 .statfs = simple_statfs, 436 .alloc_inode = bdev_alloc_inode, 437 .free_inode = bdev_free_inode, 438 .drop_inode = generic_delete_inode, 439 .evict_inode = bdev_evict_inode, 440 }; 441 442 static int bd_init_fs_context(struct fs_context *fc) 443 { 444 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC); 445 if (!ctx) 446 return -ENOMEM; 447 fc->s_iflags |= SB_I_CGROUPWB; 448 ctx->ops = &bdev_sops; 449 return 0; 450 } 451 452 static struct file_system_type bd_type = { 453 .name = "bdev", 454 .init_fs_context = bd_init_fs_context, 455 .kill_sb = kill_anon_super, 456 }; 457 458 struct super_block *blockdev_superblock __read_mostly; 459 EXPORT_SYMBOL_GPL(blockdev_superblock); 460 461 void __init bdev_cache_init(void) 462 { 463 int err; 464 static struct vfsmount *bd_mnt; 465 466 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 467 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 468 SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC), 469 init_once); 470 err = register_filesystem(&bd_type); 471 if (err) 472 panic("Cannot register bdev pseudo-fs"); 473 bd_mnt = kern_mount(&bd_type); 474 if (IS_ERR(bd_mnt)) 475 panic("Cannot create bdev pseudo-fs"); 476 blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */ 477 } 478 479 struct block_device *bdev_alloc(struct gendisk *disk, u8 partno) 480 { 481 struct block_device *bdev; 482 struct inode *inode; 483 484 inode = new_inode(blockdev_superblock); 485 if (!inode) 486 return NULL; 487 inode->i_mode = S_IFBLK; 488 inode->i_rdev = 0; 489 inode->i_data.a_ops = &def_blk_aops; 490 mapping_set_gfp_mask(&inode->i_data, GFP_USER); 491 492 bdev = I_BDEV(inode); 493 mutex_init(&bdev->bd_fsfreeze_mutex); 494 spin_lock_init(&bdev->bd_size_lock); 495 bdev->bd_partno = partno; 496 bdev->bd_inode = inode; 497 bdev->bd_queue = disk->queue; 498 bdev->bd_stats = alloc_percpu(struct disk_stats); 499 if (!bdev->bd_stats) { 500 iput(inode); 501 return NULL; 502 } 503 bdev->bd_disk = disk; 504 return bdev; 505 } 506 507 void bdev_add(struct block_device *bdev, dev_t dev) 508 { 509 bdev->bd_dev = dev; 510 bdev->bd_inode->i_rdev = dev; 511 bdev->bd_inode->i_ino = dev; 512 insert_inode_hash(bdev->bd_inode); 513 } 514 515 long nr_blockdev_pages(void) 516 { 517 struct inode *inode; 518 long ret = 0; 519 520 spin_lock(&blockdev_superblock->s_inode_list_lock); 521 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) 522 ret += inode->i_mapping->nrpages; 523 spin_unlock(&blockdev_superblock->s_inode_list_lock); 524 525 return ret; 526 } 527 528 /** 529 * bd_may_claim - test whether a block device can be claimed 530 * @bdev: block device of interest 531 * @whole: whole block device containing @bdev, may equal @bdev 532 * @holder: holder trying to claim @bdev 533 * 534 * Test whether @bdev can be claimed by @holder. 535 * 536 * CONTEXT: 537 * spin_lock(&bdev_lock). 538 * 539 * RETURNS: 540 * %true if @bdev can be claimed, %false otherwise. 541 */ 542 static bool bd_may_claim(struct block_device *bdev, struct block_device *whole, 543 void *holder) 544 { 545 if (bdev->bd_holder == holder) 546 return true; /* already a holder */ 547 else if (bdev->bd_holder != NULL) 548 return false; /* held by someone else */ 549 else if (whole == bdev) 550 return true; /* is a whole device which isn't held */ 551 552 else if (whole->bd_holder == bd_may_claim) 553 return true; /* is a partition of a device that is being partitioned */ 554 else if (whole->bd_holder != NULL) 555 return false; /* is a partition of a held device */ 556 else 557 return true; /* is a partition of an un-held device */ 558 } 559 560 /** 561 * bd_prepare_to_claim - claim a block device 562 * @bdev: block device of interest 563 * @holder: holder trying to claim @bdev 564 * 565 * Claim @bdev. This function fails if @bdev is already claimed by another 566 * holder and waits if another claiming is in progress. return, the caller 567 * has ownership of bd_claiming and bd_holder[s]. 568 * 569 * RETURNS: 570 * 0 if @bdev can be claimed, -EBUSY otherwise. 571 */ 572 int bd_prepare_to_claim(struct block_device *bdev, void *holder) 573 { 574 struct block_device *whole = bdev_whole(bdev); 575 576 if (WARN_ON_ONCE(!holder)) 577 return -EINVAL; 578 retry: 579 spin_lock(&bdev_lock); 580 /* if someone else claimed, fail */ 581 if (!bd_may_claim(bdev, whole, holder)) { 582 spin_unlock(&bdev_lock); 583 return -EBUSY; 584 } 585 586 /* if claiming is already in progress, wait for it to finish */ 587 if (whole->bd_claiming) { 588 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 589 DEFINE_WAIT(wait); 590 591 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 592 spin_unlock(&bdev_lock); 593 schedule(); 594 finish_wait(wq, &wait); 595 goto retry; 596 } 597 598 /* yay, all mine */ 599 whole->bd_claiming = holder; 600 spin_unlock(&bdev_lock); 601 return 0; 602 } 603 EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */ 604 605 static void bd_clear_claiming(struct block_device *whole, void *holder) 606 { 607 lockdep_assert_held(&bdev_lock); 608 /* tell others that we're done */ 609 BUG_ON(whole->bd_claiming != holder); 610 whole->bd_claiming = NULL; 611 wake_up_bit(&whole->bd_claiming, 0); 612 } 613 614 /** 615 * bd_finish_claiming - finish claiming of a block device 616 * @bdev: block device of interest 617 * @holder: holder that has claimed @bdev 618 * 619 * Finish exclusive open of a block device. Mark the device as exlusively 620 * open by the holder and wake up all waiters for exclusive open to finish. 621 */ 622 static void bd_finish_claiming(struct block_device *bdev, void *holder) 623 { 624 struct block_device *whole = bdev_whole(bdev); 625 626 spin_lock(&bdev_lock); 627 BUG_ON(!bd_may_claim(bdev, whole, holder)); 628 /* 629 * Note that for a whole device bd_holders will be incremented twice, 630 * and bd_holder will be set to bd_may_claim before being set to holder 631 */ 632 whole->bd_holders++; 633 whole->bd_holder = bd_may_claim; 634 bdev->bd_holders++; 635 bdev->bd_holder = holder; 636 bd_clear_claiming(whole, holder); 637 spin_unlock(&bdev_lock); 638 } 639 640 /** 641 * bd_abort_claiming - abort claiming of a block device 642 * @bdev: block device of interest 643 * @holder: holder that has claimed @bdev 644 * 645 * Abort claiming of a block device when the exclusive open failed. This can be 646 * also used when exclusive open is not actually desired and we just needed 647 * to block other exclusive openers for a while. 648 */ 649 void bd_abort_claiming(struct block_device *bdev, void *holder) 650 { 651 spin_lock(&bdev_lock); 652 bd_clear_claiming(bdev_whole(bdev), holder); 653 spin_unlock(&bdev_lock); 654 } 655 EXPORT_SYMBOL(bd_abort_claiming); 656 657 static void blkdev_flush_mapping(struct block_device *bdev) 658 { 659 WARN_ON_ONCE(bdev->bd_holders); 660 sync_blockdev(bdev); 661 kill_bdev(bdev); 662 bdev_write_inode(bdev); 663 } 664 665 static int blkdev_get_whole(struct block_device *bdev, fmode_t mode) 666 { 667 struct gendisk *disk = bdev->bd_disk; 668 int ret = 0; 669 670 if (disk->fops->open) { 671 ret = disk->fops->open(bdev, mode); 672 if (ret) { 673 /* avoid ghost partitions on a removed medium */ 674 if (ret == -ENOMEDIUM && 675 test_bit(GD_NEED_PART_SCAN, &disk->state)) 676 bdev_disk_changed(disk, true); 677 return ret; 678 } 679 } 680 681 if (!bdev->bd_openers) 682 set_init_blocksize(bdev); 683 if (test_bit(GD_NEED_PART_SCAN, &disk->state)) 684 bdev_disk_changed(disk, false); 685 bdev->bd_openers++; 686 return 0;; 687 } 688 689 static void blkdev_put_whole(struct block_device *bdev, fmode_t mode) 690 { 691 if (!--bdev->bd_openers) 692 blkdev_flush_mapping(bdev); 693 if (bdev->bd_disk->fops->release) 694 bdev->bd_disk->fops->release(bdev->bd_disk, mode); 695 } 696 697 static int blkdev_get_part(struct block_device *part, fmode_t mode) 698 { 699 struct gendisk *disk = part->bd_disk; 700 int ret; 701 702 if (part->bd_openers) 703 goto done; 704 705 ret = blkdev_get_whole(bdev_whole(part), mode); 706 if (ret) 707 return ret; 708 709 ret = -ENXIO; 710 if (!bdev_nr_sectors(part)) 711 goto out_blkdev_put; 712 713 disk->open_partitions++; 714 set_init_blocksize(part); 715 done: 716 part->bd_openers++; 717 return 0; 718 719 out_blkdev_put: 720 blkdev_put_whole(bdev_whole(part), mode); 721 return ret; 722 } 723 724 static void blkdev_put_part(struct block_device *part, fmode_t mode) 725 { 726 struct block_device *whole = bdev_whole(part); 727 728 if (--part->bd_openers) 729 return; 730 blkdev_flush_mapping(part); 731 whole->bd_disk->open_partitions--; 732 blkdev_put_whole(whole, mode); 733 } 734 735 struct block_device *blkdev_get_no_open(dev_t dev) 736 { 737 struct block_device *bdev; 738 struct inode *inode; 739 740 inode = ilookup(blockdev_superblock, dev); 741 if (!inode) { 742 blk_request_module(dev); 743 inode = ilookup(blockdev_superblock, dev); 744 if (!inode) 745 return NULL; 746 } 747 748 /* switch from the inode reference to a device mode one: */ 749 bdev = &BDEV_I(inode)->bdev; 750 if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) 751 bdev = NULL; 752 iput(inode); 753 754 if (!bdev) 755 return NULL; 756 if ((bdev->bd_disk->flags & GENHD_FL_HIDDEN) || 757 !try_module_get(bdev->bd_disk->fops->owner)) { 758 put_device(&bdev->bd_device); 759 return NULL; 760 } 761 762 return bdev; 763 } 764 765 void blkdev_put_no_open(struct block_device *bdev) 766 { 767 module_put(bdev->bd_disk->fops->owner); 768 put_device(&bdev->bd_device); 769 } 770 771 /** 772 * blkdev_get_by_dev - open a block device by device number 773 * @dev: device number of block device to open 774 * @mode: FMODE_* mask 775 * @holder: exclusive holder identifier 776 * 777 * Open the block device described by device number @dev. If @mode includes 778 * %FMODE_EXCL, the block device is opened with exclusive access. Specifying 779 * %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may nest for 780 * the same @holder. 781 * 782 * Use this interface ONLY if you really do not have anything better - i.e. when 783 * you are behind a truly sucky interface and all you are given is a device 784 * number. Everything else should use blkdev_get_by_path(). 785 * 786 * CONTEXT: 787 * Might sleep. 788 * 789 * RETURNS: 790 * Reference to the block_device on success, ERR_PTR(-errno) on failure. 791 */ 792 struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder) 793 { 794 bool unblock_events = true; 795 struct block_device *bdev; 796 struct gendisk *disk; 797 int ret; 798 799 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK, 800 MAJOR(dev), MINOR(dev), 801 ((mode & FMODE_READ) ? DEVCG_ACC_READ : 0) | 802 ((mode & FMODE_WRITE) ? DEVCG_ACC_WRITE : 0)); 803 if (ret) 804 return ERR_PTR(ret); 805 806 bdev = blkdev_get_no_open(dev); 807 if (!bdev) 808 return ERR_PTR(-ENXIO); 809 disk = bdev->bd_disk; 810 811 if (mode & FMODE_EXCL) { 812 ret = bd_prepare_to_claim(bdev, holder); 813 if (ret) 814 goto put_blkdev; 815 } 816 817 disk_block_events(disk); 818 819 mutex_lock(&disk->open_mutex); 820 ret = -ENXIO; 821 if (!disk_live(disk)) 822 goto abort_claiming; 823 if (bdev_is_partition(bdev)) 824 ret = blkdev_get_part(bdev, mode); 825 else 826 ret = blkdev_get_whole(bdev, mode); 827 if (ret) 828 goto abort_claiming; 829 if (mode & FMODE_EXCL) { 830 bd_finish_claiming(bdev, holder); 831 832 /* 833 * Block event polling for write claims if requested. Any write 834 * holder makes the write_holder state stick until all are 835 * released. This is good enough and tracking individual 836 * writeable reference is too fragile given the way @mode is 837 * used in blkdev_get/put(). 838 */ 839 if ((mode & FMODE_WRITE) && !bdev->bd_write_holder && 840 (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) { 841 bdev->bd_write_holder = true; 842 unblock_events = false; 843 } 844 } 845 mutex_unlock(&disk->open_mutex); 846 847 if (unblock_events) 848 disk_unblock_events(disk); 849 return bdev; 850 851 abort_claiming: 852 if (mode & FMODE_EXCL) 853 bd_abort_claiming(bdev, holder); 854 mutex_unlock(&disk->open_mutex); 855 disk_unblock_events(disk); 856 put_blkdev: 857 blkdev_put_no_open(bdev); 858 return ERR_PTR(ret); 859 } 860 EXPORT_SYMBOL(blkdev_get_by_dev); 861 862 /** 863 * blkdev_get_by_path - open a block device by name 864 * @path: path to the block device to open 865 * @mode: FMODE_* mask 866 * @holder: exclusive holder identifier 867 * 868 * Open the block device described by the device file at @path. If @mode 869 * includes %FMODE_EXCL, the block device is opened with exclusive access. 870 * Specifying %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may 871 * nest for the same @holder. 872 * 873 * CONTEXT: 874 * Might sleep. 875 * 876 * RETURNS: 877 * Reference to the block_device on success, ERR_PTR(-errno) on failure. 878 */ 879 struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, 880 void *holder) 881 { 882 struct block_device *bdev; 883 dev_t dev; 884 int error; 885 886 error = lookup_bdev(path, &dev); 887 if (error) 888 return ERR_PTR(error); 889 890 bdev = blkdev_get_by_dev(dev, mode, holder); 891 if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) { 892 blkdev_put(bdev, mode); 893 return ERR_PTR(-EACCES); 894 } 895 896 return bdev; 897 } 898 EXPORT_SYMBOL(blkdev_get_by_path); 899 900 void blkdev_put(struct block_device *bdev, fmode_t mode) 901 { 902 struct gendisk *disk = bdev->bd_disk; 903 904 /* 905 * Sync early if it looks like we're the last one. If someone else 906 * opens the block device between now and the decrement of bd_openers 907 * then we did a sync that we didn't need to, but that's not the end 908 * of the world and we want to avoid long (could be several minute) 909 * syncs while holding the mutex. 910 */ 911 if (bdev->bd_openers == 1) 912 sync_blockdev(bdev); 913 914 mutex_lock(&disk->open_mutex); 915 if (mode & FMODE_EXCL) { 916 struct block_device *whole = bdev_whole(bdev); 917 bool bdev_free; 918 919 /* 920 * Release a claim on the device. The holder fields 921 * are protected with bdev_lock. open_mutex is to 922 * synchronize disk_holder unlinking. 923 */ 924 spin_lock(&bdev_lock); 925 926 WARN_ON_ONCE(--bdev->bd_holders < 0); 927 WARN_ON_ONCE(--whole->bd_holders < 0); 928 929 if ((bdev_free = !bdev->bd_holders)) 930 bdev->bd_holder = NULL; 931 if (!whole->bd_holders) 932 whole->bd_holder = NULL; 933 934 spin_unlock(&bdev_lock); 935 936 /* 937 * If this was the last claim, remove holder link and 938 * unblock evpoll if it was a write holder. 939 */ 940 if (bdev_free && bdev->bd_write_holder) { 941 disk_unblock_events(disk); 942 bdev->bd_write_holder = false; 943 } 944 } 945 946 /* 947 * Trigger event checking and tell drivers to flush MEDIA_CHANGE 948 * event. This is to ensure detection of media removal commanded 949 * from userland - e.g. eject(1). 950 */ 951 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE); 952 953 if (bdev_is_partition(bdev)) 954 blkdev_put_part(bdev, mode); 955 else 956 blkdev_put_whole(bdev, mode); 957 mutex_unlock(&disk->open_mutex); 958 959 blkdev_put_no_open(bdev); 960 } 961 EXPORT_SYMBOL(blkdev_put); 962 963 /** 964 * lookup_bdev - lookup a struct block_device by name 965 * @pathname: special file representing the block device 966 * @dev: return value of the block device's dev_t 967 * 968 * Lookup the block device's dev_t at @pathname in the current 969 * namespace if possible and return it by @dev. 970 * 971 * RETURNS: 972 * 0 if succeeded, errno otherwise. 973 */ 974 int lookup_bdev(const char *pathname, dev_t *dev) 975 { 976 struct inode *inode; 977 struct path path; 978 int error; 979 980 if (!pathname || !*pathname) 981 return -EINVAL; 982 983 error = kern_path(pathname, LOOKUP_FOLLOW, &path); 984 if (error) 985 return error; 986 987 inode = d_backing_inode(path.dentry); 988 error = -ENOTBLK; 989 if (!S_ISBLK(inode->i_mode)) 990 goto out_path_put; 991 error = -EACCES; 992 if (!may_open_dev(&path)) 993 goto out_path_put; 994 995 *dev = inode->i_rdev; 996 error = 0; 997 out_path_put: 998 path_put(&path); 999 return error; 1000 } 1001 EXPORT_SYMBOL(lookup_bdev); 1002 1003 int __invalidate_device(struct block_device *bdev, bool kill_dirty) 1004 { 1005 struct super_block *sb = get_super(bdev); 1006 int res = 0; 1007 1008 if (sb) { 1009 /* 1010 * no need to lock the super, get_super holds the 1011 * read mutex so the filesystem cannot go away 1012 * under us (->put_super runs with the write lock 1013 * hold). 1014 */ 1015 shrink_dcache_sb(sb); 1016 res = invalidate_inodes(sb, kill_dirty); 1017 drop_super(sb); 1018 } 1019 invalidate_bdev(bdev); 1020 return res; 1021 } 1022 EXPORT_SYMBOL(__invalidate_device); 1023 1024 void sync_bdevs(bool wait) 1025 { 1026 struct inode *inode, *old_inode = NULL; 1027 1028 spin_lock(&blockdev_superblock->s_inode_list_lock); 1029 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 1030 struct address_space *mapping = inode->i_mapping; 1031 struct block_device *bdev; 1032 1033 spin_lock(&inode->i_lock); 1034 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 1035 mapping->nrpages == 0) { 1036 spin_unlock(&inode->i_lock); 1037 continue; 1038 } 1039 __iget(inode); 1040 spin_unlock(&inode->i_lock); 1041 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1042 /* 1043 * We hold a reference to 'inode' so it couldn't have been 1044 * removed from s_inodes list while we dropped the 1045 * s_inode_list_lock We cannot iput the inode now as we can 1046 * be holding the last reference and we cannot iput it under 1047 * s_inode_list_lock. So we keep the reference and iput it 1048 * later. 1049 */ 1050 iput(old_inode); 1051 old_inode = inode; 1052 bdev = I_BDEV(inode); 1053 1054 mutex_lock(&bdev->bd_disk->open_mutex); 1055 if (!bdev->bd_openers) { 1056 ; /* skip */ 1057 } else if (wait) { 1058 /* 1059 * We keep the error status of individual mapping so 1060 * that applications can catch the writeback error using 1061 * fsync(2). See filemap_fdatawait_keep_errors() for 1062 * details. 1063 */ 1064 filemap_fdatawait_keep_errors(inode->i_mapping); 1065 } else { 1066 filemap_fdatawrite(inode->i_mapping); 1067 } 1068 mutex_unlock(&bdev->bd_disk->open_mutex); 1069 1070 spin_lock(&blockdev_superblock->s_inode_list_lock); 1071 } 1072 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1073 iput(old_inode); 1074 } 1075