1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 5 * Copyright (C) 2016 - 2020 Christoph Hellwig 6 */ 7 8 #include <linux/init.h> 9 #include <linux/mm.h> 10 #include <linux/slab.h> 11 #include <linux/kmod.h> 12 #include <linux/major.h> 13 #include <linux/device_cgroup.h> 14 #include <linux/blkdev.h> 15 #include <linux/blk-integrity.h> 16 #include <linux/backing-dev.h> 17 #include <linux/module.h> 18 #include <linux/blkpg.h> 19 #include <linux/magic.h> 20 #include <linux/buffer_head.h> 21 #include <linux/swap.h> 22 #include <linux/writeback.h> 23 #include <linux/mount.h> 24 #include <linux/pseudo_fs.h> 25 #include <linux/uio.h> 26 #include <linux/namei.h> 27 #include <linux/part_stat.h> 28 #include <linux/uaccess.h> 29 #include <linux/stat.h> 30 #include "../fs/internal.h" 31 #include "blk.h" 32 33 /* Should we allow writing to mounted block devices? */ 34 static bool bdev_allow_write_mounted = IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED); 35 36 struct bdev_inode { 37 struct block_device bdev; 38 struct inode vfs_inode; 39 }; 40 41 static inline struct bdev_inode *BDEV_I(struct inode *inode) 42 { 43 return container_of(inode, struct bdev_inode, vfs_inode); 44 } 45 46 struct block_device *I_BDEV(struct inode *inode) 47 { 48 return &BDEV_I(inode)->bdev; 49 } 50 EXPORT_SYMBOL(I_BDEV); 51 52 struct block_device *file_bdev(struct file *bdev_file) 53 { 54 return I_BDEV(bdev_file->f_mapping->host); 55 } 56 EXPORT_SYMBOL(file_bdev); 57 58 static void bdev_write_inode(struct block_device *bdev) 59 { 60 struct inode *inode = bdev->bd_inode; 61 int ret; 62 63 spin_lock(&inode->i_lock); 64 while (inode->i_state & I_DIRTY) { 65 spin_unlock(&inode->i_lock); 66 ret = write_inode_now(inode, true); 67 if (ret) 68 pr_warn_ratelimited( 69 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n", 70 bdev, ret); 71 spin_lock(&inode->i_lock); 72 } 73 spin_unlock(&inode->i_lock); 74 } 75 76 /* Kill _all_ buffers and pagecache , dirty or not.. */ 77 static void kill_bdev(struct block_device *bdev) 78 { 79 struct address_space *mapping = bdev->bd_inode->i_mapping; 80 81 if (mapping_empty(mapping)) 82 return; 83 84 invalidate_bh_lrus(); 85 truncate_inode_pages(mapping, 0); 86 } 87 88 /* Invalidate clean unused buffers and pagecache. */ 89 void invalidate_bdev(struct block_device *bdev) 90 { 91 struct address_space *mapping = bdev->bd_inode->i_mapping; 92 93 if (mapping->nrpages) { 94 invalidate_bh_lrus(); 95 lru_add_drain_all(); /* make sure all lru add caches are flushed */ 96 invalidate_mapping_pages(mapping, 0, -1); 97 } 98 } 99 EXPORT_SYMBOL(invalidate_bdev); 100 101 /* 102 * Drop all buffers & page cache for given bdev range. This function bails 103 * with error if bdev has other exclusive owner (such as filesystem). 104 */ 105 int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode, 106 loff_t lstart, loff_t lend) 107 { 108 /* 109 * If we don't hold exclusive handle for the device, upgrade to it 110 * while we discard the buffer cache to avoid discarding buffers 111 * under live filesystem. 112 */ 113 if (!(mode & BLK_OPEN_EXCL)) { 114 int err = bd_prepare_to_claim(bdev, truncate_bdev_range, NULL); 115 if (err) 116 goto invalidate; 117 } 118 119 truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend); 120 if (!(mode & BLK_OPEN_EXCL)) 121 bd_abort_claiming(bdev, truncate_bdev_range); 122 return 0; 123 124 invalidate: 125 /* 126 * Someone else has handle exclusively open. Try invalidating instead. 127 * The 'end' argument is inclusive so the rounding is safe. 128 */ 129 return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, 130 lstart >> PAGE_SHIFT, 131 lend >> PAGE_SHIFT); 132 } 133 134 static void set_init_blocksize(struct block_device *bdev) 135 { 136 unsigned int bsize = bdev_logical_block_size(bdev); 137 loff_t size = i_size_read(bdev->bd_inode); 138 139 while (bsize < PAGE_SIZE) { 140 if (size & bsize) 141 break; 142 bsize <<= 1; 143 } 144 bdev->bd_inode->i_blkbits = blksize_bits(bsize); 145 } 146 147 int set_blocksize(struct block_device *bdev, int size) 148 { 149 /* Size must be a power of two, and between 512 and PAGE_SIZE */ 150 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 151 return -EINVAL; 152 153 /* Size cannot be smaller than the size supported by the device */ 154 if (size < bdev_logical_block_size(bdev)) 155 return -EINVAL; 156 157 /* Don't change the size if it is same as current */ 158 if (bdev->bd_inode->i_blkbits != blksize_bits(size)) { 159 sync_blockdev(bdev); 160 bdev->bd_inode->i_blkbits = blksize_bits(size); 161 kill_bdev(bdev); 162 } 163 return 0; 164 } 165 166 EXPORT_SYMBOL(set_blocksize); 167 168 int sb_set_blocksize(struct super_block *sb, int size) 169 { 170 if (set_blocksize(sb->s_bdev, size)) 171 return 0; 172 /* If we get here, we know size is power of two 173 * and it's value is between 512 and PAGE_SIZE */ 174 sb->s_blocksize = size; 175 sb->s_blocksize_bits = blksize_bits(size); 176 return sb->s_blocksize; 177 } 178 179 EXPORT_SYMBOL(sb_set_blocksize); 180 181 int sb_min_blocksize(struct super_block *sb, int size) 182 { 183 int minsize = bdev_logical_block_size(sb->s_bdev); 184 if (size < minsize) 185 size = minsize; 186 return sb_set_blocksize(sb, size); 187 } 188 189 EXPORT_SYMBOL(sb_min_blocksize); 190 191 int sync_blockdev_nowait(struct block_device *bdev) 192 { 193 if (!bdev) 194 return 0; 195 return filemap_flush(bdev->bd_inode->i_mapping); 196 } 197 EXPORT_SYMBOL_GPL(sync_blockdev_nowait); 198 199 /* 200 * Write out and wait upon all the dirty data associated with a block 201 * device via its mapping. Does not take the superblock lock. 202 */ 203 int sync_blockdev(struct block_device *bdev) 204 { 205 if (!bdev) 206 return 0; 207 return filemap_write_and_wait(bdev->bd_inode->i_mapping); 208 } 209 EXPORT_SYMBOL(sync_blockdev); 210 211 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend) 212 { 213 return filemap_write_and_wait_range(bdev->bd_inode->i_mapping, 214 lstart, lend); 215 } 216 EXPORT_SYMBOL(sync_blockdev_range); 217 218 /** 219 * bdev_freeze - lock a filesystem and force it into a consistent state 220 * @bdev: blockdevice to lock 221 * 222 * If a superblock is found on this device, we take the s_umount semaphore 223 * on it to make sure nobody unmounts until the snapshot creation is done. 224 * The reference counter (bd_fsfreeze_count) guarantees that only the last 225 * unfreeze process can unfreeze the frozen filesystem actually when multiple 226 * freeze requests arrive simultaneously. It counts up in bdev_freeze() and 227 * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze 228 * actually. 229 * 230 * Return: On success zero is returned, negative error code on failure. 231 */ 232 int bdev_freeze(struct block_device *bdev) 233 { 234 int error = 0; 235 236 mutex_lock(&bdev->bd_fsfreeze_mutex); 237 238 if (atomic_inc_return(&bdev->bd_fsfreeze_count) > 1) { 239 mutex_unlock(&bdev->bd_fsfreeze_mutex); 240 return 0; 241 } 242 243 mutex_lock(&bdev->bd_holder_lock); 244 if (bdev->bd_holder_ops && bdev->bd_holder_ops->freeze) { 245 error = bdev->bd_holder_ops->freeze(bdev); 246 lockdep_assert_not_held(&bdev->bd_holder_lock); 247 } else { 248 mutex_unlock(&bdev->bd_holder_lock); 249 error = sync_blockdev(bdev); 250 } 251 252 if (error) 253 atomic_dec(&bdev->bd_fsfreeze_count); 254 255 mutex_unlock(&bdev->bd_fsfreeze_mutex); 256 return error; 257 } 258 EXPORT_SYMBOL(bdev_freeze); 259 260 /** 261 * bdev_thaw - unlock filesystem 262 * @bdev: blockdevice to unlock 263 * 264 * Unlocks the filesystem and marks it writeable again after bdev_freeze(). 265 * 266 * Return: On success zero is returned, negative error code on failure. 267 */ 268 int bdev_thaw(struct block_device *bdev) 269 { 270 int error = -EINVAL, nr_freeze; 271 272 mutex_lock(&bdev->bd_fsfreeze_mutex); 273 274 /* 275 * If this returns < 0 it means that @bd_fsfreeze_count was 276 * already 0 and no decrement was performed. 277 */ 278 nr_freeze = atomic_dec_if_positive(&bdev->bd_fsfreeze_count); 279 if (nr_freeze < 0) 280 goto out; 281 282 error = 0; 283 if (nr_freeze > 0) 284 goto out; 285 286 mutex_lock(&bdev->bd_holder_lock); 287 if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) { 288 error = bdev->bd_holder_ops->thaw(bdev); 289 lockdep_assert_not_held(&bdev->bd_holder_lock); 290 } else { 291 mutex_unlock(&bdev->bd_holder_lock); 292 } 293 294 if (error) 295 atomic_inc(&bdev->bd_fsfreeze_count); 296 out: 297 mutex_unlock(&bdev->bd_fsfreeze_mutex); 298 return error; 299 } 300 EXPORT_SYMBOL(bdev_thaw); 301 302 /* 303 * pseudo-fs 304 */ 305 306 static __cacheline_aligned_in_smp DEFINE_MUTEX(bdev_lock); 307 static struct kmem_cache *bdev_cachep __ro_after_init; 308 309 static struct inode *bdev_alloc_inode(struct super_block *sb) 310 { 311 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL); 312 313 if (!ei) 314 return NULL; 315 memset(&ei->bdev, 0, sizeof(ei->bdev)); 316 return &ei->vfs_inode; 317 } 318 319 static void bdev_free_inode(struct inode *inode) 320 { 321 struct block_device *bdev = I_BDEV(inode); 322 323 free_percpu(bdev->bd_stats); 324 kfree(bdev->bd_meta_info); 325 326 if (!bdev_is_partition(bdev)) { 327 if (bdev->bd_disk && bdev->bd_disk->bdi) 328 bdi_put(bdev->bd_disk->bdi); 329 kfree(bdev->bd_disk); 330 } 331 332 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 333 blk_free_ext_minor(MINOR(bdev->bd_dev)); 334 335 kmem_cache_free(bdev_cachep, BDEV_I(inode)); 336 } 337 338 static void init_once(void *data) 339 { 340 struct bdev_inode *ei = data; 341 342 inode_init_once(&ei->vfs_inode); 343 } 344 345 static void bdev_evict_inode(struct inode *inode) 346 { 347 truncate_inode_pages_final(&inode->i_data); 348 invalidate_inode_buffers(inode); /* is it needed here? */ 349 clear_inode(inode); 350 } 351 352 static const struct super_operations bdev_sops = { 353 .statfs = simple_statfs, 354 .alloc_inode = bdev_alloc_inode, 355 .free_inode = bdev_free_inode, 356 .drop_inode = generic_delete_inode, 357 .evict_inode = bdev_evict_inode, 358 }; 359 360 static int bd_init_fs_context(struct fs_context *fc) 361 { 362 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC); 363 if (!ctx) 364 return -ENOMEM; 365 fc->s_iflags |= SB_I_CGROUPWB; 366 ctx->ops = &bdev_sops; 367 return 0; 368 } 369 370 static struct file_system_type bd_type = { 371 .name = "bdev", 372 .init_fs_context = bd_init_fs_context, 373 .kill_sb = kill_anon_super, 374 }; 375 376 struct super_block *blockdev_superblock __ro_after_init; 377 struct vfsmount *blockdev_mnt __ro_after_init; 378 EXPORT_SYMBOL_GPL(blockdev_superblock); 379 380 void __init bdev_cache_init(void) 381 { 382 int err; 383 384 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 385 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 386 SLAB_ACCOUNT|SLAB_PANIC), 387 init_once); 388 err = register_filesystem(&bd_type); 389 if (err) 390 panic("Cannot register bdev pseudo-fs"); 391 blockdev_mnt = kern_mount(&bd_type); 392 if (IS_ERR(blockdev_mnt)) 393 panic("Cannot create bdev pseudo-fs"); 394 blockdev_superblock = blockdev_mnt->mnt_sb; /* For writeback */ 395 } 396 397 struct block_device *bdev_alloc(struct gendisk *disk, u8 partno) 398 { 399 struct block_device *bdev; 400 struct inode *inode; 401 402 inode = new_inode(blockdev_superblock); 403 if (!inode) 404 return NULL; 405 inode->i_mode = S_IFBLK; 406 inode->i_rdev = 0; 407 inode->i_data.a_ops = &def_blk_aops; 408 mapping_set_gfp_mask(&inode->i_data, GFP_USER); 409 410 bdev = I_BDEV(inode); 411 mutex_init(&bdev->bd_fsfreeze_mutex); 412 spin_lock_init(&bdev->bd_size_lock); 413 mutex_init(&bdev->bd_holder_lock); 414 bdev->bd_partno = partno; 415 bdev->bd_inode = inode; 416 bdev->bd_queue = disk->queue; 417 if (partno) 418 bdev->bd_has_submit_bio = disk->part0->bd_has_submit_bio; 419 else 420 bdev->bd_has_submit_bio = false; 421 bdev->bd_stats = alloc_percpu(struct disk_stats); 422 if (!bdev->bd_stats) { 423 iput(inode); 424 return NULL; 425 } 426 bdev->bd_disk = disk; 427 return bdev; 428 } 429 430 void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors) 431 { 432 spin_lock(&bdev->bd_size_lock); 433 i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT); 434 bdev->bd_nr_sectors = sectors; 435 spin_unlock(&bdev->bd_size_lock); 436 } 437 438 void bdev_add(struct block_device *bdev, dev_t dev) 439 { 440 if (bdev_stable_writes(bdev)) 441 mapping_set_stable_writes(bdev->bd_inode->i_mapping); 442 bdev->bd_dev = dev; 443 bdev->bd_inode->i_rdev = dev; 444 bdev->bd_inode->i_ino = dev; 445 insert_inode_hash(bdev->bd_inode); 446 } 447 448 long nr_blockdev_pages(void) 449 { 450 struct inode *inode; 451 long ret = 0; 452 453 spin_lock(&blockdev_superblock->s_inode_list_lock); 454 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) 455 ret += inode->i_mapping->nrpages; 456 spin_unlock(&blockdev_superblock->s_inode_list_lock); 457 458 return ret; 459 } 460 461 /** 462 * bd_may_claim - test whether a block device can be claimed 463 * @bdev: block device of interest 464 * @holder: holder trying to claim @bdev 465 * @hops: holder ops 466 * 467 * Test whether @bdev can be claimed by @holder. 468 * 469 * RETURNS: 470 * %true if @bdev can be claimed, %false otherwise. 471 */ 472 static bool bd_may_claim(struct block_device *bdev, void *holder, 473 const struct blk_holder_ops *hops) 474 { 475 struct block_device *whole = bdev_whole(bdev); 476 477 lockdep_assert_held(&bdev_lock); 478 479 if (bdev->bd_holder) { 480 /* 481 * The same holder can always re-claim. 482 */ 483 if (bdev->bd_holder == holder) { 484 if (WARN_ON_ONCE(bdev->bd_holder_ops != hops)) 485 return false; 486 return true; 487 } 488 return false; 489 } 490 491 /* 492 * If the whole devices holder is set to bd_may_claim, a partition on 493 * the device is claimed, but not the whole device. 494 */ 495 if (whole != bdev && 496 whole->bd_holder && whole->bd_holder != bd_may_claim) 497 return false; 498 return true; 499 } 500 501 /** 502 * bd_prepare_to_claim - claim a block device 503 * @bdev: block device of interest 504 * @holder: holder trying to claim @bdev 505 * @hops: holder ops. 506 * 507 * Claim @bdev. This function fails if @bdev is already claimed by another 508 * holder and waits if another claiming is in progress. return, the caller 509 * has ownership of bd_claiming and bd_holder[s]. 510 * 511 * RETURNS: 512 * 0 if @bdev can be claimed, -EBUSY otherwise. 513 */ 514 int bd_prepare_to_claim(struct block_device *bdev, void *holder, 515 const struct blk_holder_ops *hops) 516 { 517 struct block_device *whole = bdev_whole(bdev); 518 519 if (WARN_ON_ONCE(!holder)) 520 return -EINVAL; 521 retry: 522 mutex_lock(&bdev_lock); 523 /* if someone else claimed, fail */ 524 if (!bd_may_claim(bdev, holder, hops)) { 525 mutex_unlock(&bdev_lock); 526 return -EBUSY; 527 } 528 529 /* if claiming is already in progress, wait for it to finish */ 530 if (whole->bd_claiming) { 531 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 532 DEFINE_WAIT(wait); 533 534 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 535 mutex_unlock(&bdev_lock); 536 schedule(); 537 finish_wait(wq, &wait); 538 goto retry; 539 } 540 541 /* yay, all mine */ 542 whole->bd_claiming = holder; 543 mutex_unlock(&bdev_lock); 544 return 0; 545 } 546 EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */ 547 548 static void bd_clear_claiming(struct block_device *whole, void *holder) 549 { 550 lockdep_assert_held(&bdev_lock); 551 /* tell others that we're done */ 552 BUG_ON(whole->bd_claiming != holder); 553 whole->bd_claiming = NULL; 554 wake_up_bit(&whole->bd_claiming, 0); 555 } 556 557 /** 558 * bd_finish_claiming - finish claiming of a block device 559 * @bdev: block device of interest 560 * @holder: holder that has claimed @bdev 561 * @hops: block device holder operations 562 * 563 * Finish exclusive open of a block device. Mark the device as exlusively 564 * open by the holder and wake up all waiters for exclusive open to finish. 565 */ 566 static void bd_finish_claiming(struct block_device *bdev, void *holder, 567 const struct blk_holder_ops *hops) 568 { 569 struct block_device *whole = bdev_whole(bdev); 570 571 mutex_lock(&bdev_lock); 572 BUG_ON(!bd_may_claim(bdev, holder, hops)); 573 /* 574 * Note that for a whole device bd_holders will be incremented twice, 575 * and bd_holder will be set to bd_may_claim before being set to holder 576 */ 577 whole->bd_holders++; 578 whole->bd_holder = bd_may_claim; 579 bdev->bd_holders++; 580 mutex_lock(&bdev->bd_holder_lock); 581 bdev->bd_holder = holder; 582 bdev->bd_holder_ops = hops; 583 mutex_unlock(&bdev->bd_holder_lock); 584 bd_clear_claiming(whole, holder); 585 mutex_unlock(&bdev_lock); 586 } 587 588 /** 589 * bd_abort_claiming - abort claiming of a block device 590 * @bdev: block device of interest 591 * @holder: holder that has claimed @bdev 592 * 593 * Abort claiming of a block device when the exclusive open failed. This can be 594 * also used when exclusive open is not actually desired and we just needed 595 * to block other exclusive openers for a while. 596 */ 597 void bd_abort_claiming(struct block_device *bdev, void *holder) 598 { 599 mutex_lock(&bdev_lock); 600 bd_clear_claiming(bdev_whole(bdev), holder); 601 mutex_unlock(&bdev_lock); 602 } 603 EXPORT_SYMBOL(bd_abort_claiming); 604 605 static void bd_end_claim(struct block_device *bdev, void *holder) 606 { 607 struct block_device *whole = bdev_whole(bdev); 608 bool unblock = false; 609 610 /* 611 * Release a claim on the device. The holder fields are protected with 612 * bdev_lock. open_mutex is used to synchronize disk_holder unlinking. 613 */ 614 mutex_lock(&bdev_lock); 615 WARN_ON_ONCE(bdev->bd_holder != holder); 616 WARN_ON_ONCE(--bdev->bd_holders < 0); 617 WARN_ON_ONCE(--whole->bd_holders < 0); 618 if (!bdev->bd_holders) { 619 mutex_lock(&bdev->bd_holder_lock); 620 bdev->bd_holder = NULL; 621 bdev->bd_holder_ops = NULL; 622 mutex_unlock(&bdev->bd_holder_lock); 623 if (bdev->bd_write_holder) 624 unblock = true; 625 } 626 if (!whole->bd_holders) 627 whole->bd_holder = NULL; 628 mutex_unlock(&bdev_lock); 629 630 /* 631 * If this was the last claim, remove holder link and unblock evpoll if 632 * it was a write holder. 633 */ 634 if (unblock) { 635 disk_unblock_events(bdev->bd_disk); 636 bdev->bd_write_holder = false; 637 } 638 } 639 640 static void blkdev_flush_mapping(struct block_device *bdev) 641 { 642 WARN_ON_ONCE(bdev->bd_holders); 643 sync_blockdev(bdev); 644 kill_bdev(bdev); 645 bdev_write_inode(bdev); 646 } 647 648 static int blkdev_get_whole(struct block_device *bdev, blk_mode_t mode) 649 { 650 struct gendisk *disk = bdev->bd_disk; 651 int ret; 652 653 if (disk->fops->open) { 654 ret = disk->fops->open(disk, mode); 655 if (ret) { 656 /* avoid ghost partitions on a removed medium */ 657 if (ret == -ENOMEDIUM && 658 test_bit(GD_NEED_PART_SCAN, &disk->state)) 659 bdev_disk_changed(disk, true); 660 return ret; 661 } 662 } 663 664 if (!atomic_read(&bdev->bd_openers)) 665 set_init_blocksize(bdev); 666 if (test_bit(GD_NEED_PART_SCAN, &disk->state)) 667 bdev_disk_changed(disk, false); 668 atomic_inc(&bdev->bd_openers); 669 return 0; 670 } 671 672 static void blkdev_put_whole(struct block_device *bdev) 673 { 674 if (atomic_dec_and_test(&bdev->bd_openers)) 675 blkdev_flush_mapping(bdev); 676 if (bdev->bd_disk->fops->release) 677 bdev->bd_disk->fops->release(bdev->bd_disk); 678 } 679 680 static int blkdev_get_part(struct block_device *part, blk_mode_t mode) 681 { 682 struct gendisk *disk = part->bd_disk; 683 int ret; 684 685 ret = blkdev_get_whole(bdev_whole(part), mode); 686 if (ret) 687 return ret; 688 689 ret = -ENXIO; 690 if (!bdev_nr_sectors(part)) 691 goto out_blkdev_put; 692 693 if (!atomic_read(&part->bd_openers)) { 694 disk->open_partitions++; 695 set_init_blocksize(part); 696 } 697 atomic_inc(&part->bd_openers); 698 return 0; 699 700 out_blkdev_put: 701 blkdev_put_whole(bdev_whole(part)); 702 return ret; 703 } 704 705 int bdev_permission(dev_t dev, blk_mode_t mode, void *holder) 706 { 707 int ret; 708 709 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK, 710 MAJOR(dev), MINOR(dev), 711 ((mode & BLK_OPEN_READ) ? DEVCG_ACC_READ : 0) | 712 ((mode & BLK_OPEN_WRITE) ? DEVCG_ACC_WRITE : 0)); 713 if (ret) 714 return ret; 715 716 /* Blocking writes requires exclusive opener */ 717 if (mode & BLK_OPEN_RESTRICT_WRITES && !holder) 718 return -EINVAL; 719 720 /* 721 * We're using error pointers to indicate to ->release() when we 722 * failed to open that block device. Also this doesn't make sense. 723 */ 724 if (WARN_ON_ONCE(IS_ERR(holder))) 725 return -EINVAL; 726 727 return 0; 728 } 729 730 static void blkdev_put_part(struct block_device *part) 731 { 732 struct block_device *whole = bdev_whole(part); 733 734 if (atomic_dec_and_test(&part->bd_openers)) { 735 blkdev_flush_mapping(part); 736 whole->bd_disk->open_partitions--; 737 } 738 blkdev_put_whole(whole); 739 } 740 741 struct block_device *blkdev_get_no_open(dev_t dev) 742 { 743 struct block_device *bdev; 744 struct inode *inode; 745 746 inode = ilookup(blockdev_superblock, dev); 747 if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) { 748 blk_request_module(dev); 749 inode = ilookup(blockdev_superblock, dev); 750 if (inode) 751 pr_warn_ratelimited( 752 "block device autoloading is deprecated and will be removed.\n"); 753 } 754 if (!inode) 755 return NULL; 756 757 /* switch from the inode reference to a device mode one: */ 758 bdev = &BDEV_I(inode)->bdev; 759 if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) 760 bdev = NULL; 761 iput(inode); 762 return bdev; 763 } 764 765 void blkdev_put_no_open(struct block_device *bdev) 766 { 767 put_device(&bdev->bd_device); 768 } 769 770 static bool bdev_writes_blocked(struct block_device *bdev) 771 { 772 return bdev->bd_writers < 0; 773 } 774 775 static void bdev_block_writes(struct block_device *bdev) 776 { 777 bdev->bd_writers--; 778 } 779 780 static void bdev_unblock_writes(struct block_device *bdev) 781 { 782 bdev->bd_writers++; 783 } 784 785 static bool bdev_may_open(struct block_device *bdev, blk_mode_t mode) 786 { 787 if (bdev_allow_write_mounted) 788 return true; 789 /* Writes blocked? */ 790 if (mode & BLK_OPEN_WRITE && bdev_writes_blocked(bdev)) 791 return false; 792 if (mode & BLK_OPEN_RESTRICT_WRITES && bdev->bd_writers > 0) 793 return false; 794 return true; 795 } 796 797 static void bdev_claim_write_access(struct block_device *bdev, blk_mode_t mode) 798 { 799 if (bdev_allow_write_mounted) 800 return; 801 802 /* Claim exclusive or shared write access. */ 803 if (mode & BLK_OPEN_RESTRICT_WRITES) 804 bdev_block_writes(bdev); 805 else if (mode & BLK_OPEN_WRITE) 806 bdev->bd_writers++; 807 } 808 809 static inline bool bdev_unclaimed(const struct file *bdev_file) 810 { 811 return bdev_file->private_data == BDEV_I(bdev_file->f_mapping->host); 812 } 813 814 static void bdev_yield_write_access(struct file *bdev_file) 815 { 816 struct block_device *bdev; 817 818 if (bdev_allow_write_mounted) 819 return; 820 821 if (bdev_unclaimed(bdev_file)) 822 return; 823 824 bdev = file_bdev(bdev_file); 825 826 if (bdev_file->f_mode & FMODE_WRITE_RESTRICTED) 827 bdev_unblock_writes(bdev); 828 else if (bdev_file->f_mode & FMODE_WRITE) 829 bdev->bd_writers--; 830 } 831 832 /** 833 * bdev_open - open a block device 834 * @bdev: block device to open 835 * @mode: open mode (BLK_OPEN_*) 836 * @holder: exclusive holder identifier 837 * @hops: holder operations 838 * @bdev_file: file for the block device 839 * 840 * Open the block device. If @holder is not %NULL, the block device is opened 841 * with exclusive access. Exclusive opens may nest for the same @holder. 842 * 843 * CONTEXT: 844 * Might sleep. 845 * 846 * RETURNS: 847 * zero on success, -errno on failure. 848 */ 849 int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder, 850 const struct blk_holder_ops *hops, struct file *bdev_file) 851 { 852 bool unblock_events = true; 853 struct gendisk *disk = bdev->bd_disk; 854 int ret; 855 856 if (holder) { 857 mode |= BLK_OPEN_EXCL; 858 ret = bd_prepare_to_claim(bdev, holder, hops); 859 if (ret) 860 return ret; 861 } else { 862 if (WARN_ON_ONCE(mode & BLK_OPEN_EXCL)) 863 return -EIO; 864 } 865 866 disk_block_events(disk); 867 868 mutex_lock(&disk->open_mutex); 869 ret = -ENXIO; 870 if (!disk_live(disk)) 871 goto abort_claiming; 872 if (!try_module_get(disk->fops->owner)) 873 goto abort_claiming; 874 ret = -EBUSY; 875 if (!bdev_may_open(bdev, mode)) 876 goto abort_claiming; 877 if (bdev_is_partition(bdev)) 878 ret = blkdev_get_part(bdev, mode); 879 else 880 ret = blkdev_get_whole(bdev, mode); 881 if (ret) 882 goto put_module; 883 bdev_claim_write_access(bdev, mode); 884 if (holder) { 885 bd_finish_claiming(bdev, holder, hops); 886 887 /* 888 * Block event polling for write claims if requested. Any write 889 * holder makes the write_holder state stick until all are 890 * released. This is good enough and tracking individual 891 * writeable reference is too fragile given the way @mode is 892 * used in blkdev_get/put(). 893 */ 894 if ((mode & BLK_OPEN_WRITE) && !bdev->bd_write_holder && 895 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) { 896 bdev->bd_write_holder = true; 897 unblock_events = false; 898 } 899 } 900 mutex_unlock(&disk->open_mutex); 901 902 if (unblock_events) 903 disk_unblock_events(disk); 904 905 bdev_file->f_flags |= O_LARGEFILE; 906 bdev_file->f_mode |= FMODE_BUF_RASYNC | FMODE_CAN_ODIRECT; 907 if (bdev_nowait(bdev)) 908 bdev_file->f_mode |= FMODE_NOWAIT; 909 if (mode & BLK_OPEN_RESTRICT_WRITES) 910 bdev_file->f_mode |= FMODE_WRITE_RESTRICTED; 911 bdev_file->f_mapping = bdev->bd_inode->i_mapping; 912 bdev_file->f_wb_err = filemap_sample_wb_err(bdev_file->f_mapping); 913 bdev_file->private_data = holder; 914 915 return 0; 916 put_module: 917 module_put(disk->fops->owner); 918 abort_claiming: 919 if (holder) 920 bd_abort_claiming(bdev, holder); 921 mutex_unlock(&disk->open_mutex); 922 disk_unblock_events(disk); 923 return ret; 924 } 925 926 /* 927 * If BLK_OPEN_WRITE_IOCTL is set then this is a historical quirk 928 * associated with the floppy driver where it has allowed ioctls if the 929 * file was opened for writing, but does not allow reads or writes. 930 * Make sure that this quirk is reflected in @f_flags. 931 * 932 * It can also happen if a block device is opened as O_RDWR | O_WRONLY. 933 */ 934 static unsigned blk_to_file_flags(blk_mode_t mode) 935 { 936 unsigned int flags = 0; 937 938 if ((mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) == 939 (BLK_OPEN_READ | BLK_OPEN_WRITE)) 940 flags |= O_RDWR; 941 else if (mode & BLK_OPEN_WRITE_IOCTL) 942 flags |= O_RDWR | O_WRONLY; 943 else if (mode & BLK_OPEN_WRITE) 944 flags |= O_WRONLY; 945 else if (mode & BLK_OPEN_READ) 946 flags |= O_RDONLY; /* homeopathic, because O_RDONLY is 0 */ 947 else 948 WARN_ON_ONCE(true); 949 950 if (mode & BLK_OPEN_NDELAY) 951 flags |= O_NDELAY; 952 953 return flags; 954 } 955 956 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, 957 const struct blk_holder_ops *hops) 958 { 959 struct file *bdev_file; 960 struct block_device *bdev; 961 unsigned int flags; 962 int ret; 963 964 ret = bdev_permission(dev, mode, holder); 965 if (ret) 966 return ERR_PTR(ret); 967 968 bdev = blkdev_get_no_open(dev); 969 if (!bdev) 970 return ERR_PTR(-ENXIO); 971 972 flags = blk_to_file_flags(mode); 973 bdev_file = alloc_file_pseudo_noaccount(bdev->bd_inode, 974 blockdev_mnt, "", flags | O_LARGEFILE, &def_blk_fops); 975 if (IS_ERR(bdev_file)) { 976 blkdev_put_no_open(bdev); 977 return bdev_file; 978 } 979 ihold(bdev->bd_inode); 980 981 ret = bdev_open(bdev, mode, holder, hops, bdev_file); 982 if (ret) { 983 /* We failed to open the block device. Let ->release() know. */ 984 bdev_file->private_data = ERR_PTR(ret); 985 fput(bdev_file); 986 return ERR_PTR(ret); 987 } 988 return bdev_file; 989 } 990 EXPORT_SYMBOL(bdev_file_open_by_dev); 991 992 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode, 993 void *holder, 994 const struct blk_holder_ops *hops) 995 { 996 struct file *file; 997 dev_t dev; 998 int error; 999 1000 error = lookup_bdev(path, &dev); 1001 if (error) 1002 return ERR_PTR(error); 1003 1004 file = bdev_file_open_by_dev(dev, mode, holder, hops); 1005 if (!IS_ERR(file) && (mode & BLK_OPEN_WRITE)) { 1006 if (bdev_read_only(file_bdev(file))) { 1007 fput(file); 1008 file = ERR_PTR(-EACCES); 1009 } 1010 } 1011 1012 return file; 1013 } 1014 EXPORT_SYMBOL(bdev_file_open_by_path); 1015 1016 static inline void bd_yield_claim(struct file *bdev_file) 1017 { 1018 struct block_device *bdev = file_bdev(bdev_file); 1019 void *holder = bdev_file->private_data; 1020 1021 lockdep_assert_held(&bdev->bd_disk->open_mutex); 1022 1023 if (WARN_ON_ONCE(IS_ERR_OR_NULL(holder))) 1024 return; 1025 1026 if (!bdev_unclaimed(bdev_file)) 1027 bd_end_claim(bdev, holder); 1028 } 1029 1030 void bdev_release(struct file *bdev_file) 1031 { 1032 struct block_device *bdev = file_bdev(bdev_file); 1033 void *holder = bdev_file->private_data; 1034 struct gendisk *disk = bdev->bd_disk; 1035 1036 /* We failed to open that block device. */ 1037 if (IS_ERR(holder)) 1038 goto put_no_open; 1039 1040 /* 1041 * Sync early if it looks like we're the last one. If someone else 1042 * opens the block device between now and the decrement of bd_openers 1043 * then we did a sync that we didn't need to, but that's not the end 1044 * of the world and we want to avoid long (could be several minute) 1045 * syncs while holding the mutex. 1046 */ 1047 if (atomic_read(&bdev->bd_openers) == 1) 1048 sync_blockdev(bdev); 1049 1050 mutex_lock(&disk->open_mutex); 1051 bdev_yield_write_access(bdev_file); 1052 1053 if (holder) 1054 bd_yield_claim(bdev_file); 1055 1056 /* 1057 * Trigger event checking and tell drivers to flush MEDIA_CHANGE 1058 * event. This is to ensure detection of media removal commanded 1059 * from userland - e.g. eject(1). 1060 */ 1061 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE); 1062 1063 if (bdev_is_partition(bdev)) 1064 blkdev_put_part(bdev); 1065 else 1066 blkdev_put_whole(bdev); 1067 mutex_unlock(&disk->open_mutex); 1068 1069 module_put(disk->fops->owner); 1070 put_no_open: 1071 blkdev_put_no_open(bdev); 1072 } 1073 1074 /** 1075 * bdev_fput - yield claim to the block device and put the file 1076 * @bdev_file: open block device 1077 * 1078 * Yield claim on the block device and put the file. Ensure that the 1079 * block device can be reclaimed before the file is closed which is a 1080 * deferred operation. 1081 */ 1082 void bdev_fput(struct file *bdev_file) 1083 { 1084 if (WARN_ON_ONCE(bdev_file->f_op != &def_blk_fops)) 1085 return; 1086 1087 if (bdev_file->private_data) { 1088 struct block_device *bdev = file_bdev(bdev_file); 1089 struct gendisk *disk = bdev->bd_disk; 1090 1091 mutex_lock(&disk->open_mutex); 1092 bdev_yield_write_access(bdev_file); 1093 bd_yield_claim(bdev_file); 1094 /* 1095 * Tell release we already gave up our hold on the 1096 * device and if write restrictions are available that 1097 * we already gave up write access to the device. 1098 */ 1099 bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host); 1100 mutex_unlock(&disk->open_mutex); 1101 } 1102 1103 fput(bdev_file); 1104 } 1105 EXPORT_SYMBOL(bdev_fput); 1106 1107 /** 1108 * lookup_bdev() - Look up a struct block_device by name. 1109 * @pathname: Name of the block device in the filesystem. 1110 * @dev: Pointer to the block device's dev_t, if found. 1111 * 1112 * Lookup the block device's dev_t at @pathname in the current 1113 * namespace if possible and return it in @dev. 1114 * 1115 * Context: May sleep. 1116 * Return: 0 if succeeded, negative errno otherwise. 1117 */ 1118 int lookup_bdev(const char *pathname, dev_t *dev) 1119 { 1120 struct inode *inode; 1121 struct path path; 1122 int error; 1123 1124 if (!pathname || !*pathname) 1125 return -EINVAL; 1126 1127 error = kern_path(pathname, LOOKUP_FOLLOW, &path); 1128 if (error) 1129 return error; 1130 1131 inode = d_backing_inode(path.dentry); 1132 error = -ENOTBLK; 1133 if (!S_ISBLK(inode->i_mode)) 1134 goto out_path_put; 1135 error = -EACCES; 1136 if (!may_open_dev(&path)) 1137 goto out_path_put; 1138 1139 *dev = inode->i_rdev; 1140 error = 0; 1141 out_path_put: 1142 path_put(&path); 1143 return error; 1144 } 1145 EXPORT_SYMBOL(lookup_bdev); 1146 1147 /** 1148 * bdev_mark_dead - mark a block device as dead 1149 * @bdev: block device to operate on 1150 * @surprise: indicate a surprise removal 1151 * 1152 * Tell the file system that this devices or media is dead. If @surprise is set 1153 * to %true the device or media is already gone, if not we are preparing for an 1154 * orderly removal. 1155 * 1156 * This calls into the file system, which then typicall syncs out all dirty data 1157 * and writes back inodes and then invalidates any cached data in the inodes on 1158 * the file system. In addition we also invalidate the block device mapping. 1159 */ 1160 void bdev_mark_dead(struct block_device *bdev, bool surprise) 1161 { 1162 mutex_lock(&bdev->bd_holder_lock); 1163 if (bdev->bd_holder_ops && bdev->bd_holder_ops->mark_dead) 1164 bdev->bd_holder_ops->mark_dead(bdev, surprise); 1165 else { 1166 mutex_unlock(&bdev->bd_holder_lock); 1167 sync_blockdev(bdev); 1168 } 1169 1170 invalidate_bdev(bdev); 1171 } 1172 /* 1173 * New drivers should not use this directly. There are some drivers however 1174 * that needs this for historical reasons. For example, the DASD driver has 1175 * historically had a shutdown to offline mode that doesn't actually remove the 1176 * gendisk that otherwise looks a lot like a safe device removal. 1177 */ 1178 EXPORT_SYMBOL_GPL(bdev_mark_dead); 1179 1180 void sync_bdevs(bool wait) 1181 { 1182 struct inode *inode, *old_inode = NULL; 1183 1184 spin_lock(&blockdev_superblock->s_inode_list_lock); 1185 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 1186 struct address_space *mapping = inode->i_mapping; 1187 struct block_device *bdev; 1188 1189 spin_lock(&inode->i_lock); 1190 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 1191 mapping->nrpages == 0) { 1192 spin_unlock(&inode->i_lock); 1193 continue; 1194 } 1195 __iget(inode); 1196 spin_unlock(&inode->i_lock); 1197 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1198 /* 1199 * We hold a reference to 'inode' so it couldn't have been 1200 * removed from s_inodes list while we dropped the 1201 * s_inode_list_lock We cannot iput the inode now as we can 1202 * be holding the last reference and we cannot iput it under 1203 * s_inode_list_lock. So we keep the reference and iput it 1204 * later. 1205 */ 1206 iput(old_inode); 1207 old_inode = inode; 1208 bdev = I_BDEV(inode); 1209 1210 mutex_lock(&bdev->bd_disk->open_mutex); 1211 if (!atomic_read(&bdev->bd_openers)) { 1212 ; /* skip */ 1213 } else if (wait) { 1214 /* 1215 * We keep the error status of individual mapping so 1216 * that applications can catch the writeback error using 1217 * fsync(2). See filemap_fdatawait_keep_errors() for 1218 * details. 1219 */ 1220 filemap_fdatawait_keep_errors(inode->i_mapping); 1221 } else { 1222 filemap_fdatawrite(inode->i_mapping); 1223 } 1224 mutex_unlock(&bdev->bd_disk->open_mutex); 1225 1226 spin_lock(&blockdev_superblock->s_inode_list_lock); 1227 } 1228 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1229 iput(old_inode); 1230 } 1231 1232 /* 1233 * Handle STATX_DIOALIGN for block devices. 1234 * 1235 * Note that the inode passed to this is the inode of a block device node file, 1236 * not the block device's internal inode. Therefore it is *not* valid to use 1237 * I_BDEV() here; the block device has to be looked up by i_rdev instead. 1238 */ 1239 void bdev_statx_dioalign(struct inode *inode, struct kstat *stat) 1240 { 1241 struct block_device *bdev; 1242 1243 bdev = blkdev_get_no_open(inode->i_rdev); 1244 if (!bdev) 1245 return; 1246 1247 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; 1248 stat->dio_offset_align = bdev_logical_block_size(bdev); 1249 stat->result_mask |= STATX_DIOALIGN; 1250 1251 blkdev_put_no_open(bdev); 1252 } 1253 1254 static int __init setup_bdev_allow_write_mounted(char *str) 1255 { 1256 if (kstrtobool(str, &bdev_allow_write_mounted)) 1257 pr_warn("Invalid option string for bdev_allow_write_mounted:" 1258 " '%s'\n", str); 1259 return 1; 1260 } 1261 __setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted); 1262