1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Portions Copyright (C) 1992 Drew Eckhardt 4 */ 5 #ifndef _LINUX_BLKDEV_H 6 #define _LINUX_BLKDEV_H 7 8 #include <linux/types.h> 9 #include <linux/blk_types.h> 10 #include <linux/blk_plug.h> 11 #include <linux/device.h> 12 #include <linux/list.h> 13 #include <linux/llist.h> 14 #include <linux/minmax.h> 15 #include <linux/timer.h> 16 #include <linux/workqueue.h> 17 #include <linux/completion.h> 18 #include <linux/wait.h> 19 #include <linux/bio.h> 20 #include <linux/gfp.h> 21 #include <linux/kdev_t.h> 22 #include <linux/rcupdate.h> 23 #include <linux/percpu-refcount.h> 24 #include <linux/blkzoned.h> 25 #include <linux/sbitmap.h> 26 #include <linux/uuid.h> 27 #include <linux/xarray.h> 28 #include <linux/file.h> 29 #include <linux/lockdep.h> 30 31 struct module; 32 struct request_queue; 33 struct elevator_queue; 34 struct blk_trace; 35 struct request; 36 struct sg_io_hdr; 37 struct blkcg_gq; 38 struct blk_flush_queue; 39 struct kiocb; 40 struct pr_ops; 41 struct rq_qos; 42 struct hd_geometry; 43 struct blk_report_zones_args; 44 struct blk_queue_stats; 45 struct blk_stat_callback; 46 struct blk_crypto_profile; 47 48 extern const struct device_type disk_type; 49 extern const struct device_type part_type; 50 extern const struct class block_class; 51 52 /* 53 * Maximum number of blkcg policies allowed to be registered concurrently. 54 * Defined here to simplify include dependency. 55 */ 56 #define BLKCG_MAX_POLS 6 57 58 #define DISK_MAX_PARTS 256 59 #define DISK_NAME_LEN 32 60 61 #define PARTITION_META_INFO_VOLNAMELTH 64 62 /* 63 * Enough for the string representation of any kind of UUID plus NULL. 64 * EFI UUID is 36 characters. MSDOS UUID is 11 characters. 65 */ 66 #define PARTITION_META_INFO_UUIDLTH (UUID_STRING_LEN + 1) 67 68 struct partition_meta_info { 69 char uuid[PARTITION_META_INFO_UUIDLTH]; 70 u8 volname[PARTITION_META_INFO_VOLNAMELTH]; 71 }; 72 73 /** 74 * DOC: genhd capability flags 75 * 76 * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to 77 * removable media. When set, the device remains present even when media is not 78 * inserted. Shall not be set for devices which are removed entirely when the 79 * media is removed. 80 * 81 * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events, 82 * doesn't appear in sysfs, and can't be opened from userspace or using 83 * blkdev_get*. Used for the underlying components of multipath devices. 84 * 85 * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not 86 * scan for partitions from add_disk, and users can't add partitions manually. 87 * 88 */ 89 enum { 90 GENHD_FL_REMOVABLE = 1 << 0, 91 GENHD_FL_HIDDEN = 1 << 1, 92 GENHD_FL_NO_PART = 1 << 2, 93 }; 94 95 enum { 96 DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */ 97 DISK_EVENT_EJECT_REQUEST = 1 << 1, /* eject requested */ 98 }; 99 100 enum { 101 /* Poll even if events_poll_msecs is unset */ 102 DISK_EVENT_FLAG_POLL = 1 << 0, 103 /* Forward events to udev */ 104 DISK_EVENT_FLAG_UEVENT = 1 << 1, 105 /* Block event polling when open for exclusive write */ 106 DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2, 107 }; 108 109 struct disk_events; 110 struct badblocks; 111 112 enum blk_integrity_checksum { 113 BLK_INTEGRITY_CSUM_NONE = 0, 114 BLK_INTEGRITY_CSUM_IP = 1, 115 BLK_INTEGRITY_CSUM_CRC = 2, 116 BLK_INTEGRITY_CSUM_CRC64 = 3, 117 } __packed ; 118 119 struct blk_integrity { 120 unsigned char flags; 121 enum blk_integrity_checksum csum_type; 122 unsigned char metadata_size; 123 unsigned char pi_offset; 124 unsigned char interval_exp; 125 unsigned char tag_size; 126 unsigned char pi_tuple_size; 127 }; 128 129 /* open for reading */ 130 #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0)) 131 /* open for writing */ 132 #define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1)) 133 /* open exclusively (vs other exclusive openers */ 134 #define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2)) 135 /* opened with O_NDELAY */ 136 #define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3)) 137 /* open for "writes" only for ioctls (specialy hack for floppy.c) */ 138 #define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4)) 139 /* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */ 140 #define BLK_OPEN_RESTRICT_WRITES ((__force blk_mode_t)(1 << 5)) 141 /* return partition scanning errors */ 142 #define BLK_OPEN_STRICT_SCAN ((__force blk_mode_t)(1 << 6)) 143 144 struct gendisk { 145 /* 146 * major/first_minor/minors should not be set by any new driver, the 147 * block core will take care of allocating them automatically. 148 */ 149 int major; 150 int first_minor; 151 int minors; 152 153 char disk_name[DISK_NAME_LEN]; /* name of major driver */ 154 155 unsigned short events; /* supported events */ 156 unsigned short event_flags; /* flags related to event processing */ 157 158 struct xarray part_tbl; 159 struct block_device *part0; 160 161 const struct block_device_operations *fops; 162 struct request_queue *queue; 163 void *private_data; 164 165 struct bio_set bio_split; 166 167 int flags; 168 unsigned long state; 169 #define GD_NEED_PART_SCAN 0 170 #define GD_READ_ONLY 1 171 #define GD_DEAD 2 172 #define GD_NATIVE_CAPACITY 3 173 #define GD_ADDED 4 174 #define GD_SUPPRESS_PART_SCAN 5 175 #define GD_OWNS_QUEUE 6 176 #define GD_ZONE_APPEND_USED 7 177 #define GD_ERROR_INJECT 8 178 179 struct mutex open_mutex; /* open/close mutex */ 180 unsigned open_partitions; /* number of open partitions */ 181 182 struct backing_dev_info *bdi; 183 struct kobject queue_kobj; /* the queue/ directory */ 184 struct kobject *slave_dir; 185 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 186 struct list_head slave_bdevs; 187 #endif 188 struct timer_rand_state *random; 189 struct disk_events *ev; 190 191 #ifdef CONFIG_BLK_DEV_ZONED 192 /* 193 * Zoned block device information. Reads of this information must be 194 * protected with blk_queue_enter() / blk_queue_exit() or by holding a 195 * lock on zone_revalidate_mutex. blk_revalidate_disk_zones() may modify 196 * this information while no requests are being processed (disk queue 197 * frozen with blk_mq_freeze_queue()) and while holding a lock on 198 * zone_revalidate_mutex. 199 */ 200 struct mutex zone_revalidate_mutex; 201 unsigned int nr_zones; 202 unsigned int zone_capacity; 203 unsigned int last_zone_capacity; 204 u8 __rcu *zones_state; 205 unsigned int zone_wplugs_hash_bits; 206 atomic_t nr_zone_wplugs; 207 spinlock_t zone_wplugs_hash_lock; 208 struct mempool *zone_wplugs_pool; 209 struct hlist_head *zone_wplugs_hash; 210 struct workqueue_struct *zone_wplugs_wq; 211 spinlock_t zone_wplugs_list_lock; 212 struct list_head zone_wplugs_list; 213 struct task_struct *zone_wplugs_worker; 214 struct completion zone_wplugs_worker_bio_done; 215 #endif /* CONFIG_BLK_DEV_ZONED */ 216 217 #if IS_ENABLED(CONFIG_CDROM) 218 struct cdrom_device_info *cdi; 219 #endif 220 int node_id; 221 struct badblocks *bb; 222 struct lockdep_map lockdep_map; 223 u64 diskseq; 224 blk_mode_t open_mode; 225 226 /* 227 * Independent sector access ranges. This is always NULL for 228 * devices that do not have multiple independent access ranges. 229 */ 230 struct blk_independent_access_ranges *ia_ranges; 231 232 #ifdef CONFIG_BLK_ERROR_INJECTION 233 struct mutex error_injection_lock; 234 struct list_head error_injection_list; 235 #endif 236 237 struct mutex rqos_state_mutex; /* rqos state change mutex */ 238 }; 239 240 /** 241 * disk_openers - returns how many openers are there for a disk 242 * @disk: disk to check 243 * 244 * This returns the number of openers for a disk. Note that this value is only 245 * stable if disk->open_mutex is held. 246 * 247 * Note: Due to a quirk in the block layer open code, each open partition is 248 * only counted once even if there are multiple openers. 249 */ 250 static inline unsigned int disk_openers(struct gendisk *disk) 251 { 252 return atomic_read(&disk->part0->bd_openers); 253 } 254 255 /** 256 * disk_has_partscan - return %true if partition scanning is enabled on a disk 257 * @disk: disk to check 258 * 259 * Returns %true if partitions scanning is enabled for @disk, or %false if 260 * partition scanning is disabled either permanently or temporarily. 261 */ 262 static inline bool disk_has_partscan(struct gendisk *disk) 263 { 264 return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) && 265 !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state); 266 } 267 268 /* 269 * The gendisk is refcounted by the part0 block_device, and the bd_device 270 * therein is also used for device model presentation in sysfs. 271 */ 272 #define dev_to_disk(device) \ 273 (dev_to_bdev(device)->bd_disk) 274 #define disk_to_dev(disk) \ 275 (&((disk)->part0->bd_device)) 276 277 #if IS_REACHABLE(CONFIG_CDROM) 278 #define disk_to_cdi(disk) ((disk)->cdi) 279 #else 280 #define disk_to_cdi(disk) NULL 281 #endif 282 283 static inline dev_t disk_devt(struct gendisk *disk) 284 { 285 return MKDEV(disk->major, disk->first_minor); 286 } 287 288 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 289 /* 290 * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER) 291 * however we constrain this to what we can validate and test. 292 */ 293 #define BLK_MAX_BLOCK_SIZE SZ_64K 294 #else 295 #define BLK_MAX_BLOCK_SIZE PAGE_SIZE 296 #endif 297 298 299 /* blk_validate_limits() validates bsize, so drivers don't usually need to */ 300 static inline int blk_validate_block_size(unsigned long bsize) 301 { 302 if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize)) 303 return -EINVAL; 304 305 return 0; 306 } 307 308 static inline bool blk_op_is_passthrough(blk_opf_t op) 309 { 310 op &= REQ_OP_MASK; 311 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT; 312 } 313 314 /* flags set by the driver in queue_limits.features */ 315 typedef unsigned int __bitwise blk_features_t; 316 317 /* supports a volatile write cache */ 318 #define BLK_FEAT_WRITE_CACHE ((__force blk_features_t)(1u << 0)) 319 320 /* supports passing on the FUA bit */ 321 #define BLK_FEAT_FUA ((__force blk_features_t)(1u << 1)) 322 323 /* rotational device (hard drive or floppy) */ 324 #define BLK_FEAT_ROTATIONAL ((__force blk_features_t)(1u << 2)) 325 326 /* contributes to the random number pool */ 327 #define BLK_FEAT_ADD_RANDOM ((__force blk_features_t)(1u << 3)) 328 329 /* do disk/partitions IO accounting */ 330 #define BLK_FEAT_IO_STAT ((__force blk_features_t)(1u << 4)) 331 332 /* don't modify data until writeback is done */ 333 #define BLK_FEAT_STABLE_WRITES ((__force blk_features_t)(1u << 5)) 334 335 /* always completes in submit context */ 336 #define BLK_FEAT_SYNCHRONOUS ((__force blk_features_t)(1u << 6)) 337 338 /* supports REQ_NOWAIT */ 339 #define BLK_FEAT_NOWAIT ((__force blk_features_t)(1u << 7)) 340 341 /* supports DAX */ 342 #define BLK_FEAT_DAX ((__force blk_features_t)(1u << 8)) 343 344 /* supports I/O polling */ 345 #define BLK_FEAT_POLL ((__force blk_features_t)(1u << 9)) 346 347 /* is a zoned device */ 348 #define BLK_FEAT_ZONED ((__force blk_features_t)(1u << 10)) 349 350 /* supports PCI(e) p2p requests */ 351 #define BLK_FEAT_PCI_P2PDMA ((__force blk_features_t)(1u << 12)) 352 353 /* skip this queue in blk_mq_(un)quiesce_tagset */ 354 #define BLK_FEAT_SKIP_TAGSET_QUIESCE ((__force blk_features_t)(1u << 13)) 355 356 /* atomic writes enabled */ 357 #define BLK_FEAT_ATOMIC_WRITES ((__force blk_features_t)(1u << 14)) 358 359 /* undocumented magic for bcache */ 360 #define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \ 361 ((__force blk_features_t)(1u << 15)) 362 363 /* 364 * Flags automatically inherited when stacking limits. 365 */ 366 #define BLK_FEAT_INHERIT_MASK \ 367 (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \ 368 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | \ 369 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE) 370 371 /* internal flags in queue_limits.flags */ 372 typedef unsigned int __bitwise blk_flags_t; 373 374 /* do not send FLUSH/FUA commands despite advertising a write cache */ 375 #define BLK_FLAG_WRITE_CACHE_DISABLED ((__force blk_flags_t)(1u << 0)) 376 377 /* I/O topology is misaligned */ 378 #define BLK_FLAG_MISALIGNED ((__force blk_flags_t)(1u << 1)) 379 380 /* passthrough command IO accounting */ 381 #define BLK_FLAG_IOSTATS_PASSTHROUGH ((__force blk_flags_t)(1u << 2)) 382 383 struct queue_limits { 384 blk_features_t features; 385 blk_flags_t flags; 386 unsigned long seg_boundary_mask; 387 unsigned long virt_boundary_mask; 388 389 unsigned int max_hw_sectors; 390 unsigned int max_dev_sectors; 391 unsigned int chunk_sectors; 392 unsigned int max_sectors; 393 unsigned int max_user_sectors; 394 unsigned int max_segment_size; 395 unsigned int max_fast_segment_size; 396 unsigned int physical_block_size; 397 unsigned int logical_block_size; 398 unsigned int alignment_offset; 399 unsigned int io_min; 400 unsigned int io_opt; 401 unsigned int max_discard_sectors; 402 unsigned int max_hw_discard_sectors; 403 unsigned int max_user_discard_sectors; 404 unsigned int max_secure_erase_sectors; 405 unsigned int max_write_zeroes_sectors; 406 unsigned int max_wzeroes_unmap_sectors; 407 unsigned int max_hw_wzeroes_unmap_sectors; 408 unsigned int max_user_wzeroes_unmap_sectors; 409 unsigned int max_hw_zone_append_sectors; 410 unsigned int max_zone_append_sectors; 411 unsigned int discard_granularity; 412 unsigned int discard_alignment; 413 unsigned int zone_write_granularity; 414 415 /* atomic write limits */ 416 unsigned int atomic_write_hw_max; 417 unsigned int atomic_write_max_sectors; 418 unsigned int atomic_write_hw_boundary; 419 unsigned int atomic_write_boundary_sectors; 420 unsigned int atomic_write_hw_unit_min; 421 unsigned int atomic_write_unit_min; 422 unsigned int atomic_write_hw_unit_max; 423 unsigned int atomic_write_unit_max; 424 425 unsigned short max_segments; 426 unsigned short max_integrity_segments; 427 unsigned short max_discard_segments; 428 429 unsigned short max_write_streams; 430 unsigned int write_stream_granularity; 431 432 unsigned int max_open_zones; 433 unsigned int max_active_zones; 434 435 /* 436 * Drivers that set dma_alignment to less than 511 must be prepared to 437 * handle individual bvec's that are not a multiple of a SECTOR_SIZE 438 * due to possible offsets. 439 */ 440 unsigned int dma_alignment; 441 unsigned int dma_pad_mask; 442 443 struct blk_integrity integrity; 444 }; 445 446 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, 447 void *data); 448 449 int disk_report_zone(struct gendisk *disk, struct blk_zone *zone, 450 unsigned int idx, struct blk_report_zones_args *args); 451 452 int blkdev_get_zone_info(struct block_device *bdev, sector_t sector, 453 struct blk_zone *zone); 454 455 #define BLK_ALL_ZONES ((unsigned int)-1) 456 int blkdev_report_zones(struct block_device *bdev, sector_t sector, 457 unsigned int nr_zones, report_zones_cb cb, void *data); 458 int blkdev_report_zones_cached(struct block_device *bdev, sector_t sector, 459 unsigned int nr_zones, report_zones_cb cb, void *data); 460 int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, 461 sector_t sectors, sector_t nr_sectors); 462 int blk_revalidate_disk_zones(struct gendisk *disk); 463 464 /* 465 * Independent access ranges: struct blk_independent_access_range describes 466 * a range of contiguous sectors that can be accessed using device command 467 * execution resources that are independent from the resources used for 468 * other access ranges. This is typically found with single-LUN multi-actuator 469 * HDDs where each access range is served by a different set of heads. 470 * The set of independent ranges supported by the device is defined using 471 * struct blk_independent_access_ranges. The independent ranges must not overlap 472 * and must include all sectors within the disk capacity (no sector holes 473 * allowed). 474 * For a device with multiple ranges, requests targeting sectors in different 475 * ranges can be executed in parallel. A request can straddle an access range 476 * boundary. 477 */ 478 struct blk_independent_access_range { 479 struct kobject kobj; 480 sector_t sector; 481 sector_t nr_sectors; 482 }; 483 484 struct blk_independent_access_ranges { 485 struct kobject kobj; 486 bool sysfs_registered; 487 unsigned int nr_ia_ranges; 488 struct blk_independent_access_range ia_range[]; 489 }; 490 491 struct request_queue { 492 /* 493 * The queue owner gets to use this for whatever they like. 494 * ll_rw_blk doesn't touch it. 495 */ 496 void *queuedata; 497 498 struct elevator_queue *elevator; 499 500 const struct blk_mq_ops *mq_ops; 501 502 /* sw queues */ 503 struct blk_mq_ctx __percpu *queue_ctx; 504 505 /* 506 * various queue flags, see QUEUE_* below 507 */ 508 unsigned long queue_flags; 509 510 unsigned int __data_racy rq_timeout; 511 512 unsigned int queue_depth; 513 514 refcount_t refs; 515 516 /* hw dispatch queues */ 517 unsigned int nr_hw_queues; 518 struct blk_mq_hw_ctx * __rcu *queue_hw_ctx __counted_by_ptr(nr_hw_queues); 519 520 struct percpu_ref q_usage_counter; 521 struct lock_class_key io_lock_cls_key; 522 struct lockdep_map io_lockdep_map; 523 524 struct lock_class_key q_lock_cls_key; 525 struct lockdep_map q_lockdep_map; 526 527 struct request *last_merge; 528 529 spinlock_t queue_lock; 530 531 int quiesce_depth; 532 533 struct gendisk *disk; 534 535 /* 536 * mq queue kobject 537 */ 538 struct kobject *mq_kobj; 539 540 struct queue_limits limits; 541 542 #ifdef CONFIG_PM 543 struct device *dev; 544 enum rpm_status rpm_status; 545 #endif 546 547 /* 548 * Number of contexts that have called blk_set_pm_only(). If this 549 * counter is above zero then only RQF_PM requests are processed. 550 */ 551 atomic_t pm_only; 552 553 struct blk_queue_stats *stats; 554 struct rq_qos *rq_qos; 555 struct mutex rq_qos_mutex; 556 557 /* 558 * ida allocated id for this queue. Used to index queues from 559 * ioctx. 560 */ 561 int id; 562 563 /* 564 * queue settings 565 */ 566 unsigned int nr_requests; /* Max # of requests */ 567 unsigned int async_depth; /* Max # of async requests */ 568 569 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 570 struct blk_crypto_profile *crypto_profile; 571 struct kobject *crypto_kobject; 572 #endif 573 574 struct timer_list timeout; 575 struct work_struct timeout_work; 576 577 atomic_t nr_active_requests_shared_tags; 578 579 struct blk_mq_tags *sched_shared_tags; 580 581 struct list_head icq_list; 582 #ifdef CONFIG_BLK_CGROUP 583 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); 584 struct blkcg_gq *root_blkg; 585 struct list_head blkg_list; 586 struct mutex blkcg_mutex; 587 #endif 588 589 int node; 590 591 spinlock_t requeue_lock; 592 struct list_head requeue_list; 593 struct delayed_work requeue_work; 594 595 #ifdef CONFIG_BLK_DEV_IO_TRACE 596 struct blk_trace __rcu *blk_trace; 597 #endif 598 /* 599 * for flush operations 600 */ 601 struct blk_flush_queue *fq; 602 struct list_head flush_list; 603 604 /* 605 * Protects against I/O scheduler switching, particularly when updating 606 * q->elevator. Since the elevator update code path may also modify q-> 607 * nr_requests and wbt latency, this lock also protects the sysfs attrs 608 * nr_requests and wbt_lat_usec. Additionally the nr_hw_queues update 609 * may modify hctx tags, reserved-tags and cpumask, so this lock also 610 * helps protect the hctx sysfs/debugfs attrs. To ensure proper locking 611 * order during an elevator or nr_hw_queue update, first freeze the 612 * queue, then acquire ->elevator_lock. 613 */ 614 struct mutex elevator_lock; 615 616 struct mutex sysfs_lock; 617 /* 618 * Protects queue limits and also sysfs attribute read_ahead_kb. 619 */ 620 struct mutex limits_lock; 621 622 /* 623 * for reusing dead hctx instance in case of updating 624 * nr_hw_queues 625 */ 626 struct list_head unused_hctx_list; 627 spinlock_t unused_hctx_lock; 628 629 int mq_freeze_depth; 630 631 #ifdef CONFIG_BLK_DEV_THROTTLING 632 /* Throttle data */ 633 struct throtl_data *td; 634 #endif 635 struct rcu_head rcu_head; 636 #ifdef CONFIG_LOCKDEP 637 struct task_struct *mq_freeze_owner; 638 int mq_freeze_owner_depth; 639 /* 640 * Records disk & queue state in current context, used in unfreeze 641 * queue 642 */ 643 bool mq_freeze_disk_dead; 644 bool mq_freeze_queue_dying; 645 #endif 646 wait_queue_head_t mq_freeze_wq; 647 /* 648 * Protect concurrent access to q_usage_counter by 649 * percpu_ref_kill() and percpu_ref_reinit(). 650 */ 651 struct mutex mq_freeze_lock; 652 653 struct blk_mq_tag_set *tag_set; 654 struct list_head tag_set_list; 655 656 struct dentry *debugfs_dir; 657 struct dentry *sched_debugfs_dir; 658 struct dentry *rqos_debugfs_dir; 659 /* 660 * Serializes all debugfs metadata operations using the above dentries. 661 */ 662 struct mutex debugfs_mutex; 663 }; 664 665 /* Keep blk_queue_flag_name[] in sync with the definitions below */ 666 enum { 667 QUEUE_FLAG_DYING, /* queue being torn down */ 668 QUEUE_FLAG_NOMERGES, /* disable merge attempts */ 669 QUEUE_FLAG_SAME_COMP, /* complete on same CPU-group */ 670 QUEUE_FLAG_FAIL_IO, /* fake timeout */ 671 QUEUE_FLAG_NOXMERGES, /* No extended merges */ 672 QUEUE_FLAG_SAME_FORCE, /* force complete on same CPU */ 673 QUEUE_FLAG_INIT_DONE, /* queue is initialized */ 674 QUEUE_FLAG_STATS, /* track IO start and completion times */ 675 QUEUE_FLAG_REGISTERED, /* queue has been registered to a disk */ 676 QUEUE_FLAG_QUIESCED, /* queue has been quiesced */ 677 QUEUE_FLAG_RQ_ALLOC_TIME, /* record rq->alloc_time_ns */ 678 QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */ 679 QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */ 680 QUEUE_FLAG_DISABLE_WBT_DEF, /* for sched to disable/enable wbt */ 681 QUEUE_FLAG_NO_ELV_SWITCH, /* a switch is running or disk is going away */ 682 QUEUE_FLAG_QOS_ENABLED, /* qos is enabled */ 683 QUEUE_FLAG_BIO_ISSUE_TIME, /* record bio->issue_time_ns */ 684 QUEUE_FLAG_ZONED_QD1_WRITES, /* Limit zoned devices writes to QD=1 */ 685 QUEUE_FLAG_MAX 686 }; 687 688 #define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP) 689 690 void blk_queue_flag_set(unsigned int flag, struct request_queue *q); 691 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); 692 693 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) 694 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags) 695 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) 696 #define blk_queue_noxmerges(q) \ 697 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) 698 #define blk_queue_rot(q) ((q)->limits.features & BLK_FEAT_ROTATIONAL) 699 #define blk_queue_io_stat(q) ((q)->limits.features & BLK_FEAT_IO_STAT) 700 #define blk_queue_passthrough_stat(q) \ 701 ((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH) 702 #define blk_queue_dax(q) ((q)->limits.features & BLK_FEAT_DAX) 703 #define blk_queue_pci_p2pdma(q) ((q)->limits.features & BLK_FEAT_PCI_P2PDMA) 704 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 705 #define blk_queue_rq_alloc_time(q) \ 706 test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags) 707 #else 708 #define blk_queue_rq_alloc_time(q) false 709 #endif 710 711 #define blk_noretry_request(rq) \ 712 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ 713 REQ_FAILFAST_DRIVER)) 714 #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) 715 #define blk_queue_pm_only(q) atomic_read(&(q)->pm_only) 716 #define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags) 717 #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags) 718 #define blk_queue_skip_tagset_quiesce(q) \ 719 ((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE) 720 #define blk_queue_disable_wbt(q) \ 721 test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags) 722 #define blk_queue_zoned_qd1_writes(q) \ 723 test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags) 724 725 extern void blk_set_pm_only(struct request_queue *q); 726 extern void blk_clear_pm_only(struct request_queue *q); 727 728 #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist) 729 730 #define dma_map_bvec(dev, bv, dir, attrs) \ 731 dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \ 732 (dir), (attrs)) 733 734 static inline bool queue_is_mq(struct request_queue *q) 735 { 736 return q->mq_ops; 737 } 738 739 #ifdef CONFIG_PM 740 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 741 { 742 return q->rpm_status; 743 } 744 #else 745 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 746 { 747 return RPM_ACTIVE; 748 } 749 #endif 750 751 static inline bool blk_queue_is_zoned(struct request_queue *q) 752 { 753 return IS_ENABLED(CONFIG_BLK_DEV_ZONED) && 754 (q->limits.features & BLK_FEAT_ZONED); 755 } 756 757 static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector) 758 { 759 if (!blk_queue_is_zoned(disk->queue)) 760 return 0; 761 return sector >> ilog2(disk->queue->limits.chunk_sectors); 762 } 763 764 static inline unsigned int bdev_max_open_zones(struct block_device *bdev) 765 { 766 return bdev->bd_disk->queue->limits.max_open_zones; 767 } 768 769 static inline unsigned int bdev_max_active_zones(struct block_device *bdev) 770 { 771 return bdev->bd_disk->queue->limits.max_active_zones; 772 } 773 774 static inline unsigned int blk_queue_depth(struct request_queue *q) 775 { 776 if (q->queue_depth) 777 return q->queue_depth; 778 779 return q->nr_requests; 780 } 781 782 /* 783 * default timeout for SG_IO if none specified 784 */ 785 #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ) 786 #define BLK_MIN_SG_TIMEOUT (7 * HZ) 787 788 /* This should not be used directly - use rq_for_each_segment */ 789 #define for_each_bio(_bio) \ 790 for (; _bio; _bio = _bio->bi_next) 791 792 int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, 793 const struct attribute_group **groups, 794 struct fwnode_handle *fwnode); 795 int __must_check device_add_disk(struct device *parent, struct gendisk *disk, 796 const struct attribute_group **groups); 797 static inline int __must_check add_disk(struct gendisk *disk) 798 { 799 return device_add_disk(NULL, disk, NULL); 800 } 801 void del_gendisk(struct gendisk *gp); 802 void invalidate_disk(struct gendisk *disk); 803 void set_disk_ro(struct gendisk *disk, bool read_only); 804 void disk_uevent(struct gendisk *disk, enum kobject_action action); 805 806 static inline u8 bdev_partno(const struct block_device *bdev) 807 { 808 return atomic_read(&bdev->__bd_flags) & BD_PARTNO; 809 } 810 811 static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag) 812 { 813 return atomic_read(&bdev->__bd_flags) & flag; 814 } 815 816 static inline void bdev_set_flag(struct block_device *bdev, unsigned flag) 817 { 818 atomic_or(flag, &bdev->__bd_flags); 819 } 820 821 static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag) 822 { 823 atomic_andnot(flag, &bdev->__bd_flags); 824 } 825 826 static inline bool get_disk_ro(struct gendisk *disk) 827 { 828 return bdev_test_flag(disk->part0, BD_READ_ONLY) || 829 test_bit(GD_READ_ONLY, &disk->state); 830 } 831 832 static inline bool bdev_read_only(struct block_device *bdev) 833 { 834 return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk); 835 } 836 837 bool set_capacity_and_notify(struct gendisk *disk, sector_t size); 838 void disk_force_media_change(struct gendisk *disk); 839 void bdev_mark_dead(struct block_device *bdev, bool surprise); 840 841 void add_disk_randomness(struct gendisk *disk) __latent_entropy; 842 void rand_initialize_disk(struct gendisk *disk); 843 844 static inline sector_t get_start_sect(struct block_device *bdev) 845 { 846 return bdev->bd_start_sect; 847 } 848 849 static inline sector_t bdev_nr_sectors(struct block_device *bdev) 850 { 851 return bdev->bd_nr_sectors; 852 } 853 854 static inline loff_t bdev_nr_bytes(struct block_device *bdev) 855 { 856 return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT; 857 } 858 859 static inline sector_t get_capacity(struct gendisk *disk) 860 { 861 return bdev_nr_sectors(disk->part0); 862 } 863 864 static inline u64 sb_bdev_nr_blocks(struct super_block *sb) 865 { 866 return bdev_nr_sectors(sb->s_bdev) >> 867 (sb->s_blocksize_bits - SECTOR_SHIFT); 868 } 869 870 #ifdef CONFIG_BLK_DEV_ZONED 871 static inline unsigned int disk_nr_zones(struct gendisk *disk) 872 { 873 return disk->nr_zones; 874 } 875 876 /** 877 * bio_needs_zone_write_plugging - Check if a BIO needs to be handled with zone 878 * write plugging 879 * @bio: The BIO being submitted 880 * 881 * Return true whenever @bio execution needs to be handled through zone 882 * write plugging (using blk_zone_plug_bio()). Return false otherwise. 883 */ 884 static inline bool bio_needs_zone_write_plugging(struct bio *bio) 885 { 886 enum req_op op = bio_op(bio); 887 888 /* 889 * Only zoned block devices have a zone write plug hash table. But not 890 * all of them have one (e.g. DM devices may not need one). 891 */ 892 if (!bio->bi_bdev->bd_disk->zone_wplugs_hash) 893 return false; 894 895 /* Only write operations need zone write plugging. */ 896 if (!op_is_write(op)) 897 return false; 898 899 /* Ignore empty flush */ 900 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 901 return false; 902 903 /* Ignore BIOs that already have been handled by zone write plugging. */ 904 if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING)) 905 return false; 906 907 /* 908 * All zone write operations must be handled through zone write plugging 909 * using blk_zone_plug_bio(). 910 */ 911 switch (op) { 912 case REQ_OP_ZONE_APPEND: 913 case REQ_OP_WRITE: 914 case REQ_OP_WRITE_ZEROES: 915 case REQ_OP_ZONE_FINISH: 916 case REQ_OP_ZONE_RESET: 917 case REQ_OP_ZONE_RESET_ALL: 918 return true; 919 default: 920 return false; 921 } 922 } 923 924 bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs); 925 926 /** 927 * disk_zone_capacity - returns the zone capacity of zone containing @sector 928 * @disk: disk to work with 929 * @sector: sector number within the querying zone 930 * 931 * Returns the zone capacity of a zone containing @sector. @sector can be any 932 * sector in the zone. 933 */ 934 static inline unsigned int disk_zone_capacity(struct gendisk *disk, 935 sector_t sector) 936 { 937 sector_t zone_sectors = disk->queue->limits.chunk_sectors; 938 939 if (sector + zone_sectors >= get_capacity(disk)) 940 return disk->last_zone_capacity; 941 return disk->zone_capacity; 942 } 943 static inline unsigned int bdev_zone_capacity(struct block_device *bdev, 944 sector_t pos) 945 { 946 return disk_zone_capacity(bdev->bd_disk, pos); 947 } 948 949 bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector); 950 951 #else /* CONFIG_BLK_DEV_ZONED */ 952 static inline unsigned int disk_nr_zones(struct gendisk *disk) 953 { 954 return 0; 955 } 956 957 static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector) 958 { 959 return false; 960 } 961 962 static inline bool bio_needs_zone_write_plugging(struct bio *bio) 963 { 964 return false; 965 } 966 967 static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) 968 { 969 return false; 970 } 971 #endif /* CONFIG_BLK_DEV_ZONED */ 972 973 static inline unsigned int bdev_nr_zones(struct block_device *bdev) 974 { 975 return disk_nr_zones(bdev->bd_disk); 976 } 977 978 int bdev_disk_changed(struct gendisk *disk, bool invalidate); 979 980 void put_disk(struct gendisk *disk); 981 struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node, 982 struct lock_class_key *lkclass); 983 984 /** 985 * blk_alloc_disk - allocate a gendisk structure 986 * @lim: queue limits to be used for this disk. 987 * @node_id: numa node to allocate on 988 * 989 * Allocate and pre-initialize a gendisk structure for use with BIO based 990 * drivers. 991 * 992 * Returns an ERR_PTR on error, else the allocated disk. 993 * 994 * Context: can sleep 995 */ 996 #define blk_alloc_disk(lim, node_id) \ 997 ({ \ 998 static struct lock_class_key __key; \ 999 \ 1000 __blk_alloc_disk(lim, node_id, &__key); \ 1001 }) 1002 1003 int __register_blkdev(unsigned int major, const char *name, 1004 void (*probe)(dev_t devt)); 1005 #define register_blkdev(major, name) \ 1006 __register_blkdev(major, name, NULL) 1007 void unregister_blkdev(unsigned int major, const char *name); 1008 1009 bool disk_check_media_change(struct gendisk *disk); 1010 void set_capacity(struct gendisk *disk, sector_t size); 1011 1012 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 1013 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk); 1014 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk); 1015 #else 1016 static inline int bd_link_disk_holder(struct block_device *bdev, 1017 struct gendisk *disk) 1018 { 1019 return 0; 1020 } 1021 static inline void bd_unlink_disk_holder(struct block_device *bdev, 1022 struct gendisk *disk) 1023 { 1024 } 1025 #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */ 1026 1027 dev_t part_devt(struct gendisk *disk, u8 partno); 1028 void inc_diskseq(struct gendisk *disk); 1029 void blk_request_module(dev_t devt); 1030 1031 extern int blk_register_queue(struct gendisk *disk); 1032 extern void blk_unregister_queue(struct gendisk *disk); 1033 void submit_bio_noacct(struct bio *bio); 1034 struct bio *bio_split_to_limits(struct bio *bio); 1035 struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors, 1036 struct bio_set *bs); 1037 1038 extern int blk_lld_busy(struct request_queue *q); 1039 extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags); 1040 extern void blk_queue_exit(struct request_queue *q); 1041 extern void blk_sync_queue(struct request_queue *q); 1042 1043 /* Convert a request operation REQ_OP_name into the string "name" */ 1044 extern const char *blk_op_str(enum req_op op); 1045 1046 int blk_status_to_errno(blk_status_t status); 1047 blk_status_t errno_to_blk_status(int errno); 1048 1049 /* only poll the hardware once, don't continue until a completion was found */ 1050 #define BLK_POLL_ONESHOT (1 << 0) 1051 int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags); 1052 int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob, 1053 unsigned int flags); 1054 1055 static inline struct request_queue *bdev_get_queue(struct block_device *bdev) 1056 { 1057 return bdev->bd_queue; /* this is never NULL */ 1058 } 1059 1060 /* Convert a zone condition BLK_ZONE_COND_name into the string "name" */ 1061 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond); 1062 1063 static inline unsigned int bio_zone_no(struct bio *bio) 1064 { 1065 return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector); 1066 } 1067 1068 static inline bool bio_straddles_zones(struct bio *bio) 1069 { 1070 return bio_sectors(bio) && 1071 bio_zone_no(bio) != 1072 disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1); 1073 } 1074 1075 /* 1076 * Return how much within the boundary is left to be used for I/O at a given 1077 * offset. 1078 */ 1079 static inline unsigned int blk_boundary_sectors_left(sector_t offset, 1080 unsigned int boundary_sectors) 1081 { 1082 if (unlikely(!is_power_of_2(boundary_sectors))) 1083 return boundary_sectors - sector_div(offset, boundary_sectors); 1084 return boundary_sectors - (offset & (boundary_sectors - 1)); 1085 } 1086 1087 /** 1088 * queue_limits_start_update - start an atomic update of queue limits 1089 * @q: queue to update 1090 * 1091 * This functions starts an atomic update of the queue limits. It takes a lock 1092 * to prevent other updates and returns a snapshot of the current limits that 1093 * the caller can modify. The caller must call queue_limits_commit_update() 1094 * to finish the update. 1095 * 1096 * Context: process context. 1097 */ 1098 static inline struct queue_limits 1099 queue_limits_start_update(struct request_queue *q) 1100 __acquires(&q->limits_lock) 1101 { 1102 mutex_lock(&q->limits_lock); 1103 return q->limits; 1104 } 1105 int queue_limits_commit_update_frozen(struct request_queue *q, 1106 struct queue_limits *lim) __releases(&q->limits_lock); 1107 int queue_limits_commit_update(struct request_queue *q, 1108 struct queue_limits *lim) __releases(&q->limits_lock); 1109 int queue_limits_set(struct request_queue *q, struct queue_limits *lim) 1110 __must_not_hold(&q->limits_lock); 1111 int blk_validate_limits(struct queue_limits *lim); 1112 1113 /** 1114 * queue_limits_cancel_update - cancel an atomic update of queue limits 1115 * @q: queue to update 1116 * 1117 * This functions cancels an atomic update of the queue limits started by 1118 * queue_limits_start_update() and should be used when an error occurs after 1119 * starting update. 1120 */ 1121 static inline void queue_limits_cancel_update(struct request_queue *q) 1122 __releases(&q->limits_lock) 1123 { 1124 mutex_unlock(&q->limits_lock); 1125 } 1126 1127 /* 1128 * These helpers are for drivers that have sloppy feature negotiation and might 1129 * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O 1130 * completion handler when the device returned an indicator that the respective 1131 * feature is not actually supported. They are racy and the driver needs to 1132 * cope with that. Try to avoid this scheme if you can. 1133 */ 1134 static inline void blk_queue_disable_discard(struct request_queue *q) 1135 { 1136 q->limits.max_discard_sectors = 0; 1137 } 1138 1139 static inline void blk_queue_disable_secure_erase(struct request_queue *q) 1140 { 1141 q->limits.max_secure_erase_sectors = 0; 1142 } 1143 1144 static inline void blk_queue_disable_write_zeroes(struct request_queue *q) 1145 { 1146 q->limits.max_write_zeroes_sectors = 0; 1147 q->limits.max_wzeroes_unmap_sectors = 0; 1148 } 1149 1150 /* 1151 * Access functions for manipulating queue properties 1152 */ 1153 extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth); 1154 extern void blk_set_stacking_limits(struct queue_limits *lim); 1155 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, 1156 sector_t offset); 1157 void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev, 1158 sector_t offset, const char *pfx); 1159 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int); 1160 1161 struct blk_independent_access_ranges * 1162 disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges); 1163 void disk_set_independent_access_ranges(struct gendisk *disk, 1164 struct blk_independent_access_ranges *iars); 1165 1166 bool __must_check blk_get_queue(struct request_queue *); 1167 extern void blk_put_queue(struct request_queue *); 1168 1169 void blk_mark_disk_dead(struct gendisk *disk); 1170 1171 #ifdef CONFIG_BLOCK 1172 int blkdev_issue_flush(struct block_device *bdev); 1173 long nr_blockdev_pages(void); 1174 #else /* CONFIG_BLOCK */ 1175 static inline int blkdev_issue_flush(struct block_device *bdev) 1176 { 1177 return 0; 1178 } 1179 1180 static inline long nr_blockdev_pages(void) 1181 { 1182 return 0; 1183 } 1184 #endif /* CONFIG_BLOCK */ 1185 1186 extern void blk_io_schedule(void); 1187 1188 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1189 sector_t nr_sects, gfp_t gfp_mask); 1190 void __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1191 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop); 1192 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 1193 sector_t nr_sects, gfp_t gfp); 1194 1195 #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */ 1196 #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */ 1197 #define BLKDEV_ZERO_KILLABLE (1 << 2) /* interruptible by fatal signals */ 1198 1199 extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1200 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 1201 unsigned flags); 1202 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1203 sector_t nr_sects, gfp_t gfp_mask, unsigned flags); 1204 1205 static inline int sb_issue_discard(struct super_block *sb, sector_t block, 1206 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) 1207 { 1208 return blkdev_issue_discard(sb->s_bdev, 1209 block << (sb->s_blocksize_bits - 1210 SECTOR_SHIFT), 1211 nr_blocks << (sb->s_blocksize_bits - 1212 SECTOR_SHIFT), 1213 gfp_mask); 1214 } 1215 static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, 1216 sector_t nr_blocks, gfp_t gfp_mask) 1217 { 1218 return blkdev_issue_zeroout(sb->s_bdev, 1219 block << (sb->s_blocksize_bits - 1220 SECTOR_SHIFT), 1221 nr_blocks << (sb->s_blocksize_bits - 1222 SECTOR_SHIFT), 1223 gfp_mask, 0); 1224 } 1225 1226 static inline bool bdev_is_partition(struct block_device *bdev) 1227 { 1228 return bdev_partno(bdev) != 0; 1229 } 1230 1231 enum blk_default_limits { 1232 BLK_MAX_SEGMENTS = 128, 1233 BLK_SAFE_MAX_SECTORS = 255, 1234 BLK_MAX_SEGMENT_SIZE = 65536, 1235 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1236 }; 1237 1238 static inline struct queue_limits *bdev_limits(struct block_device *bdev) 1239 { 1240 return &bdev_get_queue(bdev)->limits; 1241 } 1242 1243 static inline unsigned long queue_segment_boundary(const struct request_queue *q) 1244 { 1245 return q->limits.seg_boundary_mask; 1246 } 1247 1248 static inline unsigned long queue_virt_boundary(const struct request_queue *q) 1249 { 1250 return q->limits.virt_boundary_mask; 1251 } 1252 1253 static inline unsigned int queue_max_sectors(const struct request_queue *q) 1254 { 1255 return q->limits.max_sectors; 1256 } 1257 1258 static inline unsigned int queue_max_bytes(struct request_queue *q) 1259 { 1260 return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9; 1261 } 1262 1263 static inline unsigned int queue_max_hw_sectors(const struct request_queue *q) 1264 { 1265 return q->limits.max_hw_sectors; 1266 } 1267 1268 static inline unsigned short queue_max_segments(const struct request_queue *q) 1269 { 1270 return q->limits.max_segments; 1271 } 1272 1273 static inline unsigned short queue_max_discard_segments(const struct request_queue *q) 1274 { 1275 return q->limits.max_discard_segments; 1276 } 1277 1278 static inline unsigned int queue_max_segment_size(const struct request_queue *q) 1279 { 1280 return q->limits.max_segment_size; 1281 } 1282 1283 static inline bool queue_emulates_zone_append(struct request_queue *q) 1284 { 1285 return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors; 1286 } 1287 1288 static inline bool bdev_emulates_zone_append(struct block_device *bdev) 1289 { 1290 return queue_emulates_zone_append(bdev_get_queue(bdev)); 1291 } 1292 1293 static inline unsigned int 1294 bdev_max_zone_append_sectors(struct block_device *bdev) 1295 { 1296 return bdev_limits(bdev)->max_zone_append_sectors; 1297 } 1298 1299 static inline unsigned int bdev_max_segments(struct block_device *bdev) 1300 { 1301 return queue_max_segments(bdev_get_queue(bdev)); 1302 } 1303 1304 static inline unsigned short bdev_max_write_streams(struct block_device *bdev) 1305 { 1306 if (bdev_is_partition(bdev)) 1307 return 0; 1308 return bdev_limits(bdev)->max_write_streams; 1309 } 1310 1311 static inline unsigned queue_logical_block_size(const struct request_queue *q) 1312 { 1313 return q->limits.logical_block_size; 1314 } 1315 1316 static inline unsigned int bdev_logical_block_size(struct block_device *bdev) 1317 { 1318 return queue_logical_block_size(bdev_get_queue(bdev)); 1319 } 1320 1321 static inline unsigned int queue_physical_block_size(const struct request_queue *q) 1322 { 1323 return q->limits.physical_block_size; 1324 } 1325 1326 static inline unsigned int bdev_physical_block_size(struct block_device *bdev) 1327 { 1328 return queue_physical_block_size(bdev_get_queue(bdev)); 1329 } 1330 1331 static inline unsigned int queue_io_min(const struct request_queue *q) 1332 { 1333 return q->limits.io_min; 1334 } 1335 1336 static inline unsigned int bdev_io_min(struct block_device *bdev) 1337 { 1338 return queue_io_min(bdev_get_queue(bdev)); 1339 } 1340 1341 static inline unsigned int queue_io_opt(const struct request_queue *q) 1342 { 1343 return q->limits.io_opt; 1344 } 1345 1346 static inline unsigned int bdev_io_opt(struct block_device *bdev) 1347 { 1348 return queue_io_opt(bdev_get_queue(bdev)); 1349 } 1350 1351 static inline unsigned int 1352 queue_zone_write_granularity(const struct request_queue *q) 1353 { 1354 return q->limits.zone_write_granularity; 1355 } 1356 1357 static inline unsigned int 1358 bdev_zone_write_granularity(struct block_device *bdev) 1359 { 1360 return queue_zone_write_granularity(bdev_get_queue(bdev)); 1361 } 1362 1363 int bdev_alignment_offset(struct block_device *bdev); 1364 unsigned int bdev_discard_alignment(struct block_device *bdev); 1365 1366 static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev) 1367 { 1368 return bdev_limits(bdev)->max_discard_sectors; 1369 } 1370 1371 static inline unsigned int bdev_discard_granularity(struct block_device *bdev) 1372 { 1373 return bdev_limits(bdev)->discard_granularity; 1374 } 1375 1376 static inline unsigned int 1377 bdev_max_secure_erase_sectors(struct block_device *bdev) 1378 { 1379 return bdev_limits(bdev)->max_secure_erase_sectors; 1380 } 1381 1382 static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev) 1383 { 1384 return bdev_limits(bdev)->max_write_zeroes_sectors; 1385 } 1386 1387 static inline unsigned int 1388 bdev_write_zeroes_unmap_sectors(struct block_device *bdev) 1389 { 1390 return bdev_limits(bdev)->max_wzeroes_unmap_sectors; 1391 } 1392 1393 static inline bool bdev_rot(struct block_device *bdev) 1394 { 1395 return blk_queue_rot(bdev_get_queue(bdev)); 1396 } 1397 1398 static inline bool bdev_synchronous(struct block_device *bdev) 1399 { 1400 return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS; 1401 } 1402 1403 static inline bool bdev_has_integrity_csum(struct block_device *bdev) 1404 { 1405 struct queue_limits *lim = bdev_limits(bdev); 1406 1407 return IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && 1408 lim->integrity.csum_type != BLK_INTEGRITY_CSUM_NONE; 1409 } 1410 1411 static inline bool bdev_stable_writes(struct block_device *bdev) 1412 { 1413 return bdev_has_integrity_csum(bdev) || 1414 (bdev_limits(bdev)->features & BLK_FEAT_STABLE_WRITES); 1415 } 1416 1417 static inline bool blk_queue_write_cache(struct request_queue *q) 1418 { 1419 return (q->limits.features & BLK_FEAT_WRITE_CACHE) && 1420 !(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED); 1421 } 1422 1423 static inline bool bdev_write_cache(struct block_device *bdev) 1424 { 1425 return blk_queue_write_cache(bdev_get_queue(bdev)); 1426 } 1427 1428 static inline bool bdev_fua(struct block_device *bdev) 1429 { 1430 return bdev_limits(bdev)->features & BLK_FEAT_FUA; 1431 } 1432 1433 static inline bool bdev_nowait(struct block_device *bdev) 1434 { 1435 return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT; 1436 } 1437 1438 static inline bool bdev_is_zoned(struct block_device *bdev) 1439 { 1440 return blk_queue_is_zoned(bdev_get_queue(bdev)); 1441 } 1442 1443 static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec) 1444 { 1445 return disk_zone_no(bdev->bd_disk, sec); 1446 } 1447 1448 static inline sector_t bdev_zone_sectors(struct block_device *bdev) 1449 { 1450 struct request_queue *q = bdev_get_queue(bdev); 1451 1452 if (!blk_queue_is_zoned(q)) 1453 return 0; 1454 return q->limits.chunk_sectors; 1455 } 1456 1457 static inline sector_t bdev_zone_start(struct block_device *bdev, 1458 sector_t sector) 1459 { 1460 return sector & ~(bdev_zone_sectors(bdev) - 1); 1461 } 1462 1463 static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev, 1464 sector_t sector) 1465 { 1466 return sector & (bdev_zone_sectors(bdev) - 1); 1467 } 1468 1469 static inline sector_t bio_offset_from_zone_start(struct bio *bio) 1470 { 1471 return bdev_offset_from_zone_start(bio->bi_bdev, 1472 bio->bi_iter.bi_sector); 1473 } 1474 1475 static inline bool bdev_is_zone_start(struct block_device *bdev, 1476 sector_t sector) 1477 { 1478 return bdev_offset_from_zone_start(bdev, sector) == 0; 1479 } 1480 1481 /* Check whether @sector is a multiple of the zone size. */ 1482 static inline bool bdev_is_zone_aligned(struct block_device *bdev, 1483 sector_t sector) 1484 { 1485 return bdev_is_zone_start(bdev, sector); 1486 } 1487 1488 int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector, 1489 sector_t nr_sects, gfp_t gfp_mask); 1490 1491 static inline unsigned int queue_dma_alignment(const struct request_queue *q) 1492 { 1493 return q->limits.dma_alignment; 1494 } 1495 1496 static inline unsigned int 1497 queue_atomic_write_unit_max_bytes(const struct request_queue *q) 1498 { 1499 return q->limits.atomic_write_unit_max; 1500 } 1501 1502 static inline unsigned int 1503 queue_atomic_write_unit_min_bytes(const struct request_queue *q) 1504 { 1505 return q->limits.atomic_write_unit_min; 1506 } 1507 1508 static inline unsigned int 1509 queue_atomic_write_boundary_bytes(const struct request_queue *q) 1510 { 1511 return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT; 1512 } 1513 1514 static inline unsigned int 1515 queue_atomic_write_max_bytes(const struct request_queue *q) 1516 { 1517 return q->limits.atomic_write_max_sectors << SECTOR_SHIFT; 1518 } 1519 1520 static inline unsigned int bdev_dma_alignment(struct block_device *bdev) 1521 { 1522 return queue_dma_alignment(bdev_get_queue(bdev)); 1523 } 1524 1525 static inline unsigned int 1526 blk_lim_dma_alignment_and_pad(struct queue_limits *lim) 1527 { 1528 return lim->dma_alignment | lim->dma_pad_mask; 1529 } 1530 1531 static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr, 1532 unsigned int len) 1533 { 1534 unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits); 1535 1536 return !(addr & alignment) && !(len & alignment); 1537 } 1538 1539 /* assumes size > 256 */ 1540 static inline unsigned int blksize_bits(unsigned int size) 1541 { 1542 return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT; 1543 } 1544 1545 int kblockd_schedule_work(struct work_struct *work); 1546 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); 1547 1548 #define MODULE_ALIAS_BLOCKDEV(major,minor) \ 1549 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) 1550 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ 1551 MODULE_ALIAS("block-major-" __stringify(major) "-*") 1552 1553 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1554 1555 bool blk_crypto_register(struct blk_crypto_profile *profile, 1556 struct request_queue *q); 1557 1558 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1559 1560 static inline bool blk_crypto_register(struct blk_crypto_profile *profile, 1561 struct request_queue *q) 1562 { 1563 return true; 1564 } 1565 1566 #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ 1567 1568 enum blk_unique_id { 1569 /* these match the Designator Types specified in SPC */ 1570 BLK_UID_T10 = 1, 1571 BLK_UID_EUI64 = 2, 1572 BLK_UID_NAA = 3, 1573 }; 1574 1575 struct block_device_operations { 1576 void (*submit_bio)(struct bio *bio); 1577 int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, 1578 unsigned int flags); 1579 int (*open)(struct gendisk *disk, blk_mode_t mode); 1580 void (*release)(struct gendisk *disk); 1581 int (*ioctl)(struct block_device *bdev, blk_mode_t mode, 1582 unsigned cmd, unsigned long arg); 1583 int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode, 1584 unsigned cmd, unsigned long arg); 1585 unsigned int (*check_events) (struct gendisk *disk, 1586 unsigned int clearing); 1587 void (*unlock_native_capacity) (struct gendisk *); 1588 int (*getgeo)(struct gendisk *, struct hd_geometry *); 1589 int (*set_read_only)(struct block_device *bdev, bool ro); 1590 void (*free_disk)(struct gendisk *disk); 1591 /* this callback is with swap_lock and sometimes page table lock held */ 1592 void (*swap_slot_free_notify) (struct block_device *, unsigned long); 1593 int (*report_zones)(struct gendisk *, sector_t sector, 1594 unsigned int nr_zones, 1595 struct blk_report_zones_args *args); 1596 char *(*devnode)(struct gendisk *disk, umode_t *mode); 1597 /* returns the length of the identifier or a negative errno: */ 1598 int (*get_unique_id)(struct gendisk *disk, u8 id[16], 1599 enum blk_unique_id id_type); 1600 int (*init_dma_buf_io_ctx)(struct block_device *, 1601 struct dma_buf_io_ctx *); 1602 struct module *owner; 1603 const struct pr_ops *pr_ops; 1604 1605 /* 1606 * Special callback for probing GPT entry at a given sector. 1607 * Needed by Android devices, used by GPT scanner and MMC blk 1608 * driver. 1609 */ 1610 int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector); 1611 }; 1612 1613 #ifdef CONFIG_COMPAT 1614 extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t, 1615 unsigned int, unsigned long); 1616 #else 1617 #define blkdev_compat_ptr_ioctl NULL 1618 #endif 1619 1620 static inline void blk_wake_io_task(struct task_struct *waiter) 1621 { 1622 /* 1623 * If we're polling, the task itself is doing the completions. For 1624 * that case, we don't need to signal a wakeup, it's enough to just 1625 * mark us as RUNNING. 1626 */ 1627 if (waiter == current) 1628 __set_current_state(TASK_RUNNING); 1629 else 1630 wake_up_process(waiter); 1631 } 1632 1633 unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op, 1634 unsigned long start_time); 1635 void bdev_end_io_acct(struct block_device *bdev, enum req_op op, 1636 unsigned int sectors, unsigned long start_time); 1637 1638 unsigned long bio_start_io_acct(struct bio *bio); 1639 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, 1640 struct block_device *orig_bdev); 1641 1642 /** 1643 * bio_end_io_acct - end I/O accounting for bio based drivers 1644 * @bio: bio to end account for 1645 * @start_time: start time returned by bio_start_io_acct() 1646 */ 1647 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) 1648 { 1649 return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev); 1650 } 1651 1652 int bdev_validate_blocksize(struct block_device *bdev, int block_size); 1653 int set_blocksize(struct file *file, int size); 1654 1655 int lookup_bdev(const char *pathname, dev_t *dev); 1656 1657 void blkdev_show(struct seq_file *seqf, off_t offset); 1658 1659 #define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */ 1660 #define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */ 1661 #ifdef CONFIG_BLOCK 1662 #define BLKDEV_MAJOR_MAX 512 1663 #else 1664 #define BLKDEV_MAJOR_MAX 0 1665 #endif 1666 1667 struct blk_holder_ops { 1668 void (*mark_dead)(struct block_device *bdev, bool surprise) 1669 __releases(&bdev->bd_holder_lock); 1670 1671 /* 1672 * Sync the file system mounted on the block device. 1673 */ 1674 void (*sync)(struct block_device *bdev) 1675 __releases(&bdev->bd_holder_lock); 1676 1677 /* 1678 * Freeze the file system mounted on the block device. 1679 */ 1680 int (*freeze)(struct block_device *bdev) 1681 __releases(&bdev->bd_holder_lock); 1682 1683 /* 1684 * Thaw the file system mounted on the block device. 1685 */ 1686 int (*thaw)(struct block_device *bdev) 1687 __releases(&bdev->bd_holder_lock); 1688 }; 1689 1690 /* 1691 * Return the correct open flags for blkdev_get_by_* for super block flags 1692 * as stored in sb->s_flags. 1693 */ 1694 #define sb_open_mode(flags) \ 1695 (BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \ 1696 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE)) 1697 1698 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, 1699 const struct blk_holder_ops *hops); 1700 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode, 1701 void *holder, const struct blk_holder_ops *hops); 1702 int bd_prepare_to_claim(struct block_device *bdev, void *holder, 1703 const struct blk_holder_ops *hops); 1704 void bd_abort_claiming(struct block_device *bdev, void *holder); 1705 1706 struct block_device *I_BDEV(struct inode *inode); 1707 struct block_device *file_bdev(struct file *bdev_file); 1708 bool disk_live(struct gendisk *disk); 1709 unsigned int block_size(struct block_device *bdev); 1710 1711 #ifdef CONFIG_BLOCK 1712 void invalidate_bdev(struct block_device *bdev); 1713 int sync_blockdev(struct block_device *bdev); 1714 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend); 1715 int sync_blockdev_nowait(struct block_device *bdev); 1716 void sync_bdevs(bool wait); 1717 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask); 1718 void printk_all_partitions(void); 1719 int __init early_lookup_bdev(const char *pathname, dev_t *dev); 1720 #else 1721 static inline void invalidate_bdev(struct block_device *bdev) 1722 { 1723 } 1724 static inline int sync_blockdev(struct block_device *bdev) 1725 { 1726 return 0; 1727 } 1728 static inline int sync_blockdev_nowait(struct block_device *bdev) 1729 { 1730 return 0; 1731 } 1732 static inline void sync_bdevs(bool wait) 1733 { 1734 } 1735 static inline void bdev_statx(const struct path *path, struct kstat *stat, 1736 u32 request_mask) 1737 { 1738 } 1739 static inline void printk_all_partitions(void) 1740 { 1741 } 1742 static inline int early_lookup_bdev(const char *pathname, dev_t *dev) 1743 { 1744 return -EINVAL; 1745 } 1746 #endif /* CONFIG_BLOCK */ 1747 1748 int bdev_freeze(struct block_device *bdev); 1749 int bdev_thaw(struct block_device *bdev); 1750 int bdev_deny_freeze(struct block_device *bdev); 1751 void bdev_allow_freeze(struct block_device *bdev); 1752 void bdev_fput(struct file *bdev_file); 1753 void bdev_yield_claim(struct file *bdev_file); 1754 1755 struct io_comp_batch { 1756 struct rq_list req_list; 1757 bool need_ts; 1758 void (*complete)(struct io_comp_batch *); 1759 void *poll_ctx; 1760 }; 1761 1762 static inline bool blk_atomic_write_start_sect_aligned(sector_t sector, 1763 struct queue_limits *limits) 1764 { 1765 unsigned int alignment = max(limits->atomic_write_hw_unit_min, 1766 limits->atomic_write_hw_boundary); 1767 1768 return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT); 1769 } 1770 1771 static inline bool bdev_can_atomic_write(struct block_device *bdev) 1772 { 1773 struct request_queue *bd_queue = bdev->bd_queue; 1774 struct queue_limits *limits = &bd_queue->limits; 1775 1776 if (!limits->atomic_write_unit_min) 1777 return false; 1778 1779 if (bdev_is_partition(bdev)) 1780 return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect, 1781 limits); 1782 1783 return true; 1784 } 1785 1786 static inline unsigned int 1787 bdev_atomic_write_unit_min_bytes(struct block_device *bdev) 1788 { 1789 if (!bdev_can_atomic_write(bdev)) 1790 return 0; 1791 return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev)); 1792 } 1793 1794 static inline unsigned int 1795 bdev_atomic_write_unit_max_bytes(struct block_device *bdev) 1796 { 1797 if (!bdev_can_atomic_write(bdev)) 1798 return 0; 1799 return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev)); 1800 } 1801 1802 static inline int bio_split_rw_at(struct bio *bio, 1803 const struct queue_limits *lim, 1804 unsigned *segs, unsigned max_bytes) 1805 { 1806 return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment); 1807 } 1808 1809 /* 1810 * Maximum contiguous integrity buffer allocation. 1811 */ 1812 #define BLK_INTEGRITY_MAX_SIZE SZ_2M 1813 1814 /* 1815 * Maximum size of I/O that needs a block layer integrity buffer. Limited 1816 * by the number of intervals for which we can fit the integrity buffer into 1817 * the buffer size. Because the buffer is a single segment it is also limited 1818 * by the maximum segment size. 1819 */ 1820 static inline unsigned int max_integrity_io_size(struct queue_limits *lim) 1821 { 1822 unsigned int buf_size = min_t(unsigned int, lim->max_segment_size, 1823 BLK_INTEGRITY_MAX_SIZE); 1824 1825 return min_t(u64, UINT_MAX, 1826 (u64)(buf_size / lim->integrity.metadata_size) << 1827 lim->integrity.interval_exp); 1828 } 1829 1830 #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { } 1831 1832 #endif /* _LINUX_BLKDEV_H */ 1833