1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Portions Copyright (C) 1992 Drew Eckhardt 4 */ 5 #ifndef _LINUX_BLKDEV_H 6 #define _LINUX_BLKDEV_H 7 8 #include <linux/types.h> 9 #include <linux/blk_types.h> 10 #include <linux/blk_plug.h> 11 #include <linux/device.h> 12 #include <linux/list.h> 13 #include <linux/llist.h> 14 #include <linux/minmax.h> 15 #include <linux/timer.h> 16 #include <linux/workqueue.h> 17 #include <linux/completion.h> 18 #include <linux/wait.h> 19 #include <linux/bio.h> 20 #include <linux/gfp.h> 21 #include <linux/kdev_t.h> 22 #include <linux/rcupdate.h> 23 #include <linux/percpu-refcount.h> 24 #include <linux/blkzoned.h> 25 #include <linux/sbitmap.h> 26 #include <linux/uuid.h> 27 #include <linux/xarray.h> 28 #include <linux/file.h> 29 #include <linux/lockdep.h> 30 31 struct module; 32 struct request_queue; 33 struct elevator_queue; 34 struct blk_trace; 35 struct request; 36 struct sg_io_hdr; 37 struct blkcg_gq; 38 struct blk_flush_queue; 39 struct kiocb; 40 struct pr_ops; 41 struct rq_qos; 42 struct hd_geometry; 43 struct blk_report_zones_args; 44 struct blk_queue_stats; 45 struct blk_stat_callback; 46 struct blk_crypto_profile; 47 48 extern const struct device_type disk_type; 49 extern const struct device_type part_type; 50 extern const struct class block_class; 51 52 /* 53 * Maximum number of blkcg policies allowed to be registered concurrently. 54 * Defined here to simplify include dependency. 55 */ 56 #define BLKCG_MAX_POLS 6 57 58 #define DISK_MAX_PARTS 256 59 #define DISK_NAME_LEN 32 60 61 #define PARTITION_META_INFO_VOLNAMELTH 64 62 /* 63 * Enough for the string representation of any kind of UUID plus NULL. 64 * EFI UUID is 36 characters. MSDOS UUID is 11 characters. 65 */ 66 #define PARTITION_META_INFO_UUIDLTH (UUID_STRING_LEN + 1) 67 68 struct partition_meta_info { 69 char uuid[PARTITION_META_INFO_UUIDLTH]; 70 u8 volname[PARTITION_META_INFO_VOLNAMELTH]; 71 }; 72 73 /** 74 * DOC: genhd capability flags 75 * 76 * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to 77 * removable media. When set, the device remains present even when media is not 78 * inserted. Shall not be set for devices which are removed entirely when the 79 * media is removed. 80 * 81 * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events, 82 * doesn't appear in sysfs, and can't be opened from userspace or using 83 * blkdev_get*. Used for the underlying components of multipath devices. 84 * 85 * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not 86 * scan for partitions from add_disk, and users can't add partitions manually. 87 * 88 */ 89 enum { 90 GENHD_FL_REMOVABLE = 1 << 0, 91 GENHD_FL_HIDDEN = 1 << 1, 92 GENHD_FL_NO_PART = 1 << 2, 93 }; 94 95 enum { 96 DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */ 97 DISK_EVENT_EJECT_REQUEST = 1 << 1, /* eject requested */ 98 }; 99 100 enum { 101 /* Poll even if events_poll_msecs is unset */ 102 DISK_EVENT_FLAG_POLL = 1 << 0, 103 /* Forward events to udev */ 104 DISK_EVENT_FLAG_UEVENT = 1 << 1, 105 /* Block event polling when open for exclusive write */ 106 DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2, 107 }; 108 109 struct disk_events; 110 struct badblocks; 111 112 enum blk_integrity_checksum { 113 BLK_INTEGRITY_CSUM_NONE = 0, 114 BLK_INTEGRITY_CSUM_IP = 1, 115 BLK_INTEGRITY_CSUM_CRC = 2, 116 BLK_INTEGRITY_CSUM_CRC64 = 3, 117 } __packed ; 118 119 struct blk_integrity { 120 unsigned char flags; 121 enum blk_integrity_checksum csum_type; 122 unsigned char metadata_size; 123 unsigned char pi_offset; 124 unsigned char interval_exp; 125 unsigned char tag_size; 126 unsigned char pi_tuple_size; 127 }; 128 129 /* open for reading */ 130 #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0)) 131 /* open for writing */ 132 #define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1)) 133 /* open exclusively (vs other exclusive openers */ 134 #define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2)) 135 /* opened with O_NDELAY */ 136 #define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3)) 137 /* open for "writes" only for ioctls (specialy hack for floppy.c) */ 138 #define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4)) 139 /* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */ 140 #define BLK_OPEN_RESTRICT_WRITES ((__force blk_mode_t)(1 << 5)) 141 /* return partition scanning errors */ 142 #define BLK_OPEN_STRICT_SCAN ((__force blk_mode_t)(1 << 6)) 143 144 struct gendisk { 145 /* 146 * major/first_minor/minors should not be set by any new driver, the 147 * block core will take care of allocating them automatically. 148 */ 149 int major; 150 int first_minor; 151 int minors; 152 153 char disk_name[DISK_NAME_LEN]; /* name of major driver */ 154 155 unsigned short events; /* supported events */ 156 unsigned short event_flags; /* flags related to event processing */ 157 158 struct xarray part_tbl; 159 struct block_device *part0; 160 161 const struct block_device_operations *fops; 162 struct request_queue *queue; 163 void *private_data; 164 165 struct bio_set bio_split; 166 167 int flags; 168 unsigned long state; 169 #define GD_NEED_PART_SCAN 0 170 #define GD_READ_ONLY 1 171 #define GD_DEAD 2 172 #define GD_NATIVE_CAPACITY 3 173 #define GD_ADDED 4 174 #define GD_SUPPRESS_PART_SCAN 5 175 #define GD_OWNS_QUEUE 6 176 #define GD_ZONE_APPEND_USED 7 177 #define GD_ERROR_INJECT 8 178 179 struct mutex open_mutex; /* open/close mutex */ 180 unsigned open_partitions; /* number of open partitions */ 181 182 struct backing_dev_info *bdi; 183 struct kobject queue_kobj; /* the queue/ directory */ 184 struct kobject *slave_dir; 185 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 186 struct list_head slave_bdevs; 187 #endif 188 struct timer_rand_state *random; 189 struct disk_events *ev; 190 191 #ifdef CONFIG_BLK_DEV_ZONED 192 /* 193 * Zoned block device information. Reads of this information must be 194 * protected with blk_queue_enter() / blk_queue_exit() or by holding a 195 * lock on zone_revalidate_mutex. blk_revalidate_disk_zones() may modify 196 * this information while no requests are being processed (disk queue 197 * frozen with blk_mq_freeze_queue()) and while holding a lock on 198 * zone_revalidate_mutex. 199 */ 200 struct mutex zone_revalidate_mutex; 201 unsigned int nr_zones; 202 unsigned int zone_capacity; 203 unsigned int last_zone_capacity; 204 u8 __rcu *zones_state; 205 unsigned int zone_wplugs_hash_bits; 206 atomic_t nr_zone_wplugs; 207 spinlock_t zone_wplugs_hash_lock; 208 struct mempool *zone_wplugs_pool; 209 struct hlist_head *zone_wplugs_hash; 210 struct workqueue_struct *zone_wplugs_wq; 211 spinlock_t zone_wplugs_list_lock; 212 struct list_head zone_wplugs_list; 213 struct task_struct *zone_wplugs_worker; 214 struct completion zone_wplugs_worker_bio_done; 215 #endif /* CONFIG_BLK_DEV_ZONED */ 216 217 #if IS_ENABLED(CONFIG_CDROM) 218 struct cdrom_device_info *cdi; 219 #endif 220 int node_id; 221 struct badblocks *bb; 222 struct lockdep_map lockdep_map; 223 u64 diskseq; 224 blk_mode_t open_mode; 225 226 /* 227 * Independent sector access ranges. This is always NULL for 228 * devices that do not have multiple independent access ranges. 229 */ 230 struct blk_independent_access_ranges *ia_ranges; 231 232 #ifdef CONFIG_BLK_ERROR_INJECTION 233 struct mutex error_injection_lock; 234 struct list_head error_injection_list; 235 #endif 236 237 struct mutex rqos_state_mutex; /* rqos state change mutex */ 238 }; 239 240 /** 241 * disk_openers - returns how many openers are there for a disk 242 * @disk: disk to check 243 * 244 * This returns the number of openers for a disk. Note that this value is only 245 * stable if disk->open_mutex is held. 246 * 247 * Note: Due to a quirk in the block layer open code, each open partition is 248 * only counted once even if there are multiple openers. 249 */ 250 static inline unsigned int disk_openers(struct gendisk *disk) 251 { 252 return atomic_read(&disk->part0->bd_openers); 253 } 254 255 /** 256 * disk_has_partscan - return %true if partition scanning is enabled on a disk 257 * @disk: disk to check 258 * 259 * Returns %true if partitions scanning is enabled for @disk, or %false if 260 * partition scanning is disabled either permanently or temporarily. 261 */ 262 static inline bool disk_has_partscan(struct gendisk *disk) 263 { 264 return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) && 265 !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state); 266 } 267 268 /* 269 * The gendisk is refcounted by the part0 block_device, and the bd_device 270 * therein is also used for device model presentation in sysfs. 271 */ 272 #define dev_to_disk(device) \ 273 (dev_to_bdev(device)->bd_disk) 274 #define disk_to_dev(disk) \ 275 (&((disk)->part0->bd_device)) 276 277 #if IS_REACHABLE(CONFIG_CDROM) 278 #define disk_to_cdi(disk) ((disk)->cdi) 279 #else 280 #define disk_to_cdi(disk) NULL 281 #endif 282 283 static inline dev_t disk_devt(struct gendisk *disk) 284 { 285 return MKDEV(disk->major, disk->first_minor); 286 } 287 288 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 289 /* 290 * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER) 291 * however we constrain this to what we can validate and test. 292 */ 293 #define BLK_MAX_BLOCK_SIZE SZ_64K 294 #else 295 #define BLK_MAX_BLOCK_SIZE PAGE_SIZE 296 #endif 297 298 299 /* blk_validate_limits() validates bsize, so drivers don't usually need to */ 300 static inline int blk_validate_block_size(unsigned long bsize) 301 { 302 if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize)) 303 return -EINVAL; 304 305 return 0; 306 } 307 308 static inline bool blk_op_is_passthrough(blk_opf_t op) 309 { 310 op &= REQ_OP_MASK; 311 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT; 312 } 313 314 /* flags set by the driver in queue_limits.features */ 315 typedef unsigned int __bitwise blk_features_t; 316 317 /* supports a volatile write cache */ 318 #define BLK_FEAT_WRITE_CACHE ((__force blk_features_t)(1u << 0)) 319 320 /* supports passing on the FUA bit */ 321 #define BLK_FEAT_FUA ((__force blk_features_t)(1u << 1)) 322 323 /* rotational device (hard drive or floppy) */ 324 #define BLK_FEAT_ROTATIONAL ((__force blk_features_t)(1u << 2)) 325 326 /* contributes to the random number pool */ 327 #define BLK_FEAT_ADD_RANDOM ((__force blk_features_t)(1u << 3)) 328 329 /* do disk/partitions IO accounting */ 330 #define BLK_FEAT_IO_STAT ((__force blk_features_t)(1u << 4)) 331 332 /* don't modify data until writeback is done */ 333 #define BLK_FEAT_STABLE_WRITES ((__force blk_features_t)(1u << 5)) 334 335 /* always completes in submit context */ 336 #define BLK_FEAT_SYNCHRONOUS ((__force blk_features_t)(1u << 6)) 337 338 /* supports REQ_NOWAIT */ 339 #define BLK_FEAT_NOWAIT ((__force blk_features_t)(1u << 7)) 340 341 /* supports DAX */ 342 #define BLK_FEAT_DAX ((__force blk_features_t)(1u << 8)) 343 344 /* supports I/O polling */ 345 #define BLK_FEAT_POLL ((__force blk_features_t)(1u << 9)) 346 347 /* is a zoned device */ 348 #define BLK_FEAT_ZONED ((__force blk_features_t)(1u << 10)) 349 350 /* supports PCI(e) p2p requests */ 351 #define BLK_FEAT_PCI_P2PDMA ((__force blk_features_t)(1u << 12)) 352 353 /* skip this queue in blk_mq_(un)quiesce_tagset */ 354 #define BLK_FEAT_SKIP_TAGSET_QUIESCE ((__force blk_features_t)(1u << 13)) 355 356 /* atomic writes enabled */ 357 #define BLK_FEAT_ATOMIC_WRITES ((__force blk_features_t)(1u << 14)) 358 359 /* undocumented magic for bcache */ 360 #define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \ 361 ((__force blk_features_t)(1u << 15)) 362 363 /* 364 * Flags automatically inherited when stacking limits. 365 */ 366 #define BLK_FEAT_INHERIT_MASK \ 367 (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \ 368 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | \ 369 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE) 370 371 /* internal flags in queue_limits.flags */ 372 typedef unsigned int __bitwise blk_flags_t; 373 374 /* do not send FLUSH/FUA commands despite advertising a write cache */ 375 #define BLK_FLAG_WRITE_CACHE_DISABLED ((__force blk_flags_t)(1u << 0)) 376 377 /* I/O topology is misaligned */ 378 #define BLK_FLAG_MISALIGNED ((__force blk_flags_t)(1u << 1)) 379 380 /* passthrough command IO accounting */ 381 #define BLK_FLAG_IOSTATS_PASSTHROUGH ((__force blk_flags_t)(1u << 2)) 382 383 struct queue_limits { 384 blk_features_t features; 385 blk_flags_t flags; 386 unsigned long seg_boundary_mask; 387 unsigned long virt_boundary_mask; 388 389 unsigned int max_hw_sectors; 390 unsigned int max_dev_sectors; 391 unsigned int chunk_sectors; 392 unsigned int max_sectors; 393 unsigned int max_user_sectors; 394 unsigned int max_segment_size; 395 unsigned int max_fast_segment_size; 396 unsigned int physical_block_size; 397 unsigned int logical_block_size; 398 unsigned int alignment_offset; 399 unsigned int io_min; 400 unsigned int io_opt; 401 unsigned int max_discard_sectors; 402 unsigned int max_hw_discard_sectors; 403 unsigned int max_user_discard_sectors; 404 unsigned int max_secure_erase_sectors; 405 unsigned int max_write_zeroes_sectors; 406 unsigned int max_wzeroes_unmap_sectors; 407 unsigned int max_hw_wzeroes_unmap_sectors; 408 unsigned int max_user_wzeroes_unmap_sectors; 409 unsigned int max_hw_zone_append_sectors; 410 unsigned int max_zone_append_sectors; 411 unsigned int discard_granularity; 412 unsigned int discard_alignment; 413 unsigned int zone_write_granularity; 414 415 /* atomic write limits */ 416 unsigned int atomic_write_hw_max; 417 unsigned int atomic_write_max_sectors; 418 unsigned int atomic_write_hw_boundary; 419 unsigned int atomic_write_boundary_sectors; 420 unsigned int atomic_write_hw_unit_min; 421 unsigned int atomic_write_unit_min; 422 unsigned int atomic_write_hw_unit_max; 423 unsigned int atomic_write_unit_max; 424 425 unsigned short max_segments; 426 unsigned short max_integrity_segments; 427 unsigned short max_discard_segments; 428 429 unsigned short max_write_streams; 430 unsigned int write_stream_granularity; 431 432 unsigned int max_open_zones; 433 unsigned int max_active_zones; 434 435 /* 436 * Drivers that set dma_alignment to less than 511 must be prepared to 437 * handle individual bvec's that are not a multiple of a SECTOR_SIZE 438 * due to possible offsets. 439 */ 440 unsigned int dma_alignment; 441 unsigned int dma_pad_mask; 442 443 struct blk_integrity integrity; 444 }; 445 446 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, 447 void *data); 448 449 int disk_report_zone(struct gendisk *disk, struct blk_zone *zone, 450 unsigned int idx, struct blk_report_zones_args *args); 451 452 int blkdev_get_zone_info(struct block_device *bdev, sector_t sector, 453 struct blk_zone *zone); 454 455 #define BLK_ALL_ZONES ((unsigned int)-1) 456 int blkdev_report_zones(struct block_device *bdev, sector_t sector, 457 unsigned int nr_zones, report_zones_cb cb, void *data); 458 int blkdev_report_zones_cached(struct block_device *bdev, sector_t sector, 459 unsigned int nr_zones, report_zones_cb cb, void *data); 460 int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, 461 sector_t sectors, sector_t nr_sectors); 462 int blk_revalidate_disk_zones(struct gendisk *disk); 463 464 /* 465 * Independent access ranges: struct blk_independent_access_range describes 466 * a range of contiguous sectors that can be accessed using device command 467 * execution resources that are independent from the resources used for 468 * other access ranges. This is typically found with single-LUN multi-actuator 469 * HDDs where each access range is served by a different set of heads. 470 * The set of independent ranges supported by the device is defined using 471 * struct blk_independent_access_ranges. The independent ranges must not overlap 472 * and must include all sectors within the disk capacity (no sector holes 473 * allowed). 474 * For a device with multiple ranges, requests targeting sectors in different 475 * ranges can be executed in parallel. A request can straddle an access range 476 * boundary. 477 */ 478 struct blk_independent_access_range { 479 struct kobject kobj; 480 sector_t sector; 481 sector_t nr_sectors; 482 }; 483 484 struct blk_independent_access_ranges { 485 struct kobject kobj; 486 bool sysfs_registered; 487 unsigned int nr_ia_ranges; 488 struct blk_independent_access_range ia_range[]; 489 }; 490 491 struct request_queue { 492 /* 493 * The queue owner gets to use this for whatever they like. 494 * ll_rw_blk doesn't touch it. 495 */ 496 void *queuedata; 497 498 struct elevator_queue *elevator; 499 500 const struct blk_mq_ops *mq_ops; 501 502 /* sw queues */ 503 struct blk_mq_ctx __percpu *queue_ctx; 504 505 /* 506 * various queue flags, see QUEUE_* below 507 */ 508 unsigned long queue_flags; 509 510 unsigned int __data_racy rq_timeout; 511 512 unsigned int queue_depth; 513 514 refcount_t refs; 515 516 /* hw dispatch queues */ 517 unsigned int nr_hw_queues; 518 struct blk_mq_hw_ctx * __rcu *queue_hw_ctx __counted_by_ptr(nr_hw_queues); 519 520 struct percpu_ref q_usage_counter; 521 struct lock_class_key io_lock_cls_key; 522 struct lockdep_map io_lockdep_map; 523 524 struct lock_class_key q_lock_cls_key; 525 struct lockdep_map q_lockdep_map; 526 527 struct request *last_merge; 528 529 spinlock_t queue_lock; 530 531 int quiesce_depth; 532 533 struct gendisk *disk; 534 535 /* 536 * mq queue kobject 537 */ 538 struct kobject *mq_kobj; 539 540 struct queue_limits limits; 541 542 #ifdef CONFIG_PM 543 struct device *dev; 544 enum rpm_status rpm_status; 545 #endif 546 547 /* 548 * Number of contexts that have called blk_set_pm_only(). If this 549 * counter is above zero then only RQF_PM requests are processed. 550 */ 551 atomic_t pm_only; 552 553 struct blk_queue_stats *stats; 554 struct rq_qos *rq_qos; 555 struct mutex rq_qos_mutex; 556 557 /* 558 * ida allocated id for this queue. Used to index queues from 559 * ioctx. 560 */ 561 int id; 562 563 /* 564 * queue settings 565 */ 566 unsigned int nr_requests; /* Max # of requests */ 567 unsigned int async_depth; /* Max # of async requests */ 568 569 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 570 struct blk_crypto_profile *crypto_profile; 571 struct kobject *crypto_kobject; 572 #endif 573 574 struct timer_list timeout; 575 struct work_struct timeout_work; 576 577 atomic_t nr_active_requests_shared_tags; 578 579 struct blk_mq_tags *sched_shared_tags; 580 581 struct list_head icq_list; 582 #ifdef CONFIG_BLK_CGROUP 583 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); 584 struct blkcg_gq *root_blkg; 585 struct list_head blkg_list; 586 struct mutex blkcg_mutex; 587 #endif 588 589 int node; 590 591 spinlock_t requeue_lock; 592 struct list_head requeue_list; 593 struct delayed_work requeue_work; 594 595 #ifdef CONFIG_BLK_DEV_IO_TRACE 596 struct blk_trace __rcu *blk_trace; 597 #endif 598 /* 599 * for flush operations 600 */ 601 struct blk_flush_queue *fq; 602 struct list_head flush_list; 603 604 /* 605 * Protects against I/O scheduler switching, particularly when updating 606 * q->elevator. Since the elevator update code path may also modify q-> 607 * nr_requests and wbt latency, this lock also protects the sysfs attrs 608 * nr_requests and wbt_lat_usec. Additionally the nr_hw_queues update 609 * may modify hctx tags, reserved-tags and cpumask, so this lock also 610 * helps protect the hctx sysfs/debugfs attrs. To ensure proper locking 611 * order during an elevator or nr_hw_queue update, first freeze the 612 * queue, then acquire ->elevator_lock. 613 */ 614 struct mutex elevator_lock; 615 616 struct mutex sysfs_lock; 617 /* 618 * Protects queue limits and also sysfs attribute read_ahead_kb. 619 */ 620 struct mutex limits_lock; 621 622 /* 623 * for reusing dead hctx instance in case of updating 624 * nr_hw_queues 625 */ 626 struct list_head unused_hctx_list; 627 spinlock_t unused_hctx_lock; 628 629 int mq_freeze_depth; 630 631 #ifdef CONFIG_BLK_DEV_THROTTLING 632 /* Throttle data */ 633 struct throtl_data *td; 634 #endif 635 struct rcu_head rcu_head; 636 #ifdef CONFIG_LOCKDEP 637 struct task_struct *mq_freeze_owner; 638 int mq_freeze_owner_depth; 639 /* 640 * Records disk & queue state in current context, used in unfreeze 641 * queue 642 */ 643 bool mq_freeze_disk_dead; 644 bool mq_freeze_queue_dying; 645 #endif 646 wait_queue_head_t mq_freeze_wq; 647 /* 648 * Protect concurrent access to q_usage_counter by 649 * percpu_ref_kill() and percpu_ref_reinit(). 650 */ 651 struct mutex mq_freeze_lock; 652 653 struct blk_mq_tag_set *tag_set; 654 struct list_head tag_set_list; 655 656 struct dentry *debugfs_dir; 657 struct dentry *sched_debugfs_dir; 658 struct dentry *rqos_debugfs_dir; 659 /* 660 * Serializes all debugfs metadata operations using the above dentries. 661 */ 662 struct mutex debugfs_mutex; 663 }; 664 665 /* Keep blk_queue_flag_name[] in sync with the definitions below */ 666 enum { 667 QUEUE_FLAG_DYING, /* queue being torn down */ 668 QUEUE_FLAG_NOMERGES, /* disable merge attempts */ 669 QUEUE_FLAG_SAME_COMP, /* complete on same CPU-group */ 670 QUEUE_FLAG_FAIL_IO, /* fake timeout */ 671 QUEUE_FLAG_NOXMERGES, /* No extended merges */ 672 QUEUE_FLAG_SAME_FORCE, /* force complete on same CPU */ 673 QUEUE_FLAG_INIT_DONE, /* queue is initialized */ 674 QUEUE_FLAG_STATS, /* track IO start and completion times */ 675 QUEUE_FLAG_REGISTERED, /* queue has been registered to a disk */ 676 QUEUE_FLAG_QUIESCED, /* queue has been quiesced */ 677 QUEUE_FLAG_RQ_ALLOC_TIME, /* record rq->alloc_time_ns */ 678 QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */ 679 QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */ 680 QUEUE_FLAG_DISABLE_WBT_DEF, /* for sched to disable/enable wbt */ 681 QUEUE_FLAG_NO_ELV_SWITCH, /* can't switch elevator any more */ 682 QUEUE_FLAG_QOS_ENABLED, /* qos is enabled */ 683 QUEUE_FLAG_BIO_ISSUE_TIME, /* record bio->issue_time_ns */ 684 QUEUE_FLAG_ZONED_QD1_WRITES, /* Limit zoned devices writes to QD=1 */ 685 QUEUE_FLAG_MAX 686 }; 687 688 #define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP) 689 690 void blk_queue_flag_set(unsigned int flag, struct request_queue *q); 691 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); 692 693 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) 694 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags) 695 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) 696 #define blk_queue_noxmerges(q) \ 697 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) 698 #define blk_queue_rot(q) ((q)->limits.features & BLK_FEAT_ROTATIONAL) 699 #define blk_queue_io_stat(q) ((q)->limits.features & BLK_FEAT_IO_STAT) 700 #define blk_queue_passthrough_stat(q) \ 701 ((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH) 702 #define blk_queue_dax(q) ((q)->limits.features & BLK_FEAT_DAX) 703 #define blk_queue_pci_p2pdma(q) ((q)->limits.features & BLK_FEAT_PCI_P2PDMA) 704 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 705 #define blk_queue_rq_alloc_time(q) \ 706 test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags) 707 #else 708 #define blk_queue_rq_alloc_time(q) false 709 #endif 710 711 #define blk_noretry_request(rq) \ 712 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ 713 REQ_FAILFAST_DRIVER)) 714 #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) 715 #define blk_queue_pm_only(q) atomic_read(&(q)->pm_only) 716 #define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags) 717 #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags) 718 #define blk_queue_skip_tagset_quiesce(q) \ 719 ((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE) 720 #define blk_queue_disable_wbt(q) \ 721 test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags) 722 #define blk_queue_no_elv_switch(q) \ 723 test_bit(QUEUE_FLAG_NO_ELV_SWITCH, &(q)->queue_flags) 724 #define blk_queue_zoned_qd1_writes(q) \ 725 test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags) 726 727 extern void blk_set_pm_only(struct request_queue *q); 728 extern void blk_clear_pm_only(struct request_queue *q); 729 730 #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist) 731 732 #define dma_map_bvec(dev, bv, dir, attrs) \ 733 dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \ 734 (dir), (attrs)) 735 736 static inline bool queue_is_mq(struct request_queue *q) 737 { 738 return q->mq_ops; 739 } 740 741 #ifdef CONFIG_PM 742 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 743 { 744 return q->rpm_status; 745 } 746 #else 747 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 748 { 749 return RPM_ACTIVE; 750 } 751 #endif 752 753 static inline bool blk_queue_is_zoned(struct request_queue *q) 754 { 755 return IS_ENABLED(CONFIG_BLK_DEV_ZONED) && 756 (q->limits.features & BLK_FEAT_ZONED); 757 } 758 759 static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector) 760 { 761 if (!blk_queue_is_zoned(disk->queue)) 762 return 0; 763 return sector >> ilog2(disk->queue->limits.chunk_sectors); 764 } 765 766 static inline unsigned int bdev_max_open_zones(struct block_device *bdev) 767 { 768 return bdev->bd_disk->queue->limits.max_open_zones; 769 } 770 771 static inline unsigned int bdev_max_active_zones(struct block_device *bdev) 772 { 773 return bdev->bd_disk->queue->limits.max_active_zones; 774 } 775 776 static inline unsigned int blk_queue_depth(struct request_queue *q) 777 { 778 if (q->queue_depth) 779 return q->queue_depth; 780 781 return q->nr_requests; 782 } 783 784 /* 785 * default timeout for SG_IO if none specified 786 */ 787 #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ) 788 #define BLK_MIN_SG_TIMEOUT (7 * HZ) 789 790 /* This should not be used directly - use rq_for_each_segment */ 791 #define for_each_bio(_bio) \ 792 for (; _bio; _bio = _bio->bi_next) 793 794 int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, 795 const struct attribute_group **groups, 796 struct fwnode_handle *fwnode); 797 int __must_check device_add_disk(struct device *parent, struct gendisk *disk, 798 const struct attribute_group **groups); 799 static inline int __must_check add_disk(struct gendisk *disk) 800 { 801 return device_add_disk(NULL, disk, NULL); 802 } 803 void del_gendisk(struct gendisk *gp); 804 void invalidate_disk(struct gendisk *disk); 805 void set_disk_ro(struct gendisk *disk, bool read_only); 806 void disk_uevent(struct gendisk *disk, enum kobject_action action); 807 808 static inline u8 bdev_partno(const struct block_device *bdev) 809 { 810 return atomic_read(&bdev->__bd_flags) & BD_PARTNO; 811 } 812 813 static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag) 814 { 815 return atomic_read(&bdev->__bd_flags) & flag; 816 } 817 818 static inline void bdev_set_flag(struct block_device *bdev, unsigned flag) 819 { 820 atomic_or(flag, &bdev->__bd_flags); 821 } 822 823 static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag) 824 { 825 atomic_andnot(flag, &bdev->__bd_flags); 826 } 827 828 static inline bool get_disk_ro(struct gendisk *disk) 829 { 830 return bdev_test_flag(disk->part0, BD_READ_ONLY) || 831 test_bit(GD_READ_ONLY, &disk->state); 832 } 833 834 static inline bool bdev_read_only(struct block_device *bdev) 835 { 836 return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk); 837 } 838 839 bool set_capacity_and_notify(struct gendisk *disk, sector_t size); 840 void disk_force_media_change(struct gendisk *disk); 841 void bdev_mark_dead(struct block_device *bdev, bool surprise); 842 843 void add_disk_randomness(struct gendisk *disk) __latent_entropy; 844 void rand_initialize_disk(struct gendisk *disk); 845 846 static inline sector_t get_start_sect(struct block_device *bdev) 847 { 848 return bdev->bd_start_sect; 849 } 850 851 static inline sector_t bdev_nr_sectors(struct block_device *bdev) 852 { 853 return bdev->bd_nr_sectors; 854 } 855 856 static inline loff_t bdev_nr_bytes(struct block_device *bdev) 857 { 858 return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT; 859 } 860 861 static inline sector_t get_capacity(struct gendisk *disk) 862 { 863 return bdev_nr_sectors(disk->part0); 864 } 865 866 static inline u64 sb_bdev_nr_blocks(struct super_block *sb) 867 { 868 return bdev_nr_sectors(sb->s_bdev) >> 869 (sb->s_blocksize_bits - SECTOR_SHIFT); 870 } 871 872 #ifdef CONFIG_BLK_DEV_ZONED 873 static inline unsigned int disk_nr_zones(struct gendisk *disk) 874 { 875 return disk->nr_zones; 876 } 877 878 /** 879 * bio_needs_zone_write_plugging - Check if a BIO needs to be handled with zone 880 * write plugging 881 * @bio: The BIO being submitted 882 * 883 * Return true whenever @bio execution needs to be handled through zone 884 * write plugging (using blk_zone_plug_bio()). Return false otherwise. 885 */ 886 static inline bool bio_needs_zone_write_plugging(struct bio *bio) 887 { 888 enum req_op op = bio_op(bio); 889 890 /* 891 * Only zoned block devices have a zone write plug hash table. But not 892 * all of them have one (e.g. DM devices may not need one). 893 */ 894 if (!bio->bi_bdev->bd_disk->zone_wplugs_hash) 895 return false; 896 897 /* Only write operations need zone write plugging. */ 898 if (!op_is_write(op)) 899 return false; 900 901 /* Ignore empty flush */ 902 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 903 return false; 904 905 /* Ignore BIOs that already have been handled by zone write plugging. */ 906 if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING)) 907 return false; 908 909 /* 910 * All zone write operations must be handled through zone write plugging 911 * using blk_zone_plug_bio(). 912 */ 913 switch (op) { 914 case REQ_OP_ZONE_APPEND: 915 case REQ_OP_WRITE: 916 case REQ_OP_WRITE_ZEROES: 917 case REQ_OP_ZONE_FINISH: 918 case REQ_OP_ZONE_RESET: 919 case REQ_OP_ZONE_RESET_ALL: 920 return true; 921 default: 922 return false; 923 } 924 } 925 926 bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs); 927 928 /** 929 * disk_zone_capacity - returns the zone capacity of zone containing @sector 930 * @disk: disk to work with 931 * @sector: sector number within the querying zone 932 * 933 * Returns the zone capacity of a zone containing @sector. @sector can be any 934 * sector in the zone. 935 */ 936 static inline unsigned int disk_zone_capacity(struct gendisk *disk, 937 sector_t sector) 938 { 939 sector_t zone_sectors = disk->queue->limits.chunk_sectors; 940 941 if (sector + zone_sectors >= get_capacity(disk)) 942 return disk->last_zone_capacity; 943 return disk->zone_capacity; 944 } 945 static inline unsigned int bdev_zone_capacity(struct block_device *bdev, 946 sector_t pos) 947 { 948 return disk_zone_capacity(bdev->bd_disk, pos); 949 } 950 951 bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector); 952 953 #else /* CONFIG_BLK_DEV_ZONED */ 954 static inline unsigned int disk_nr_zones(struct gendisk *disk) 955 { 956 return 0; 957 } 958 959 static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector) 960 { 961 return false; 962 } 963 964 static inline bool bio_needs_zone_write_plugging(struct bio *bio) 965 { 966 return false; 967 } 968 969 static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) 970 { 971 return false; 972 } 973 #endif /* CONFIG_BLK_DEV_ZONED */ 974 975 static inline unsigned int bdev_nr_zones(struct block_device *bdev) 976 { 977 return disk_nr_zones(bdev->bd_disk); 978 } 979 980 int bdev_disk_changed(struct gendisk *disk, bool invalidate); 981 982 void put_disk(struct gendisk *disk); 983 struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node, 984 struct lock_class_key *lkclass); 985 986 /** 987 * blk_alloc_disk - allocate a gendisk structure 988 * @lim: queue limits to be used for this disk. 989 * @node_id: numa node to allocate on 990 * 991 * Allocate and pre-initialize a gendisk structure for use with BIO based 992 * drivers. 993 * 994 * Returns an ERR_PTR on error, else the allocated disk. 995 * 996 * Context: can sleep 997 */ 998 #define blk_alloc_disk(lim, node_id) \ 999 ({ \ 1000 static struct lock_class_key __key; \ 1001 \ 1002 __blk_alloc_disk(lim, node_id, &__key); \ 1003 }) 1004 1005 int __register_blkdev(unsigned int major, const char *name, 1006 void (*probe)(dev_t devt)); 1007 #define register_blkdev(major, name) \ 1008 __register_blkdev(major, name, NULL) 1009 void unregister_blkdev(unsigned int major, const char *name); 1010 1011 bool disk_check_media_change(struct gendisk *disk); 1012 void set_capacity(struct gendisk *disk, sector_t size); 1013 1014 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 1015 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk); 1016 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk); 1017 #else 1018 static inline int bd_link_disk_holder(struct block_device *bdev, 1019 struct gendisk *disk) 1020 { 1021 return 0; 1022 } 1023 static inline void bd_unlink_disk_holder(struct block_device *bdev, 1024 struct gendisk *disk) 1025 { 1026 } 1027 #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */ 1028 1029 dev_t part_devt(struct gendisk *disk, u8 partno); 1030 void inc_diskseq(struct gendisk *disk); 1031 void blk_request_module(dev_t devt); 1032 1033 extern int blk_register_queue(struct gendisk *disk); 1034 extern void blk_unregister_queue(struct gendisk *disk); 1035 void submit_bio_noacct(struct bio *bio); 1036 struct bio *bio_split_to_limits(struct bio *bio); 1037 struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors, 1038 struct bio_set *bs); 1039 1040 extern int blk_lld_busy(struct request_queue *q); 1041 extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags); 1042 extern void blk_queue_exit(struct request_queue *q); 1043 extern void blk_sync_queue(struct request_queue *q); 1044 1045 /* Convert a request operation REQ_OP_name into the string "name" */ 1046 extern const char *blk_op_str(enum req_op op); 1047 1048 int blk_status_to_errno(blk_status_t status); 1049 blk_status_t errno_to_blk_status(int errno); 1050 1051 /* only poll the hardware once, don't continue until a completion was found */ 1052 #define BLK_POLL_ONESHOT (1 << 0) 1053 int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags); 1054 int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob, 1055 unsigned int flags); 1056 1057 static inline struct request_queue *bdev_get_queue(struct block_device *bdev) 1058 { 1059 return bdev->bd_queue; /* this is never NULL */ 1060 } 1061 1062 /* Convert a zone condition BLK_ZONE_COND_name into the string "name" */ 1063 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond); 1064 1065 static inline unsigned int bio_zone_no(struct bio *bio) 1066 { 1067 return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector); 1068 } 1069 1070 static inline bool bio_straddles_zones(struct bio *bio) 1071 { 1072 return bio_sectors(bio) && 1073 bio_zone_no(bio) != 1074 disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1); 1075 } 1076 1077 /* 1078 * Return how much within the boundary is left to be used for I/O at a given 1079 * offset. 1080 */ 1081 static inline unsigned int blk_boundary_sectors_left(sector_t offset, 1082 unsigned int boundary_sectors) 1083 { 1084 if (unlikely(!is_power_of_2(boundary_sectors))) 1085 return boundary_sectors - sector_div(offset, boundary_sectors); 1086 return boundary_sectors - (offset & (boundary_sectors - 1)); 1087 } 1088 1089 /** 1090 * queue_limits_start_update - start an atomic update of queue limits 1091 * @q: queue to update 1092 * 1093 * This functions starts an atomic update of the queue limits. It takes a lock 1094 * to prevent other updates and returns a snapshot of the current limits that 1095 * the caller can modify. The caller must call queue_limits_commit_update() 1096 * to finish the update. 1097 * 1098 * Context: process context. 1099 */ 1100 static inline struct queue_limits 1101 queue_limits_start_update(struct request_queue *q) 1102 __acquires(&q->limits_lock) 1103 { 1104 mutex_lock(&q->limits_lock); 1105 return q->limits; 1106 } 1107 int queue_limits_commit_update_frozen(struct request_queue *q, 1108 struct queue_limits *lim) __releases(&q->limits_lock); 1109 int queue_limits_commit_update(struct request_queue *q, 1110 struct queue_limits *lim) __releases(&q->limits_lock); 1111 int queue_limits_set(struct request_queue *q, struct queue_limits *lim) 1112 __must_not_hold(&q->limits_lock); 1113 int blk_validate_limits(struct queue_limits *lim); 1114 1115 /** 1116 * queue_limits_cancel_update - cancel an atomic update of queue limits 1117 * @q: queue to update 1118 * 1119 * This functions cancels an atomic update of the queue limits started by 1120 * queue_limits_start_update() and should be used when an error occurs after 1121 * starting update. 1122 */ 1123 static inline void queue_limits_cancel_update(struct request_queue *q) 1124 __releases(&q->limits_lock) 1125 { 1126 mutex_unlock(&q->limits_lock); 1127 } 1128 1129 /* 1130 * These helpers are for drivers that have sloppy feature negotiation and might 1131 * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O 1132 * completion handler when the device returned an indicator that the respective 1133 * feature is not actually supported. They are racy and the driver needs to 1134 * cope with that. Try to avoid this scheme if you can. 1135 */ 1136 static inline void blk_queue_disable_discard(struct request_queue *q) 1137 { 1138 q->limits.max_discard_sectors = 0; 1139 } 1140 1141 static inline void blk_queue_disable_secure_erase(struct request_queue *q) 1142 { 1143 q->limits.max_secure_erase_sectors = 0; 1144 } 1145 1146 static inline void blk_queue_disable_write_zeroes(struct request_queue *q) 1147 { 1148 q->limits.max_write_zeroes_sectors = 0; 1149 q->limits.max_wzeroes_unmap_sectors = 0; 1150 } 1151 1152 /* 1153 * Access functions for manipulating queue properties 1154 */ 1155 extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth); 1156 extern void blk_set_stacking_limits(struct queue_limits *lim); 1157 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, 1158 sector_t offset); 1159 void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev, 1160 sector_t offset, const char *pfx); 1161 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int); 1162 1163 struct blk_independent_access_ranges * 1164 disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges); 1165 void disk_set_independent_access_ranges(struct gendisk *disk, 1166 struct blk_independent_access_ranges *iars); 1167 1168 bool __must_check blk_get_queue(struct request_queue *); 1169 extern void blk_put_queue(struct request_queue *); 1170 1171 void blk_mark_disk_dead(struct gendisk *disk); 1172 1173 #ifdef CONFIG_BLOCK 1174 int blkdev_issue_flush(struct block_device *bdev); 1175 long nr_blockdev_pages(void); 1176 #else /* CONFIG_BLOCK */ 1177 static inline int blkdev_issue_flush(struct block_device *bdev) 1178 { 1179 return 0; 1180 } 1181 1182 static inline long nr_blockdev_pages(void) 1183 { 1184 return 0; 1185 } 1186 #endif /* CONFIG_BLOCK */ 1187 1188 extern void blk_io_schedule(void); 1189 1190 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1191 sector_t nr_sects, gfp_t gfp_mask); 1192 void __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1193 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop); 1194 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 1195 sector_t nr_sects, gfp_t gfp); 1196 1197 #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */ 1198 #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */ 1199 #define BLKDEV_ZERO_KILLABLE (1 << 2) /* interruptible by fatal signals */ 1200 1201 extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1202 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 1203 unsigned flags); 1204 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1205 sector_t nr_sects, gfp_t gfp_mask, unsigned flags); 1206 1207 static inline int sb_issue_discard(struct super_block *sb, sector_t block, 1208 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) 1209 { 1210 return blkdev_issue_discard(sb->s_bdev, 1211 block << (sb->s_blocksize_bits - 1212 SECTOR_SHIFT), 1213 nr_blocks << (sb->s_blocksize_bits - 1214 SECTOR_SHIFT), 1215 gfp_mask); 1216 } 1217 static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, 1218 sector_t nr_blocks, gfp_t gfp_mask) 1219 { 1220 return blkdev_issue_zeroout(sb->s_bdev, 1221 block << (sb->s_blocksize_bits - 1222 SECTOR_SHIFT), 1223 nr_blocks << (sb->s_blocksize_bits - 1224 SECTOR_SHIFT), 1225 gfp_mask, 0); 1226 } 1227 1228 static inline bool bdev_is_partition(struct block_device *bdev) 1229 { 1230 return bdev_partno(bdev) != 0; 1231 } 1232 1233 enum blk_default_limits { 1234 BLK_MAX_SEGMENTS = 128, 1235 BLK_SAFE_MAX_SECTORS = 255, 1236 BLK_MAX_SEGMENT_SIZE = 65536, 1237 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1238 }; 1239 1240 static inline struct queue_limits *bdev_limits(struct block_device *bdev) 1241 { 1242 return &bdev_get_queue(bdev)->limits; 1243 } 1244 1245 static inline unsigned long queue_segment_boundary(const struct request_queue *q) 1246 { 1247 return q->limits.seg_boundary_mask; 1248 } 1249 1250 static inline unsigned long queue_virt_boundary(const struct request_queue *q) 1251 { 1252 return q->limits.virt_boundary_mask; 1253 } 1254 1255 static inline unsigned int queue_max_sectors(const struct request_queue *q) 1256 { 1257 return q->limits.max_sectors; 1258 } 1259 1260 static inline unsigned int queue_max_bytes(struct request_queue *q) 1261 { 1262 return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9; 1263 } 1264 1265 static inline unsigned int queue_max_hw_sectors(const struct request_queue *q) 1266 { 1267 return q->limits.max_hw_sectors; 1268 } 1269 1270 static inline unsigned short queue_max_segments(const struct request_queue *q) 1271 { 1272 return q->limits.max_segments; 1273 } 1274 1275 static inline unsigned short queue_max_discard_segments(const struct request_queue *q) 1276 { 1277 return q->limits.max_discard_segments; 1278 } 1279 1280 static inline unsigned int queue_max_segment_size(const struct request_queue *q) 1281 { 1282 return q->limits.max_segment_size; 1283 } 1284 1285 static inline bool queue_emulates_zone_append(struct request_queue *q) 1286 { 1287 return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors; 1288 } 1289 1290 static inline bool bdev_emulates_zone_append(struct block_device *bdev) 1291 { 1292 return queue_emulates_zone_append(bdev_get_queue(bdev)); 1293 } 1294 1295 static inline unsigned int 1296 bdev_max_zone_append_sectors(struct block_device *bdev) 1297 { 1298 return bdev_limits(bdev)->max_zone_append_sectors; 1299 } 1300 1301 static inline unsigned int bdev_max_segments(struct block_device *bdev) 1302 { 1303 return queue_max_segments(bdev_get_queue(bdev)); 1304 } 1305 1306 static inline unsigned short bdev_max_write_streams(struct block_device *bdev) 1307 { 1308 if (bdev_is_partition(bdev)) 1309 return 0; 1310 return bdev_limits(bdev)->max_write_streams; 1311 } 1312 1313 static inline unsigned queue_logical_block_size(const struct request_queue *q) 1314 { 1315 return q->limits.logical_block_size; 1316 } 1317 1318 static inline unsigned int bdev_logical_block_size(struct block_device *bdev) 1319 { 1320 return queue_logical_block_size(bdev_get_queue(bdev)); 1321 } 1322 1323 static inline unsigned int queue_physical_block_size(const struct request_queue *q) 1324 { 1325 return q->limits.physical_block_size; 1326 } 1327 1328 static inline unsigned int bdev_physical_block_size(struct block_device *bdev) 1329 { 1330 return queue_physical_block_size(bdev_get_queue(bdev)); 1331 } 1332 1333 static inline unsigned int queue_io_min(const struct request_queue *q) 1334 { 1335 return q->limits.io_min; 1336 } 1337 1338 static inline unsigned int bdev_io_min(struct block_device *bdev) 1339 { 1340 return queue_io_min(bdev_get_queue(bdev)); 1341 } 1342 1343 static inline unsigned int queue_io_opt(const struct request_queue *q) 1344 { 1345 return q->limits.io_opt; 1346 } 1347 1348 static inline unsigned int bdev_io_opt(struct block_device *bdev) 1349 { 1350 return queue_io_opt(bdev_get_queue(bdev)); 1351 } 1352 1353 static inline unsigned int 1354 queue_zone_write_granularity(const struct request_queue *q) 1355 { 1356 return q->limits.zone_write_granularity; 1357 } 1358 1359 static inline unsigned int 1360 bdev_zone_write_granularity(struct block_device *bdev) 1361 { 1362 return queue_zone_write_granularity(bdev_get_queue(bdev)); 1363 } 1364 1365 int bdev_alignment_offset(struct block_device *bdev); 1366 unsigned int bdev_discard_alignment(struct block_device *bdev); 1367 1368 static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev) 1369 { 1370 return bdev_limits(bdev)->max_discard_sectors; 1371 } 1372 1373 static inline unsigned int bdev_discard_granularity(struct block_device *bdev) 1374 { 1375 return bdev_limits(bdev)->discard_granularity; 1376 } 1377 1378 static inline unsigned int 1379 bdev_max_secure_erase_sectors(struct block_device *bdev) 1380 { 1381 return bdev_limits(bdev)->max_secure_erase_sectors; 1382 } 1383 1384 static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev) 1385 { 1386 return bdev_limits(bdev)->max_write_zeroes_sectors; 1387 } 1388 1389 static inline unsigned int 1390 bdev_write_zeroes_unmap_sectors(struct block_device *bdev) 1391 { 1392 return bdev_limits(bdev)->max_wzeroes_unmap_sectors; 1393 } 1394 1395 static inline bool bdev_rot(struct block_device *bdev) 1396 { 1397 return blk_queue_rot(bdev_get_queue(bdev)); 1398 } 1399 1400 static inline bool bdev_synchronous(struct block_device *bdev) 1401 { 1402 return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS; 1403 } 1404 1405 static inline bool bdev_has_integrity_csum(struct block_device *bdev) 1406 { 1407 struct queue_limits *lim = bdev_limits(bdev); 1408 1409 return IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && 1410 lim->integrity.csum_type != BLK_INTEGRITY_CSUM_NONE; 1411 } 1412 1413 static inline bool bdev_stable_writes(struct block_device *bdev) 1414 { 1415 return bdev_has_integrity_csum(bdev) || 1416 (bdev_limits(bdev)->features & BLK_FEAT_STABLE_WRITES); 1417 } 1418 1419 static inline bool blk_queue_write_cache(struct request_queue *q) 1420 { 1421 return (q->limits.features & BLK_FEAT_WRITE_CACHE) && 1422 !(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED); 1423 } 1424 1425 static inline bool bdev_write_cache(struct block_device *bdev) 1426 { 1427 return blk_queue_write_cache(bdev_get_queue(bdev)); 1428 } 1429 1430 static inline bool bdev_fua(struct block_device *bdev) 1431 { 1432 return bdev_limits(bdev)->features & BLK_FEAT_FUA; 1433 } 1434 1435 static inline bool bdev_nowait(struct block_device *bdev) 1436 { 1437 return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT; 1438 } 1439 1440 static inline bool bdev_is_zoned(struct block_device *bdev) 1441 { 1442 return blk_queue_is_zoned(bdev_get_queue(bdev)); 1443 } 1444 1445 static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec) 1446 { 1447 return disk_zone_no(bdev->bd_disk, sec); 1448 } 1449 1450 static inline sector_t bdev_zone_sectors(struct block_device *bdev) 1451 { 1452 struct request_queue *q = bdev_get_queue(bdev); 1453 1454 if (!blk_queue_is_zoned(q)) 1455 return 0; 1456 return q->limits.chunk_sectors; 1457 } 1458 1459 static inline sector_t bdev_zone_start(struct block_device *bdev, 1460 sector_t sector) 1461 { 1462 return sector & ~(bdev_zone_sectors(bdev) - 1); 1463 } 1464 1465 static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev, 1466 sector_t sector) 1467 { 1468 return sector & (bdev_zone_sectors(bdev) - 1); 1469 } 1470 1471 static inline sector_t bio_offset_from_zone_start(struct bio *bio) 1472 { 1473 return bdev_offset_from_zone_start(bio->bi_bdev, 1474 bio->bi_iter.bi_sector); 1475 } 1476 1477 static inline bool bdev_is_zone_start(struct block_device *bdev, 1478 sector_t sector) 1479 { 1480 return bdev_offset_from_zone_start(bdev, sector) == 0; 1481 } 1482 1483 /* Check whether @sector is a multiple of the zone size. */ 1484 static inline bool bdev_is_zone_aligned(struct block_device *bdev, 1485 sector_t sector) 1486 { 1487 return bdev_is_zone_start(bdev, sector); 1488 } 1489 1490 int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector, 1491 sector_t nr_sects, gfp_t gfp_mask); 1492 1493 static inline unsigned int queue_dma_alignment(const struct request_queue *q) 1494 { 1495 return q->limits.dma_alignment; 1496 } 1497 1498 static inline unsigned int 1499 queue_atomic_write_unit_max_bytes(const struct request_queue *q) 1500 { 1501 return q->limits.atomic_write_unit_max; 1502 } 1503 1504 static inline unsigned int 1505 queue_atomic_write_unit_min_bytes(const struct request_queue *q) 1506 { 1507 return q->limits.atomic_write_unit_min; 1508 } 1509 1510 static inline unsigned int 1511 queue_atomic_write_boundary_bytes(const struct request_queue *q) 1512 { 1513 return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT; 1514 } 1515 1516 static inline unsigned int 1517 queue_atomic_write_max_bytes(const struct request_queue *q) 1518 { 1519 return q->limits.atomic_write_max_sectors << SECTOR_SHIFT; 1520 } 1521 1522 static inline unsigned int bdev_dma_alignment(struct block_device *bdev) 1523 { 1524 return queue_dma_alignment(bdev_get_queue(bdev)); 1525 } 1526 1527 static inline unsigned int 1528 blk_lim_dma_alignment_and_pad(struct queue_limits *lim) 1529 { 1530 return lim->dma_alignment | lim->dma_pad_mask; 1531 } 1532 1533 static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr, 1534 unsigned int len) 1535 { 1536 unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits); 1537 1538 return !(addr & alignment) && !(len & alignment); 1539 } 1540 1541 /* assumes size > 256 */ 1542 static inline unsigned int blksize_bits(unsigned int size) 1543 { 1544 return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT; 1545 } 1546 1547 int kblockd_schedule_work(struct work_struct *work); 1548 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); 1549 1550 #define MODULE_ALIAS_BLOCKDEV(major,minor) \ 1551 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) 1552 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ 1553 MODULE_ALIAS("block-major-" __stringify(major) "-*") 1554 1555 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1556 1557 bool blk_crypto_register(struct blk_crypto_profile *profile, 1558 struct request_queue *q); 1559 1560 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1561 1562 static inline bool blk_crypto_register(struct blk_crypto_profile *profile, 1563 struct request_queue *q) 1564 { 1565 return true; 1566 } 1567 1568 #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ 1569 1570 enum blk_unique_id { 1571 /* these match the Designator Types specified in SPC */ 1572 BLK_UID_T10 = 1, 1573 BLK_UID_EUI64 = 2, 1574 BLK_UID_NAA = 3, 1575 }; 1576 1577 struct block_device_operations { 1578 void (*submit_bio)(struct bio *bio); 1579 int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, 1580 unsigned int flags); 1581 int (*open)(struct gendisk *disk, blk_mode_t mode); 1582 void (*release)(struct gendisk *disk); 1583 int (*ioctl)(struct block_device *bdev, blk_mode_t mode, 1584 unsigned cmd, unsigned long arg); 1585 int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode, 1586 unsigned cmd, unsigned long arg); 1587 unsigned int (*check_events) (struct gendisk *disk, 1588 unsigned int clearing); 1589 void (*unlock_native_capacity) (struct gendisk *); 1590 int (*getgeo)(struct gendisk *, struct hd_geometry *); 1591 int (*set_read_only)(struct block_device *bdev, bool ro); 1592 void (*free_disk)(struct gendisk *disk); 1593 /* this callback is with swap_lock and sometimes page table lock held */ 1594 void (*swap_slot_free_notify) (struct block_device *, unsigned long); 1595 int (*report_zones)(struct gendisk *, sector_t sector, 1596 unsigned int nr_zones, 1597 struct blk_report_zones_args *args); 1598 char *(*devnode)(struct gendisk *disk, umode_t *mode); 1599 /* returns the length of the identifier or a negative errno: */ 1600 int (*get_unique_id)(struct gendisk *disk, u8 id[16], 1601 enum blk_unique_id id_type); 1602 struct module *owner; 1603 const struct pr_ops *pr_ops; 1604 1605 /* 1606 * Special callback for probing GPT entry at a given sector. 1607 * Needed by Android devices, used by GPT scanner and MMC blk 1608 * driver. 1609 */ 1610 int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector); 1611 }; 1612 1613 #ifdef CONFIG_COMPAT 1614 extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t, 1615 unsigned int, unsigned long); 1616 #else 1617 #define blkdev_compat_ptr_ioctl NULL 1618 #endif 1619 1620 static inline void blk_wake_io_task(struct task_struct *waiter) 1621 { 1622 /* 1623 * If we're polling, the task itself is doing the completions. For 1624 * that case, we don't need to signal a wakeup, it's enough to just 1625 * mark us as RUNNING. 1626 */ 1627 if (waiter == current) 1628 __set_current_state(TASK_RUNNING); 1629 else 1630 wake_up_process(waiter); 1631 } 1632 1633 unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op, 1634 unsigned long start_time); 1635 void bdev_end_io_acct(struct block_device *bdev, enum req_op op, 1636 unsigned int sectors, unsigned long start_time); 1637 1638 unsigned long bio_start_io_acct(struct bio *bio); 1639 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, 1640 struct block_device *orig_bdev); 1641 1642 /** 1643 * bio_end_io_acct - end I/O accounting for bio based drivers 1644 * @bio: bio to end account for 1645 * @start_time: start time returned by bio_start_io_acct() 1646 */ 1647 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) 1648 { 1649 return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev); 1650 } 1651 1652 int bdev_validate_blocksize(struct block_device *bdev, int block_size); 1653 int set_blocksize(struct file *file, int size); 1654 1655 int lookup_bdev(const char *pathname, dev_t *dev); 1656 1657 void blkdev_show(struct seq_file *seqf, off_t offset); 1658 1659 #define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */ 1660 #define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */ 1661 #ifdef CONFIG_BLOCK 1662 #define BLKDEV_MAJOR_MAX 512 1663 #else 1664 #define BLKDEV_MAJOR_MAX 0 1665 #endif 1666 1667 struct blk_holder_ops { 1668 void (*mark_dead)(struct block_device *bdev, bool surprise) 1669 __releases(&bdev->bd_holder_lock); 1670 1671 /* 1672 * Sync the file system mounted on the block device. 1673 */ 1674 void (*sync)(struct block_device *bdev) 1675 __releases(&bdev->bd_holder_lock); 1676 1677 /* 1678 * Freeze the file system mounted on the block device. 1679 */ 1680 int (*freeze)(struct block_device *bdev) 1681 __releases(&bdev->bd_holder_lock); 1682 1683 /* 1684 * Thaw the file system mounted on the block device. 1685 */ 1686 int (*thaw)(struct block_device *bdev) 1687 __releases(&bdev->bd_holder_lock); 1688 }; 1689 1690 /* 1691 * Return the correct open flags for blkdev_get_by_* for super block flags 1692 * as stored in sb->s_flags. 1693 */ 1694 #define sb_open_mode(flags) \ 1695 (BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \ 1696 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE)) 1697 1698 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, 1699 const struct blk_holder_ops *hops); 1700 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode, 1701 void *holder, const struct blk_holder_ops *hops); 1702 int bd_prepare_to_claim(struct block_device *bdev, void *holder, 1703 const struct blk_holder_ops *hops); 1704 void bd_abort_claiming(struct block_device *bdev, void *holder); 1705 1706 struct block_device *I_BDEV(struct inode *inode); 1707 struct block_device *file_bdev(struct file *bdev_file); 1708 bool disk_live(struct gendisk *disk); 1709 unsigned int block_size(struct block_device *bdev); 1710 1711 #ifdef CONFIG_BLOCK 1712 void invalidate_bdev(struct block_device *bdev); 1713 int sync_blockdev(struct block_device *bdev); 1714 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend); 1715 int sync_blockdev_nowait(struct block_device *bdev); 1716 void sync_bdevs(bool wait); 1717 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask); 1718 void printk_all_partitions(void); 1719 int __init early_lookup_bdev(const char *pathname, dev_t *dev); 1720 #else 1721 static inline void invalidate_bdev(struct block_device *bdev) 1722 { 1723 } 1724 static inline int sync_blockdev(struct block_device *bdev) 1725 { 1726 return 0; 1727 } 1728 static inline int sync_blockdev_nowait(struct block_device *bdev) 1729 { 1730 return 0; 1731 } 1732 static inline void sync_bdevs(bool wait) 1733 { 1734 } 1735 static inline void bdev_statx(const struct path *path, struct kstat *stat, 1736 u32 request_mask) 1737 { 1738 } 1739 static inline void printk_all_partitions(void) 1740 { 1741 } 1742 static inline int early_lookup_bdev(const char *pathname, dev_t *dev) 1743 { 1744 return -EINVAL; 1745 } 1746 #endif /* CONFIG_BLOCK */ 1747 1748 int bdev_freeze(struct block_device *bdev); 1749 int bdev_thaw(struct block_device *bdev); 1750 int bdev_deny_freeze(struct block_device *bdev); 1751 void bdev_allow_freeze(struct block_device *bdev); 1752 void bdev_fput(struct file *bdev_file); 1753 void bdev_yield_claim(struct file *bdev_file); 1754 1755 struct io_comp_batch { 1756 struct rq_list req_list; 1757 bool need_ts; 1758 void (*complete)(struct io_comp_batch *); 1759 void *poll_ctx; 1760 }; 1761 1762 static inline bool blk_atomic_write_start_sect_aligned(sector_t sector, 1763 struct queue_limits *limits) 1764 { 1765 unsigned int alignment = max(limits->atomic_write_hw_unit_min, 1766 limits->atomic_write_hw_boundary); 1767 1768 return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT); 1769 } 1770 1771 static inline bool bdev_can_atomic_write(struct block_device *bdev) 1772 { 1773 struct request_queue *bd_queue = bdev->bd_queue; 1774 struct queue_limits *limits = &bd_queue->limits; 1775 1776 if (!limits->atomic_write_unit_min) 1777 return false; 1778 1779 if (bdev_is_partition(bdev)) 1780 return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect, 1781 limits); 1782 1783 return true; 1784 } 1785 1786 static inline unsigned int 1787 bdev_atomic_write_unit_min_bytes(struct block_device *bdev) 1788 { 1789 if (!bdev_can_atomic_write(bdev)) 1790 return 0; 1791 return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev)); 1792 } 1793 1794 static inline unsigned int 1795 bdev_atomic_write_unit_max_bytes(struct block_device *bdev) 1796 { 1797 if (!bdev_can_atomic_write(bdev)) 1798 return 0; 1799 return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev)); 1800 } 1801 1802 static inline int bio_split_rw_at(struct bio *bio, 1803 const struct queue_limits *lim, 1804 unsigned *segs, unsigned max_bytes) 1805 { 1806 return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment); 1807 } 1808 1809 /* 1810 * Maximum contiguous integrity buffer allocation. 1811 */ 1812 #define BLK_INTEGRITY_MAX_SIZE SZ_2M 1813 1814 /* 1815 * Maximum size of I/O that needs a block layer integrity buffer. Limited 1816 * by the number of intervals for which we can fit the integrity buffer into 1817 * the buffer size. Because the buffer is a single segment it is also limited 1818 * by the maximum segment size. 1819 */ 1820 static inline unsigned int max_integrity_io_size(struct queue_limits *lim) 1821 { 1822 u64 max_intervals; 1823 1824 max_intervals = BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size; 1825 return min_t(u64, lim->max_segment_size, 1826 max_intervals << lim->integrity.interval_exp); 1827 } 1828 1829 #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { } 1830 1831 #endif /* _LINUX_BLKDEV_H */ 1832