1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef BLK_INTERNAL_H 3 #define BLK_INTERNAL_H 4 5 #include <linux/bio-integrity.h> 6 #include <linux/blk-crypto.h> 7 #include <linux/lockdep.h> 8 #include <linux/memblock.h> /* for max_pfn/max_low_pfn */ 9 #include <linux/sched/sysctl.h> 10 #include <linux/timekeeping.h> 11 #include <xen/xen.h> 12 #include "blk-crypto-internal.h" 13 14 struct elevator_type; 15 16 #define BLK_DEV_MAX_SECTORS (LLONG_MAX >> 9) 17 #define BLK_MIN_SEGMENT_SIZE 4096 18 19 /* Max future timer expiry for timeouts */ 20 #define BLK_MAX_TIMEOUT (5 * HZ) 21 22 extern struct dentry *blk_debugfs_root; 23 24 struct blk_flush_queue { 25 spinlock_t mq_flush_lock; 26 unsigned int flush_pending_idx:1; 27 unsigned int flush_running_idx:1; 28 blk_status_t rq_status; 29 unsigned long flush_pending_since; 30 struct list_head flush_queue[2]; 31 unsigned long flush_data_in_flight; 32 struct request *flush_rq; 33 }; 34 35 bool is_flush_rq(struct request *req); 36 37 struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size, 38 gfp_t flags); 39 void blk_free_flush_queue(struct blk_flush_queue *q); 40 41 bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic); 42 bool blk_queue_start_drain(struct request_queue *q); 43 bool __blk_freeze_queue_start(struct request_queue *q, 44 struct task_struct *owner); 45 int __bio_queue_enter(struct request_queue *q, struct bio *bio); 46 void submit_bio_noacct_nocheck(struct bio *bio); 47 void bio_await_chain(struct bio *bio); 48 49 static inline bool blk_try_enter_queue(struct request_queue *q, bool pm) 50 { 51 rcu_read_lock(); 52 if (!percpu_ref_tryget_live_rcu(&q->q_usage_counter)) 53 goto fail; 54 55 /* 56 * The code that increments the pm_only counter must ensure that the 57 * counter is globally visible before the queue is unfrozen. 58 */ 59 if (blk_queue_pm_only(q) && 60 (!pm || queue_rpm_status(q) == RPM_SUSPENDED)) 61 goto fail_put; 62 63 rcu_read_unlock(); 64 return true; 65 66 fail_put: 67 blk_queue_exit(q); 68 fail: 69 rcu_read_unlock(); 70 return false; 71 } 72 73 static inline int bio_queue_enter(struct bio *bio) 74 { 75 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 76 77 if (blk_try_enter_queue(q, false)) { 78 rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_); 79 rwsem_release(&q->io_lockdep_map, _RET_IP_); 80 return 0; 81 } 82 return __bio_queue_enter(q, bio); 83 } 84 85 static inline void blk_wait_io(struct completion *done) 86 { 87 /* Prevent hang_check timer from firing at us during very long I/O */ 88 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2; 89 90 if (timeout) 91 while (!wait_for_completion_io_timeout(done, timeout)) 92 ; 93 else 94 wait_for_completion_io(done); 95 } 96 97 #define BIO_INLINE_VECS 4 98 struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, 99 gfp_t gfp_mask); 100 void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs); 101 102 bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv, 103 struct page *page, unsigned len, unsigned offset, 104 bool *same_page); 105 106 static inline bool biovec_phys_mergeable(struct request_queue *q, 107 struct bio_vec *vec1, struct bio_vec *vec2) 108 { 109 unsigned long mask = queue_segment_boundary(q); 110 phys_addr_t addr1 = bvec_phys(vec1); 111 phys_addr_t addr2 = bvec_phys(vec2); 112 113 /* 114 * Merging adjacent physical pages may not work correctly under KMSAN 115 * if their metadata pages aren't adjacent. Just disable merging. 116 */ 117 if (IS_ENABLED(CONFIG_KMSAN)) 118 return false; 119 120 if (addr1 + vec1->bv_len != addr2) 121 return false; 122 if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) 123 return false; 124 if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) 125 return false; 126 return true; 127 } 128 129 static inline bool __bvec_gap_to_prev(const struct queue_limits *lim, 130 struct bio_vec *bprv, unsigned int offset) 131 { 132 return (offset & lim->virt_boundary_mask) || 133 ((bprv->bv_offset + bprv->bv_len) & lim->virt_boundary_mask); 134 } 135 136 /* 137 * Check if adding a bio_vec after bprv with offset would create a gap in 138 * the SG list. Most drivers don't care about this, but some do. 139 */ 140 static inline bool bvec_gap_to_prev(const struct queue_limits *lim, 141 struct bio_vec *bprv, unsigned int offset) 142 { 143 if (!lim->virt_boundary_mask) 144 return false; 145 return __bvec_gap_to_prev(lim, bprv, offset); 146 } 147 148 static inline bool rq_mergeable(struct request *rq) 149 { 150 if (blk_rq_is_passthrough(rq)) 151 return false; 152 153 if (req_op(rq) == REQ_OP_FLUSH) 154 return false; 155 156 if (req_op(rq) == REQ_OP_WRITE_ZEROES) 157 return false; 158 159 if (req_op(rq) == REQ_OP_ZONE_APPEND) 160 return false; 161 162 if (rq->cmd_flags & REQ_NOMERGE_FLAGS) 163 return false; 164 if (rq->rq_flags & RQF_NOMERGE_FLAGS) 165 return false; 166 167 return true; 168 } 169 170 /* 171 * There are two different ways to handle DISCARD merges: 172 * 1) If max_discard_segments > 1, the driver treats every bio as a range and 173 * send the bios to controller together. The ranges don't need to be 174 * contiguous. 175 * 2) Otherwise, the request will be normal read/write requests. The ranges 176 * need to be contiguous. 177 */ 178 static inline bool blk_discard_mergable(struct request *req) 179 { 180 if (req_op(req) == REQ_OP_DISCARD && 181 queue_max_discard_segments(req->q) > 1) 182 return true; 183 return false; 184 } 185 186 static inline unsigned int blk_rq_get_max_segments(struct request *rq) 187 { 188 if (req_op(rq) == REQ_OP_DISCARD) 189 return queue_max_discard_segments(rq->q); 190 return queue_max_segments(rq->q); 191 } 192 193 static inline unsigned int blk_queue_get_max_sectors(struct request *rq) 194 { 195 struct request_queue *q = rq->q; 196 enum req_op op = req_op(rq); 197 198 if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) 199 return min(q->limits.max_discard_sectors, 200 UINT_MAX >> SECTOR_SHIFT); 201 202 if (unlikely(op == REQ_OP_WRITE_ZEROES)) 203 return q->limits.max_write_zeroes_sectors; 204 205 if (rq->cmd_flags & REQ_ATOMIC) 206 return q->limits.atomic_write_max_sectors; 207 208 return q->limits.max_sectors; 209 } 210 211 #ifdef CONFIG_BLK_DEV_INTEGRITY 212 void blk_flush_integrity(void); 213 void bio_integrity_free(struct bio *bio); 214 215 /* 216 * Integrity payloads can either be owned by the submitter, in which case 217 * bio_uninit will free them, or owned and generated by the block layer, 218 * in which case we'll verify them here (for reads) and free them before 219 * the bio is handed back to the submitted. 220 */ 221 bool __bio_integrity_endio(struct bio *bio); 222 static inline bool bio_integrity_endio(struct bio *bio) 223 { 224 struct bio_integrity_payload *bip = bio_integrity(bio); 225 226 if (bip && (bip->bip_flags & BIP_BLOCK_INTEGRITY)) 227 return __bio_integrity_endio(bio); 228 return true; 229 } 230 231 bool blk_integrity_merge_rq(struct request_queue *, struct request *, 232 struct request *); 233 bool blk_integrity_merge_bio(struct request_queue *, struct request *, 234 struct bio *); 235 236 static inline bool integrity_req_gap_back_merge(struct request *req, 237 struct bio *next) 238 { 239 struct bio_integrity_payload *bip = bio_integrity(req->bio); 240 struct bio_integrity_payload *bip_next = bio_integrity(next); 241 242 return bvec_gap_to_prev(&req->q->limits, 243 &bip->bip_vec[bip->bip_vcnt - 1], 244 bip_next->bip_vec[0].bv_offset); 245 } 246 247 static inline bool integrity_req_gap_front_merge(struct request *req, 248 struct bio *bio) 249 { 250 struct bio_integrity_payload *bip = bio_integrity(bio); 251 struct bio_integrity_payload *bip_next = bio_integrity(req->bio); 252 253 return bvec_gap_to_prev(&req->q->limits, 254 &bip->bip_vec[bip->bip_vcnt - 1], 255 bip_next->bip_vec[0].bv_offset); 256 } 257 258 extern const struct attribute_group blk_integrity_attr_group; 259 #else /* CONFIG_BLK_DEV_INTEGRITY */ 260 static inline bool blk_integrity_merge_rq(struct request_queue *rq, 261 struct request *r1, struct request *r2) 262 { 263 return true; 264 } 265 static inline bool blk_integrity_merge_bio(struct request_queue *rq, 266 struct request *r, struct bio *b) 267 { 268 return true; 269 } 270 static inline bool integrity_req_gap_back_merge(struct request *req, 271 struct bio *next) 272 { 273 return false; 274 } 275 static inline bool integrity_req_gap_front_merge(struct request *req, 276 struct bio *bio) 277 { 278 return false; 279 } 280 281 static inline void blk_flush_integrity(void) 282 { 283 } 284 static inline bool bio_integrity_endio(struct bio *bio) 285 { 286 return true; 287 } 288 static inline void bio_integrity_free(struct bio *bio) 289 { 290 } 291 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 292 293 unsigned long blk_rq_timeout(unsigned long timeout); 294 void blk_add_timer(struct request *req); 295 296 enum bio_merge_status { 297 BIO_MERGE_OK, 298 BIO_MERGE_NONE, 299 BIO_MERGE_FAILED, 300 }; 301 302 enum bio_merge_status bio_attempt_back_merge(struct request *req, 303 struct bio *bio, unsigned int nr_segs); 304 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 305 unsigned int nr_segs); 306 bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, 307 struct bio *bio, unsigned int nr_segs); 308 309 /* 310 * Plug flush limits 311 */ 312 #define BLK_MAX_REQUEST_COUNT 32 313 #define BLK_PLUG_FLUSH_SIZE (128 * 1024) 314 315 /* 316 * Internal elevator interface 317 */ 318 #define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED) 319 320 bool blk_insert_flush(struct request *rq); 321 322 int elevator_switch(struct request_queue *q, struct elevator_type *new_e); 323 void elevator_disable(struct request_queue *q); 324 void elevator_exit(struct request_queue *q); 325 int elv_register_queue(struct request_queue *q, bool uevent); 326 void elv_unregister_queue(struct request_queue *q); 327 328 ssize_t part_size_show(struct device *dev, struct device_attribute *attr, 329 char *buf); 330 ssize_t part_stat_show(struct device *dev, struct device_attribute *attr, 331 char *buf); 332 ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr, 333 char *buf); 334 ssize_t part_fail_show(struct device *dev, struct device_attribute *attr, 335 char *buf); 336 ssize_t part_fail_store(struct device *dev, struct device_attribute *attr, 337 const char *buf, size_t count); 338 ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); 339 ssize_t part_timeout_store(struct device *, struct device_attribute *, 340 const char *, size_t); 341 342 struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, 343 unsigned *nsegs); 344 struct bio *bio_split_write_zeroes(struct bio *bio, 345 const struct queue_limits *lim, unsigned *nsegs); 346 struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, 347 unsigned *nr_segs); 348 struct bio *bio_split_zone_append(struct bio *bio, 349 const struct queue_limits *lim, unsigned *nr_segs); 350 351 /* 352 * All drivers must accept single-segments bios that are smaller than PAGE_SIZE. 353 * 354 * This is a quick and dirty check that relies on the fact that bi_io_vec[0] is 355 * always valid if a bio has data. The check might lead to occasional false 356 * positives when bios are cloned, but compared to the performance impact of 357 * cloned bios themselves the loop below doesn't matter anyway. 358 */ 359 static inline bool bio_may_need_split(struct bio *bio, 360 const struct queue_limits *lim) 361 { 362 if (lim->chunk_sectors) 363 return true; 364 if (bio->bi_vcnt != 1) 365 return true; 366 return bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > 367 lim->min_segment_size; 368 } 369 370 /** 371 * __bio_split_to_limits - split a bio to fit the queue limits 372 * @bio: bio to be split 373 * @lim: queue limits to split based on 374 * @nr_segs: returns the number of segments in the returned bio 375 * 376 * Check if @bio needs splitting based on the queue limits, and if so split off 377 * a bio fitting the limits from the beginning of @bio and return it. @bio is 378 * shortened to the remainder and re-submitted. 379 * 380 * The split bio is allocated from @q->bio_split, which is provided by the 381 * block layer. 382 */ 383 static inline struct bio *__bio_split_to_limits(struct bio *bio, 384 const struct queue_limits *lim, unsigned int *nr_segs) 385 { 386 switch (bio_op(bio)) { 387 case REQ_OP_READ: 388 case REQ_OP_WRITE: 389 if (bio_may_need_split(bio, lim)) 390 return bio_split_rw(bio, lim, nr_segs); 391 *nr_segs = 1; 392 return bio; 393 case REQ_OP_ZONE_APPEND: 394 return bio_split_zone_append(bio, lim, nr_segs); 395 case REQ_OP_DISCARD: 396 case REQ_OP_SECURE_ERASE: 397 return bio_split_discard(bio, lim, nr_segs); 398 case REQ_OP_WRITE_ZEROES: 399 return bio_split_write_zeroes(bio, lim, nr_segs); 400 default: 401 /* other operations can't be split */ 402 *nr_segs = 0; 403 return bio; 404 } 405 } 406 407 int ll_back_merge_fn(struct request *req, struct bio *bio, 408 unsigned int nr_segs); 409 bool blk_attempt_req_merge(struct request_queue *q, struct request *rq, 410 struct request *next); 411 unsigned int blk_recalc_rq_segments(struct request *rq); 412 bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 413 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio); 414 415 int blk_set_default_limits(struct queue_limits *lim); 416 void blk_apply_bdi_limits(struct backing_dev_info *bdi, 417 struct queue_limits *lim); 418 int blk_dev_init(void); 419 420 void update_io_ticks(struct block_device *part, unsigned long now, bool end); 421 unsigned int part_in_flight(struct block_device *part); 422 423 static inline void req_set_nomerge(struct request_queue *q, struct request *req) 424 { 425 req->cmd_flags |= REQ_NOMERGE; 426 if (req == q->last_merge) 427 q->last_merge = NULL; 428 } 429 430 /* 431 * Internal io_context interface 432 */ 433 struct io_cq *ioc_find_get_icq(struct request_queue *q); 434 struct io_cq *ioc_lookup_icq(struct request_queue *q); 435 #ifdef CONFIG_BLK_ICQ 436 void ioc_clear_queue(struct request_queue *q); 437 #else 438 static inline void ioc_clear_queue(struct request_queue *q) 439 { 440 } 441 #endif /* CONFIG_BLK_ICQ */ 442 443 struct bio *__blk_queue_bounce(struct bio *bio, struct request_queue *q); 444 445 static inline bool blk_queue_may_bounce(struct request_queue *q) 446 { 447 return IS_ENABLED(CONFIG_BOUNCE) && 448 (q->limits.features & BLK_FEAT_BOUNCE_HIGH) && 449 max_low_pfn >= max_pfn; 450 } 451 452 static inline struct bio *blk_queue_bounce(struct bio *bio, 453 struct request_queue *q) 454 { 455 if (unlikely(blk_queue_may_bounce(q) && bio_has_data(bio))) 456 return __blk_queue_bounce(bio, q); 457 return bio; 458 } 459 460 #ifdef CONFIG_BLK_DEV_ZONED 461 void disk_init_zone_resources(struct gendisk *disk); 462 void disk_free_zone_resources(struct gendisk *disk); 463 static inline bool bio_zone_write_plugging(struct bio *bio) 464 { 465 return bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING); 466 } 467 void blk_zone_write_plug_bio_merged(struct bio *bio); 468 void blk_zone_write_plug_init_request(struct request *rq); 469 static inline void blk_zone_update_request_bio(struct request *rq, 470 struct bio *bio) 471 { 472 /* 473 * For zone append requests, the request sector indicates the location 474 * at which the BIO data was written. Return this value to the BIO 475 * issuer through the BIO iter sector. 476 * For plugged zone writes, which include emulated zone append, we need 477 * the original BIO sector so that blk_zone_write_plug_bio_endio() can 478 * lookup the zone write plug. 479 */ 480 if (req_op(rq) == REQ_OP_ZONE_APPEND || bio_zone_write_plugging(bio)) 481 bio->bi_iter.bi_sector = rq->__sector; 482 } 483 void blk_zone_write_plug_bio_endio(struct bio *bio); 484 static inline void blk_zone_bio_endio(struct bio *bio) 485 { 486 /* 487 * For write BIOs to zoned devices, signal the completion of the BIO so 488 * that the next write BIO can be submitted by zone write plugging. 489 */ 490 if (bio_zone_write_plugging(bio)) 491 blk_zone_write_plug_bio_endio(bio); 492 } 493 494 void blk_zone_write_plug_finish_request(struct request *rq); 495 static inline void blk_zone_finish_request(struct request *rq) 496 { 497 if (rq->rq_flags & RQF_ZONE_WRITE_PLUGGING) 498 blk_zone_write_plug_finish_request(rq); 499 } 500 int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd, 501 unsigned long arg); 502 int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, 503 unsigned int cmd, unsigned long arg); 504 #else /* CONFIG_BLK_DEV_ZONED */ 505 static inline void disk_init_zone_resources(struct gendisk *disk) 506 { 507 } 508 static inline void disk_free_zone_resources(struct gendisk *disk) 509 { 510 } 511 static inline bool bio_zone_write_plugging(struct bio *bio) 512 { 513 return false; 514 } 515 static inline void blk_zone_write_plug_bio_merged(struct bio *bio) 516 { 517 } 518 static inline void blk_zone_write_plug_init_request(struct request *rq) 519 { 520 } 521 static inline void blk_zone_update_request_bio(struct request *rq, 522 struct bio *bio) 523 { 524 } 525 static inline void blk_zone_bio_endio(struct bio *bio) 526 { 527 } 528 static inline void blk_zone_finish_request(struct request *rq) 529 { 530 } 531 static inline int blkdev_report_zones_ioctl(struct block_device *bdev, 532 unsigned int cmd, unsigned long arg) 533 { 534 return -ENOTTY; 535 } 536 static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev, 537 blk_mode_t mode, unsigned int cmd, unsigned long arg) 538 { 539 return -ENOTTY; 540 } 541 #endif /* CONFIG_BLK_DEV_ZONED */ 542 543 struct block_device *bdev_alloc(struct gendisk *disk, u8 partno); 544 void bdev_add(struct block_device *bdev, dev_t dev); 545 void bdev_unhash(struct block_device *bdev); 546 void bdev_drop(struct block_device *bdev); 547 548 int blk_alloc_ext_minor(void); 549 void blk_free_ext_minor(unsigned int minor); 550 #define ADDPART_FLAG_NONE 0 551 #define ADDPART_FLAG_RAID 1 552 #define ADDPART_FLAG_WHOLEDISK 2 553 #define ADDPART_FLAG_READONLY 4 554 int bdev_add_partition(struct gendisk *disk, int partno, sector_t start, 555 sector_t length); 556 int bdev_del_partition(struct gendisk *disk, int partno); 557 int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start, 558 sector_t length); 559 void drop_partition(struct block_device *part); 560 561 void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors); 562 563 struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, 564 struct lock_class_key *lkclass); 565 566 /* 567 * Clean up a page appropriately, where the page may be pinned, may have a 568 * ref taken on it or neither. 569 */ 570 static inline void bio_release_page(struct bio *bio, struct page *page) 571 { 572 if (bio_flagged(bio, BIO_PAGE_PINNED)) 573 unpin_user_page(page); 574 } 575 576 struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id); 577 578 int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode); 579 580 int disk_alloc_events(struct gendisk *disk); 581 void disk_add_events(struct gendisk *disk); 582 void disk_del_events(struct gendisk *disk); 583 void disk_release_events(struct gendisk *disk); 584 void disk_block_events(struct gendisk *disk); 585 void disk_unblock_events(struct gendisk *disk); 586 void disk_flush_events(struct gendisk *disk, unsigned int mask); 587 extern struct device_attribute dev_attr_events; 588 extern struct device_attribute dev_attr_events_async; 589 extern struct device_attribute dev_attr_events_poll_msecs; 590 591 extern struct attribute_group blk_trace_attr_group; 592 593 blk_mode_t file_to_blk_mode(struct file *file); 594 int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode, 595 loff_t lstart, loff_t lend); 596 long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg); 597 int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); 598 long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg); 599 600 extern const struct address_space_operations def_blk_aops; 601 602 int disk_register_independent_access_ranges(struct gendisk *disk); 603 void disk_unregister_independent_access_ranges(struct gendisk *disk); 604 605 #ifdef CONFIG_FAIL_MAKE_REQUEST 606 bool should_fail_request(struct block_device *part, unsigned int bytes); 607 #else /* CONFIG_FAIL_MAKE_REQUEST */ 608 static inline bool should_fail_request(struct block_device *part, 609 unsigned int bytes) 610 { 611 return false; 612 } 613 #endif /* CONFIG_FAIL_MAKE_REQUEST */ 614 615 /* 616 * Optimized request reference counting. Ideally we'd make timeouts be more 617 * clever, as that's the only reason we need references at all... But until 618 * this happens, this is faster than using refcount_t. Also see: 619 * 620 * abc54d634334 ("io_uring: switch to atomic_t for io_kiocb reference count") 621 */ 622 #define req_ref_zero_or_close_to_overflow(req) \ 623 ((unsigned int) atomic_read(&(req->ref)) + 127u <= 127u) 624 625 static inline bool req_ref_inc_not_zero(struct request *req) 626 { 627 return atomic_inc_not_zero(&req->ref); 628 } 629 630 static inline bool req_ref_put_and_test(struct request *req) 631 { 632 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req)); 633 return atomic_dec_and_test(&req->ref); 634 } 635 636 static inline void req_ref_set(struct request *req, int value) 637 { 638 atomic_set(&req->ref, value); 639 } 640 641 static inline int req_ref_read(struct request *req) 642 { 643 return atomic_read(&req->ref); 644 } 645 646 static inline u64 blk_time_get_ns(void) 647 { 648 struct blk_plug *plug = current->plug; 649 650 if (!plug || !in_task()) 651 return ktime_get_ns(); 652 653 /* 654 * 0 could very well be a valid time, but rather than flag "this is 655 * a valid timestamp" separately, just accept that we'll do an extra 656 * ktime_get_ns() if we just happen to get 0 as the current time. 657 */ 658 if (!plug->cur_ktime) { 659 plug->cur_ktime = ktime_get_ns(); 660 current->flags |= PF_BLOCK_TS; 661 } 662 return plug->cur_ktime; 663 } 664 665 static inline ktime_t blk_time_get(void) 666 { 667 return ns_to_ktime(blk_time_get_ns()); 668 } 669 670 /* 671 * From most significant bit: 672 * 1 bit: reserved for other usage, see below 673 * 12 bits: original size of bio 674 * 51 bits: issue time of bio 675 */ 676 #define BIO_ISSUE_RES_BITS 1 677 #define BIO_ISSUE_SIZE_BITS 12 678 #define BIO_ISSUE_RES_SHIFT (64 - BIO_ISSUE_RES_BITS) 679 #define BIO_ISSUE_SIZE_SHIFT (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS) 680 #define BIO_ISSUE_TIME_MASK ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1) 681 #define BIO_ISSUE_SIZE_MASK \ 682 (((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT) 683 #define BIO_ISSUE_RES_MASK (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1)) 684 685 /* Reserved bit for blk-throtl */ 686 #define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63) 687 688 static inline u64 __bio_issue_time(u64 time) 689 { 690 return time & BIO_ISSUE_TIME_MASK; 691 } 692 693 static inline u64 bio_issue_time(struct bio_issue *issue) 694 { 695 return __bio_issue_time(issue->value); 696 } 697 698 static inline sector_t bio_issue_size(struct bio_issue *issue) 699 { 700 return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT); 701 } 702 703 static inline void bio_issue_init(struct bio_issue *issue, 704 sector_t size) 705 { 706 size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1; 707 issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | 708 (blk_time_get_ns() & BIO_ISSUE_TIME_MASK) | 709 ((u64)size << BIO_ISSUE_SIZE_SHIFT)); 710 } 711 712 void bdev_release(struct file *bdev_file); 713 int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder, 714 const struct blk_holder_ops *hops, struct file *bdev_file); 715 int bdev_permission(dev_t dev, blk_mode_t mode, void *holder); 716 717 void blk_integrity_generate(struct bio *bio); 718 void blk_integrity_verify(struct bio *bio); 719 void blk_integrity_prepare(struct request *rq); 720 void blk_integrity_complete(struct request *rq, unsigned int nr_bytes); 721 722 #ifdef CONFIG_LOCKDEP 723 static inline void blk_freeze_acquire_lock(struct request_queue *q) 724 { 725 if (!q->mq_freeze_disk_dead) 726 rwsem_acquire(&q->io_lockdep_map, 0, 1, _RET_IP_); 727 if (!q->mq_freeze_queue_dying) 728 rwsem_acquire(&q->q_lockdep_map, 0, 1, _RET_IP_); 729 } 730 731 static inline void blk_unfreeze_release_lock(struct request_queue *q) 732 { 733 if (!q->mq_freeze_queue_dying) 734 rwsem_release(&q->q_lockdep_map, _RET_IP_); 735 if (!q->mq_freeze_disk_dead) 736 rwsem_release(&q->io_lockdep_map, _RET_IP_); 737 } 738 #else 739 static inline void blk_freeze_acquire_lock(struct request_queue *q) 740 { 741 } 742 static inline void blk_unfreeze_release_lock(struct request_queue *q) 743 { 744 } 745 #endif 746 747 #endif /* BLK_INTERNAL_H */ 748