37 void blk_set_stacking_limits(struct queue_limits *lim)  in blk_set_stacking_limits()  argument
39 	memset(lim, 0, sizeof(*lim));  in blk_set_stacking_limits()
40 	lim->logical_block_size = SECTOR_SIZE;  in blk_set_stacking_limits()
41 	lim->physical_block_size = SECTOR_SIZE;  in blk_set_stacking_limits()
42 	lim->io_min = SECTOR_SIZE;  in blk_set_stacking_limits()
43 	lim->discard_granularity = SECTOR_SIZE;  in blk_set_stacking_limits()
44 	lim->dma_alignment = SECTOR_SIZE - 1;  in blk_set_stacking_limits()
45 	lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK;  in blk_set_stacking_limits()
48 	lim->max_segments = USHRT_MAX;  in blk_set_stacking_limits()
49 	lim->max_discard_segments = USHRT_MAX;  in blk_set_stacking_limits()
50 	lim->max_hw_sectors = UINT_MAX;  in blk_set_stacking_limits()
51 	lim->max_segment_size = UINT_MAX;  in blk_set_stacking_limits()
52 	lim->max_sectors = UINT_MAX;  in blk_set_stacking_limits()
53 	lim->max_dev_sectors = UINT_MAX;  in blk_set_stacking_limits()
54 	lim->max_write_zeroes_sectors = UINT_MAX;  in blk_set_stacking_limits()
55 	lim->max_hw_wzeroes_unmap_sectors = UINT_MAX;  in blk_set_stacking_limits()
56 	lim->max_user_wzeroes_unmap_sectors = UINT_MAX;  in blk_set_stacking_limits()
57 	lim->max_hw_zone_append_sectors = UINT_MAX;  in blk_set_stacking_limits()
58 	lim->max_user_discard_sectors = UINT_MAX;  in blk_set_stacking_limits()
59 	lim->atomic_write_hw_max = UINT_MAX;  in blk_set_stacking_limits()
64 		struct queue_limits *lim)  in blk_apply_bdi_limits()  argument
66 	u64 io_opt = lim->io_opt;  in blk_apply_bdi_limits()
79 	if (!io_opt && (lim->features & BLK_FEAT_ROTATIONAL))  in blk_apply_bdi_limits()
80 		io_opt = (u64)lim->max_sectors << SECTOR_SHIFT;  in blk_apply_bdi_limits()
85 	bdi->io_pages = lim->max_sectors >> PAGE_SECTORS_SHIFT;  in blk_apply_bdi_limits()
88 static int blk_validate_zoned_limits(struct queue_limits *lim)  in blk_validate_zoned_limits()  argument
90 	if (!(lim->features & BLK_FEAT_ZONED)) {  in blk_validate_zoned_limits()
91 		if (WARN_ON_ONCE(lim->max_open_zones) ||  in blk_validate_zoned_limits()
92 		    WARN_ON_ONCE(lim->max_active_zones) ||  in blk_validate_zoned_limits()
93 		    WARN_ON_ONCE(lim->zone_write_granularity) ||  in blk_validate_zoned_limits()
94 		    WARN_ON_ONCE(lim->max_zone_append_sectors))  in blk_validate_zoned_limits()
106 	if (lim->max_active_zones &&  in blk_validate_zoned_limits()
107 	    lim->max_open_zones > lim->max_active_zones)  in blk_validate_zoned_limits()
110 	if (lim->zone_write_granularity < lim->logical_block_size)  in blk_validate_zoned_limits()
111 		lim->zone_write_granularity = lim->logical_block_size;  in blk_validate_zoned_limits()
120 	lim->max_zone_append_sectors =  in blk_validate_zoned_limits()
121 		min_not_zero(lim->max_hw_zone_append_sectors,  in blk_validate_zoned_limits()
122 			min(lim->chunk_sectors, lim->max_hw_sectors));  in blk_validate_zoned_limits()
132 static inline unsigned int max_integrity_io_size(struct queue_limits *lim)  in max_integrity_io_size()  argument
134 	return min_t(unsigned int, lim->max_segment_size,  in max_integrity_io_size()
135 		(BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) <<  in max_integrity_io_size()
136 			lim->integrity.interval_exp);  in max_integrity_io_size()
139 static int blk_validate_integrity_limits(struct queue_limits *lim)  in blk_validate_integrity_limits()  argument
141 	struct blk_integrity *bi = &lim->integrity;  in blk_validate_integrity_limits()
197 		bi->interval_exp = ilog2(lim->logical_block_size);  in blk_validate_integrity_limits()
199 		   bi->interval_exp > ilog2(lim->logical_block_size)) {  in blk_validate_integrity_limits()
210 		lim->dma_alignment = max(lim->dma_alignment,  in blk_validate_integrity_limits()
219 	lim->max_sectors = min(lim->max_sectors,  in blk_validate_integrity_limits()
220 		max_integrity_io_size(lim) >> SECTOR_SHIFT);  in blk_validate_integrity_limits()
232 static unsigned int blk_queue_max_guaranteed_bio(struct queue_limits *lim)  in blk_queue_max_guaranteed_bio()  argument
234 	unsigned int max_segments = min(BIO_MAX_VECS, lim->max_segments);  in blk_queue_max_guaranteed_bio()
237 	length = min(max_segments, 2) * lim->logical_block_size;  in blk_queue_max_guaranteed_bio()
244 static void blk_atomic_writes_update_limits(struct queue_limits *lim)  in blk_atomic_writes_update_limits()  argument
246 	unsigned int unit_limit = min(lim->max_hw_sectors << SECTOR_SHIFT,  in blk_atomic_writes_update_limits()
247 					blk_queue_max_guaranteed_bio(lim));  in blk_atomic_writes_update_limits()
251 	lim->atomic_write_max_sectors =  in blk_atomic_writes_update_limits()
252 		min(lim->atomic_write_hw_max >> SECTOR_SHIFT,  in blk_atomic_writes_update_limits()
253 			lim->max_hw_sectors);  in blk_atomic_writes_update_limits()
254 	lim->atomic_write_unit_min =  in blk_atomic_writes_update_limits()
255 		min(lim->atomic_write_hw_unit_min, unit_limit);  in blk_atomic_writes_update_limits()
256 	lim->atomic_write_unit_max =  in blk_atomic_writes_update_limits()
257 		min(lim->atomic_write_hw_unit_max, unit_limit);  in blk_atomic_writes_update_limits()
258 	lim->atomic_write_boundary_sectors =  in blk_atomic_writes_update_limits()
259 		lim->atomic_write_hw_boundary >> SECTOR_SHIFT;  in blk_atomic_writes_update_limits()
283 static void blk_validate_atomic_write_limits(struct queue_limits *lim)  in blk_validate_atomic_write_limits()  argument
287 			lim->atomic_write_hw_max >> SECTOR_SHIFT;  in blk_validate_atomic_write_limits()
289 	if (!(lim->features & BLK_FEAT_ATOMIC_WRITES))  in blk_validate_atomic_write_limits()
293 	if (lim->atomic_write_hw_max == UINT_MAX)  in blk_validate_atomic_write_limits()
296 	if (!lim->atomic_write_hw_max)  in blk_validate_atomic_write_limits()
299 	if (WARN_ON_ONCE(!is_power_of_2(lim->atomic_write_hw_unit_min)))  in blk_validate_atomic_write_limits()
302 	if (WARN_ON_ONCE(!is_power_of_2(lim->atomic_write_hw_unit_max)))  in blk_validate_atomic_write_limits()
305 	if (WARN_ON_ONCE(lim->atomic_write_hw_unit_min >  in blk_validate_atomic_write_limits()
306 			 lim->atomic_write_hw_unit_max))  in blk_validate_atomic_write_limits()
309 	if (WARN_ON_ONCE(lim->atomic_write_hw_unit_max >  in blk_validate_atomic_write_limits()
310 			 lim->atomic_write_hw_max))  in blk_validate_atomic_write_limits()
313 	if (WARN_ON_ONCE(lim->chunk_sectors &&  in blk_validate_atomic_write_limits()
314 			atomic_write_hw_max_sectors > lim->chunk_sectors))  in blk_validate_atomic_write_limits()
317 	boundary_sectors = lim->atomic_write_hw_boundary >> SECTOR_SHIFT;  in blk_validate_atomic_write_limits()
320 		if (WARN_ON_ONCE(lim->atomic_write_hw_max >  in blk_validate_atomic_write_limits()
321 				 lim->atomic_write_hw_boundary))  in blk_validate_atomic_write_limits()
325 			lim->chunk_sectors, boundary_sectors)))  in blk_validate_atomic_write_limits()
339 	blk_atomic_writes_update_limits(lim);  in blk_validate_atomic_write_limits()
343 	lim->atomic_write_max_sectors = 0;  in blk_validate_atomic_write_limits()
344 	lim->atomic_write_boundary_sectors = 0;  in blk_validate_atomic_write_limits()
345 	lim->atomic_write_unit_min = 0;  in blk_validate_atomic_write_limits()
346 	lim->atomic_write_unit_max = 0;  in blk_validate_atomic_write_limits()
353 int blk_validate_limits(struct queue_limits *lim)  in blk_validate_limits()  argument
364 	if (!lim->logical_block_size)  in blk_validate_limits()
365 		lim->logical_block_size = SECTOR_SIZE;  in blk_validate_limits()
366 	else if (blk_validate_block_size(lim->logical_block_size)) {  in blk_validate_limits()
367 		pr_warn("Invalid logical block size (%d)\n", lim->logical_block_size);  in blk_validate_limits()
370 	if (lim->physical_block_size < lim->logical_block_size) {  in blk_validate_limits()
371 		lim->physical_block_size = lim->logical_block_size;  in blk_validate_limits()
372 	} else if (!is_power_of_2(lim->physical_block_size)) {  in blk_validate_limits()
373 		pr_warn("Invalid physical block size (%d)\n", lim->physical_block_size);  in blk_validate_limits()
381 	if (lim->io_min < lim->physical_block_size)  in blk_validate_limits()
382 		lim->io_min = lim->physical_block_size;  in blk_validate_limits()
389 	lim->io_opt = round_down(lim->io_opt, lim->physical_block_size);  in blk_validate_limits()
400 	if (!lim->max_hw_sectors)  in blk_validate_limits()
401 		lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS;  in blk_validate_limits()
402 	if (WARN_ON_ONCE(lim->max_hw_sectors < PAGE_SECTORS))  in blk_validate_limits()
404 	logical_block_sectors = lim->logical_block_size >> SECTOR_SHIFT;  in blk_validate_limits()
405 	if (WARN_ON_ONCE(logical_block_sectors > lim->max_hw_sectors))  in blk_validate_limits()
407 	lim->max_hw_sectors = round_down(lim->max_hw_sectors,  in blk_validate_limits()
416 	max_hw_sectors = min_not_zero(lim->max_hw_sectors,  in blk_validate_limits()
417 				lim->max_dev_sectors);  in blk_validate_limits()
418 	if (lim->max_user_sectors) {  in blk_validate_limits()
419 		if (lim->max_user_sectors < BLK_MIN_SEGMENT_SIZE / SECTOR_SIZE)  in blk_validate_limits()
421 		lim->max_sectors = min(max_hw_sectors, lim->max_user_sectors);  in blk_validate_limits()
422 	} else if (lim->io_opt > (BLK_DEF_MAX_SECTORS_CAP << SECTOR_SHIFT)) {  in blk_validate_limits()
423 		lim->max_sectors =  in blk_validate_limits()
424 			min(max_hw_sectors, lim->io_opt >> SECTOR_SHIFT);  in blk_validate_limits()
425 	} else if (lim->io_min > (BLK_DEF_MAX_SECTORS_CAP << SECTOR_SHIFT)) {  in blk_validate_limits()
426 		lim->max_sectors =  in blk_validate_limits()
427 			min(max_hw_sectors, lim->io_min >> SECTOR_SHIFT);  in blk_validate_limits()
429 		lim->max_sectors = min(max_hw_sectors, BLK_DEF_MAX_SECTORS_CAP);  in blk_validate_limits()
431 	lim->max_sectors = round_down(lim->max_sectors,  in blk_validate_limits()
438 	if (!lim->max_segments)  in blk_validate_limits()
439 		lim->max_segments = BLK_MAX_SEGMENTS;  in blk_validate_limits()
441 	if (lim->max_hw_wzeroes_unmap_sectors &&  in blk_validate_limits()
442 	    lim->max_hw_wzeroes_unmap_sectors != lim->max_write_zeroes_sectors)  in blk_validate_limits()
444 	lim->max_wzeroes_unmap_sectors = min(lim->max_hw_wzeroes_unmap_sectors,  in blk_validate_limits()
445 			lim->max_user_wzeroes_unmap_sectors);  in blk_validate_limits()
447 	lim->max_discard_sectors =  in blk_validate_limits()
448 		min(lim->max_hw_discard_sectors, lim->max_user_discard_sectors);  in blk_validate_limits()
454 	if (lim->max_discard_sectors)  in blk_validate_limits()
455 		lim->discard_granularity =  in blk_validate_limits()
456 			max(lim->discard_granularity, lim->physical_block_size);  in blk_validate_limits()
458 		lim->discard_granularity = 0;  in blk_validate_limits()
460 	if (!lim->max_discard_segments)  in blk_validate_limits()
461 		lim->max_discard_segments = 1;  in blk_validate_limits()
468 	if (!lim->seg_boundary_mask)  in blk_validate_limits()
469 		lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK;  in blk_validate_limits()
470 	if (WARN_ON_ONCE(lim->seg_boundary_mask < BLK_MIN_SEGMENT_SIZE - 1))  in blk_validate_limits()
480 	if (lim->virt_boundary_mask) {  in blk_validate_limits()
481 		if (!lim->max_segment_size)  in blk_validate_limits()
482 			lim->max_segment_size = UINT_MAX;  in blk_validate_limits()
489 		if (!lim->max_segment_size)  in blk_validate_limits()
490 			lim->max_segment_size = BLK_MAX_SEGMENT_SIZE;  in blk_validate_limits()
491 		if (WARN_ON_ONCE(lim->max_segment_size < BLK_MIN_SEGMENT_SIZE))  in blk_validate_limits()
496 	if (lim->seg_boundary_mask > lim->max_segment_size - 1)  in blk_validate_limits()
497 		seg_size = lim->max_segment_size;  in blk_validate_limits()
499 		seg_size = lim->seg_boundary_mask + 1;  in blk_validate_limits()
500 	lim->max_fast_segment_size = min_t(unsigned int, seg_size, PAGE_SIZE);  in blk_validate_limits()
508 	if (!lim->dma_alignment)  in blk_validate_limits()
509 		lim->dma_alignment = SECTOR_SIZE - 1;  in blk_validate_limits()
510 	if (WARN_ON_ONCE(lim->dma_alignment > PAGE_SIZE))  in blk_validate_limits()
513 	if (lim->alignment_offset) {  in blk_validate_limits()
514 		lim->alignment_offset &= (lim->physical_block_size - 1);  in blk_validate_limits()
515 		lim->flags &= ~BLK_FLAG_MISALIGNED;  in blk_validate_limits()
518 	if (!(lim->features & BLK_FEAT_WRITE_CACHE))  in blk_validate_limits()
519 		lim->features &= ~BLK_FEAT_FUA;  in blk_validate_limits()
521 	blk_validate_atomic_write_limits(lim);  in blk_validate_limits()
523 	err = blk_validate_integrity_limits(lim);  in blk_validate_limits()
526 	return blk_validate_zoned_limits(lim);  in blk_validate_limits()
535 int blk_set_default_limits(struct queue_limits *lim)  in blk_set_default_limits()  argument
542 	lim->max_user_discard_sectors = UINT_MAX;  in blk_set_default_limits()
543 	lim->max_user_wzeroes_unmap_sectors = UINT_MAX;  in blk_set_default_limits()
544 	return blk_validate_limits(lim);  in blk_set_default_limits()
559 		struct queue_limits *lim)  in queue_limits_commit_update()  argument
565 	error = blk_validate_limits(lim);  in queue_limits_commit_update()
570 	if (q->crypto_profile && lim->integrity.tag_size) {  in queue_limits_commit_update()
577 	q->limits = *lim;  in queue_limits_commit_update()
579 		blk_apply_bdi_limits(q->disk->bdi, lim);  in queue_limits_commit_update()
598 		struct queue_limits *lim)  in queue_limits_commit_update_frozen()  argument
604 	ret = queue_limits_commit_update(q, lim);  in queue_limits_commit_update_frozen()
622 int queue_limits_set(struct request_queue *q, struct queue_limits *lim)  in queue_limits_set()  argument
625 	return queue_limits_commit_update(q, lim);  in queue_limits_set()
629 static int queue_limit_alignment_offset(const struct queue_limits *lim,  in queue_limit_alignment_offset()  argument
632 	unsigned int granularity = max(lim->physical_block_size, lim->io_min);  in queue_limit_alignment_offset()
636 	return (granularity + lim->alignment_offset - alignment) % granularity;  in queue_limit_alignment_offset()
640 		const struct queue_limits *lim, sector_t sector)  in queue_limit_discard_alignment()  argument
644 	if (!lim->max_discard_sectors)  in queue_limit_discard_alignment()
648 	alignment = lim->discard_alignment >> SECTOR_SHIFT;  in queue_limit_discard_alignment()
649 	granularity = lim->discard_granularity >> SECTOR_SHIFT;  in queue_limit_discard_alignment()