1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to generic helpers functions 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/scatterlist.h> 10 11 #include "blk.h" 12 13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector) 14 { 15 unsigned int discard_granularity = bdev_discard_granularity(bdev); 16 sector_t granularity_aligned_sector; 17 18 if (bdev_is_partition(bdev)) 19 sector += bdev->bd_start_sect; 20 21 granularity_aligned_sector = 22 round_up(sector, discard_granularity >> SECTOR_SHIFT); 23 24 /* 25 * Make sure subsequent bios start aligned to the discard granularity if 26 * it needs to be split. 27 */ 28 if (granularity_aligned_sector != sector) 29 return granularity_aligned_sector - sector; 30 31 /* 32 * Align the bio size to the discard granularity to make splitting the bio 33 * at discard granularity boundaries easier in the driver if needed. 34 */ 35 return round_down(BIO_MAX_SIZE, discard_granularity) >> SECTOR_SHIFT; 36 } 37 38 struct bio *blk_alloc_discard_bio(struct block_device *bdev, 39 sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask) 40 { 41 sector_t bio_sects = min(*nr_sects, bio_discard_limit(bdev, *sector)); 42 struct bio *bio; 43 44 if (!bio_sects) 45 return NULL; 46 47 bio = bio_alloc(bdev, 0, REQ_OP_DISCARD, gfp_mask); 48 if (!bio) 49 return NULL; 50 bio->bi_iter.bi_sector = *sector; 51 bio->bi_iter.bi_size = bio_sects << SECTOR_SHIFT; 52 *sector += bio_sects; 53 *nr_sects -= bio_sects; 54 /* 55 * We can loop for a long time in here if someone does full device 56 * discards (like mkfs). Be nice and allow us to schedule out to avoid 57 * softlocking if preempt is disabled. 58 */ 59 cond_resched(); 60 return bio; 61 } 62 63 void __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 64 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop) 65 { 66 struct bio *bio; 67 68 while ((bio = blk_alloc_discard_bio(bdev, §or, &nr_sects, 69 gfp_mask))) 70 *biop = bio_chain_and_submit(*biop, bio); 71 } 72 EXPORT_SYMBOL(__blkdev_issue_discard); 73 74 /** 75 * blkdev_issue_discard - queue a discard 76 * @bdev: blockdev to issue discard for 77 * @sector: start sector 78 * @nr_sects: number of sectors to discard 79 * @gfp_mask: memory allocation flags (for bio_alloc) 80 * 81 * Description: 82 * Issue a discard request for the sectors in question. 83 */ 84 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 85 sector_t nr_sects, gfp_t gfp_mask) 86 { 87 struct bio *bio = NULL; 88 struct blk_plug plug; 89 int ret = 0; 90 91 blk_start_plug(&plug); 92 __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio); 93 if (bio) { 94 ret = submit_bio_wait(bio); 95 if (ret == -EOPNOTSUPP) 96 ret = 0; 97 bio_put(bio); 98 } 99 blk_finish_plug(&plug); 100 101 return ret; 102 } 103 EXPORT_SYMBOL(blkdev_issue_discard); 104 105 static sector_t bio_write_zeroes_limit(struct block_device *bdev) 106 { 107 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 108 109 return min(bdev_write_zeroes_sectors(bdev), BIO_MAX_SECTORS & ~bs_mask); 110 } 111 112 /* 113 * There is no reliable way for the SCSI subsystem to determine whether a 114 * device supports a WRITE SAME operation without actually performing a write 115 * to media. As a result, write_zeroes is enabled by default and will be 116 * disabled if a zeroing operation subsequently fails. This means that this 117 * queue limit is likely to change at runtime. 118 */ 119 static void __blkdev_issue_write_zeroes(struct block_device *bdev, 120 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 121 struct bio **biop, unsigned flags, sector_t limit) 122 { 123 124 while (nr_sects) { 125 unsigned int len = min(nr_sects, limit); 126 struct bio *bio; 127 128 if ((flags & BLKDEV_ZERO_KILLABLE) && 129 fatal_signal_pending(current)) 130 break; 131 132 bio = bio_alloc(bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask); 133 bio->bi_iter.bi_sector = sector; 134 if (flags & BLKDEV_ZERO_NOUNMAP) 135 bio->bi_opf |= REQ_NOUNMAP; 136 137 bio->bi_iter.bi_size = len << SECTOR_SHIFT; 138 *biop = bio_chain_and_submit(*biop, bio); 139 140 nr_sects -= len; 141 sector += len; 142 cond_resched(); 143 } 144 } 145 146 static int blkdev_issue_write_zeroes(struct block_device *bdev, sector_t sector, 147 sector_t nr_sects, gfp_t gfp, unsigned flags) 148 { 149 sector_t limit = bio_write_zeroes_limit(bdev); 150 struct bio *bio = NULL; 151 struct blk_plug plug; 152 int ret = 0; 153 154 blk_start_plug(&plug); 155 __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp, &bio, 156 flags, limit); 157 if (bio) { 158 ret = bio_submit_or_kill(bio, flags); 159 bio_put(bio); 160 } 161 blk_finish_plug(&plug); 162 163 /* 164 * For some devices there is no non-destructive way to verify whether 165 * WRITE ZEROES is actually supported. These will clear the capability 166 * on an I/O error, in which case we'll turn any error into 167 * "not supported" here. 168 */ 169 if (ret && !bdev_write_zeroes_sectors(bdev)) 170 return -EOPNOTSUPP; 171 return ret; 172 } 173 174 /* 175 * Convert a number of 512B sectors to a number of pages. 176 * The result is limited to a number of pages that can fit into a BIO. 177 * Also make sure that the result is always at least 1 (page) for the cases 178 * where nr_sects is lower than the number of sectors in a page. 179 */ 180 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) 181 { 182 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512); 183 184 return min(pages, (sector_t)BIO_MAX_VECS); 185 } 186 187 static void __blkdev_issue_zero_pages(struct block_device *bdev, 188 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 189 struct bio **biop, unsigned int flags) 190 { 191 struct folio *zero_folio = largest_zero_folio(); 192 193 while (nr_sects) { 194 unsigned int nr_vecs = __blkdev_sectors_to_bio_pages(nr_sects); 195 struct bio *bio; 196 197 if ((flags & BLKDEV_ZERO_KILLABLE) && 198 fatal_signal_pending(current)) 199 break; 200 201 bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask); 202 bio->bi_iter.bi_sector = sector; 203 204 do { 205 unsigned int len; 206 207 len = min_t(sector_t, folio_size(zero_folio), 208 nr_sects << SECTOR_SHIFT); 209 if (!bio_add_folio(bio, zero_folio, len, 0)) 210 break; 211 nr_sects -= len >> SECTOR_SHIFT; 212 sector += len >> SECTOR_SHIFT; 213 } while (nr_sects); 214 215 *biop = bio_chain_and_submit(*biop, bio); 216 cond_resched(); 217 } 218 } 219 220 static int blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector, 221 sector_t nr_sects, gfp_t gfp, unsigned flags) 222 { 223 struct bio *bio = NULL; 224 struct blk_plug plug; 225 int ret = 0; 226 227 if (flags & BLKDEV_ZERO_NOFALLBACK) 228 return -EOPNOTSUPP; 229 230 blk_start_plug(&plug); 231 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp, &bio, flags); 232 if (bio) { 233 ret = bio_submit_or_kill(bio, flags); 234 bio_put(bio); 235 } 236 blk_finish_plug(&plug); 237 238 return ret; 239 } 240 241 /** 242 * __blkdev_issue_zeroout - generate number of zero filed write bios 243 * @bdev: blockdev to issue 244 * @sector: start sector 245 * @nr_sects: number of sectors to write 246 * @gfp_mask: memory allocation flags (for bio_alloc) 247 * @biop: pointer to anchor bio 248 * @flags: controls detailed behavior 249 * 250 * Description: 251 * Zero-fill a block range, either using hardware offload or by explicitly 252 * writing zeroes to the device. 253 * 254 * If a device is using logical block provisioning, the underlying space will 255 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP. 256 * 257 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return 258 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided. 259 */ 260 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 261 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 262 unsigned flags) 263 { 264 sector_t limit = bio_write_zeroes_limit(bdev); 265 266 if (bdev_read_only(bdev)) 267 return -EPERM; 268 269 if (limit) { 270 __blkdev_issue_write_zeroes(bdev, sector, nr_sects, 271 gfp_mask, biop, flags, limit); 272 } else { 273 if (flags & BLKDEV_ZERO_NOFALLBACK) 274 return -EOPNOTSUPP; 275 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, 276 biop, flags); 277 } 278 return 0; 279 } 280 EXPORT_SYMBOL(__blkdev_issue_zeroout); 281 282 /** 283 * blkdev_issue_zeroout - zero-fill a block range 284 * @bdev: blockdev to write 285 * @sector: start sector 286 * @nr_sects: number of sectors to write 287 * @gfp_mask: memory allocation flags (for bio_alloc) 288 * @flags: controls detailed behavior 289 * 290 * Description: 291 * Zero-fill a block range, either using hardware offload or by explicitly 292 * writing zeroes to the device. See __blkdev_issue_zeroout() for the 293 * valid values for %flags. 294 */ 295 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 296 sector_t nr_sects, gfp_t gfp_mask, unsigned flags) 297 { 298 int ret; 299 300 if ((sector | nr_sects) & ((bdev_logical_block_size(bdev) >> 9) - 1)) 301 return -EINVAL; 302 if (bdev_read_only(bdev)) 303 return -EPERM; 304 305 if (bdev_write_zeroes_sectors(bdev)) { 306 ret = blkdev_issue_write_zeroes(bdev, sector, nr_sects, 307 gfp_mask, flags); 308 if (ret != -EOPNOTSUPP) 309 return ret; 310 } 311 312 return blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, flags); 313 } 314 EXPORT_SYMBOL(blkdev_issue_zeroout); 315 316 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 317 sector_t nr_sects, gfp_t gfp) 318 { 319 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 320 unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev); 321 struct bio *bio = NULL; 322 struct blk_plug plug; 323 int ret = 0; 324 325 /* make sure that "len << SECTOR_SHIFT" doesn't overflow */ 326 if (max_sectors > BIO_MAX_SECTORS) 327 max_sectors = BIO_MAX_SECTORS; 328 max_sectors &= ~bs_mask; 329 330 if (max_sectors == 0) 331 return -EOPNOTSUPP; 332 if ((sector | nr_sects) & bs_mask) 333 return -EINVAL; 334 if (bdev_read_only(bdev)) 335 return -EPERM; 336 337 blk_start_plug(&plug); 338 while (nr_sects) { 339 unsigned int len = min_t(sector_t, nr_sects, max_sectors); 340 341 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp); 342 bio->bi_iter.bi_sector = sector; 343 bio->bi_iter.bi_size = len << SECTOR_SHIFT; 344 345 sector += len; 346 nr_sects -= len; 347 cond_resched(); 348 } 349 if (bio) { 350 ret = submit_bio_wait(bio); 351 bio_put(bio); 352 } 353 blk_finish_plug(&plug); 354 355 return ret; 356 } 357 EXPORT_SYMBOL(blkdev_issue_secure_erase); 358