1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to generic helpers functions 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/scatterlist.h> 10 11 #include "blk.h" 12 13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector) 14 { 15 unsigned int discard_granularity = bdev_discard_granularity(bdev); 16 sector_t granularity_aligned_sector; 17 18 if (bdev_is_partition(bdev)) 19 sector += bdev->bd_start_sect; 20 21 granularity_aligned_sector = 22 round_up(sector, discard_granularity >> SECTOR_SHIFT); 23 24 /* 25 * Make sure subsequent bios start aligned to the discard granularity if 26 * it needs to be split. 27 */ 28 if (granularity_aligned_sector != sector) 29 return granularity_aligned_sector - sector; 30 31 /* 32 * Align the bio size to the discard granularity to make splitting the bio 33 * at discard granularity boundaries easier in the driver if needed. 34 */ 35 return round_down(BIO_MAX_SIZE, discard_granularity) >> SECTOR_SHIFT; 36 } 37 38 struct bio *blk_alloc_discard_bio(struct block_device *bdev, 39 sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask) 40 { 41 sector_t bio_sects = min(*nr_sects, bio_discard_limit(bdev, *sector)); 42 struct bio *bio; 43 44 if (!bio_sects) 45 return NULL; 46 47 bio = bio_alloc(bdev, 0, REQ_OP_DISCARD, gfp_mask); 48 if (!bio) 49 return NULL; 50 bio->bi_iter.bi_sector = *sector; 51 bio->bi_iter.bi_size = bio_sects << SECTOR_SHIFT; 52 *sector += bio_sects; 53 *nr_sects -= bio_sects; 54 /* 55 * We can loop for a long time in here if someone does full device 56 * discards (like mkfs). Be nice and allow us to schedule out to avoid 57 * softlocking if preempt is disabled. 58 */ 59 cond_resched(); 60 return bio; 61 } 62 63 void __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 64 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop) 65 { 66 struct bio *bio; 67 68 while ((bio = blk_alloc_discard_bio(bdev, §or, &nr_sects, 69 gfp_mask))) 70 *biop = bio_chain_and_submit(*biop, bio); 71 } 72 EXPORT_SYMBOL(__blkdev_issue_discard); 73 74 /** 75 * blkdev_issue_discard - queue a discard 76 * @bdev: blockdev to issue discard for 77 * @sector: start sector 78 * @nr_sects: number of sectors to discard 79 * @gfp_mask: memory allocation flags (for bio_alloc) 80 * 81 * Description: 82 * Issue a discard request for the sectors in question. 83 */ 84 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 85 sector_t nr_sects, gfp_t gfp_mask) 86 { 87 struct bio *bio = NULL; 88 struct blk_plug plug; 89 int ret = 0; 90 91 blk_start_plug(&plug); 92 __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio); 93 if (bio) { 94 ret = submit_bio_wait(bio); 95 if (ret == -EOPNOTSUPP) 96 ret = 0; 97 bio_put(bio); 98 } 99 blk_finish_plug(&plug); 100 101 return ret; 102 } 103 EXPORT_SYMBOL(blkdev_issue_discard); 104 105 static sector_t bio_write_zeroes_limit(struct block_device *bdev) 106 { 107 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 108 109 return min(bdev_write_zeroes_sectors(bdev), BIO_MAX_SECTORS & ~bs_mask); 110 } 111 112 /* 113 * There is no reliable way for the SCSI subsystem to determine whether a 114 * device supports a WRITE SAME operation without actually performing a write 115 * to media. As a result, write_zeroes is enabled by default and will be 116 * disabled if a zeroing operation subsequently fails. This means that this 117 * queue limit is likely to change at runtime. 118 */ 119 static void __blkdev_issue_write_zeroes(struct block_device *bdev, 120 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 121 struct bio **biop, unsigned flags, sector_t limit) 122 { 123 124 while (nr_sects) { 125 unsigned int len = min(nr_sects, limit); 126 struct bio *bio; 127 128 if ((flags & BLKDEV_ZERO_KILLABLE) && 129 fatal_signal_pending(current)) 130 break; 131 132 bio = bio_alloc(bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask); 133 bio->bi_iter.bi_sector = sector; 134 if (flags & BLKDEV_ZERO_NOUNMAP) 135 bio->bi_opf |= REQ_NOUNMAP; 136 137 bio->bi_iter.bi_size = len << SECTOR_SHIFT; 138 *biop = bio_chain_and_submit(*biop, bio); 139 140 nr_sects -= len; 141 sector += len; 142 cond_resched(); 143 } 144 } 145 146 static int blkdev_issue_write_zeroes(struct block_device *bdev, sector_t sector, 147 sector_t nr_sects, gfp_t gfp, unsigned flags) 148 { 149 sector_t limit = bio_write_zeroes_limit(bdev); 150 struct bio *bio = NULL; 151 struct blk_plug plug; 152 int ret = 0; 153 154 blk_start_plug(&plug); 155 __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp, &bio, 156 flags, limit); 157 if (bio) { 158 if ((flags & BLKDEV_ZERO_KILLABLE) && 159 fatal_signal_pending(current)) { 160 bio_await_chain(bio); 161 blk_finish_plug(&plug); 162 return -EINTR; 163 } 164 ret = submit_bio_wait(bio); 165 bio_put(bio); 166 } 167 blk_finish_plug(&plug); 168 169 /* 170 * For some devices there is no non-destructive way to verify whether 171 * WRITE ZEROES is actually supported. These will clear the capability 172 * on an I/O error, in which case we'll turn any error into 173 * "not supported" here. 174 */ 175 if (ret && !bdev_write_zeroes_sectors(bdev)) 176 return -EOPNOTSUPP; 177 return ret; 178 } 179 180 /* 181 * Convert a number of 512B sectors to a number of pages. 182 * The result is limited to a number of pages that can fit into a BIO. 183 * Also make sure that the result is always at least 1 (page) for the cases 184 * where nr_sects is lower than the number of sectors in a page. 185 */ 186 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) 187 { 188 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512); 189 190 return min(pages, (sector_t)BIO_MAX_VECS); 191 } 192 193 static void __blkdev_issue_zero_pages(struct block_device *bdev, 194 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 195 struct bio **biop, unsigned int flags) 196 { 197 struct folio *zero_folio = largest_zero_folio(); 198 199 while (nr_sects) { 200 unsigned int nr_vecs = __blkdev_sectors_to_bio_pages(nr_sects); 201 struct bio *bio; 202 203 if ((flags & BLKDEV_ZERO_KILLABLE) && 204 fatal_signal_pending(current)) 205 break; 206 207 bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask); 208 bio->bi_iter.bi_sector = sector; 209 210 do { 211 unsigned int len; 212 213 len = min_t(sector_t, folio_size(zero_folio), 214 nr_sects << SECTOR_SHIFT); 215 if (!bio_add_folio(bio, zero_folio, len, 0)) 216 break; 217 nr_sects -= len >> SECTOR_SHIFT; 218 sector += len >> SECTOR_SHIFT; 219 } while (nr_sects); 220 221 *biop = bio_chain_and_submit(*biop, bio); 222 cond_resched(); 223 } 224 } 225 226 static int blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector, 227 sector_t nr_sects, gfp_t gfp, unsigned flags) 228 { 229 struct bio *bio = NULL; 230 struct blk_plug plug; 231 int ret = 0; 232 233 if (flags & BLKDEV_ZERO_NOFALLBACK) 234 return -EOPNOTSUPP; 235 236 blk_start_plug(&plug); 237 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp, &bio, flags); 238 if (bio) { 239 if ((flags & BLKDEV_ZERO_KILLABLE) && 240 fatal_signal_pending(current)) { 241 bio_await_chain(bio); 242 blk_finish_plug(&plug); 243 return -EINTR; 244 } 245 ret = submit_bio_wait(bio); 246 bio_put(bio); 247 } 248 blk_finish_plug(&plug); 249 250 return ret; 251 } 252 253 /** 254 * __blkdev_issue_zeroout - generate number of zero filed write bios 255 * @bdev: blockdev to issue 256 * @sector: start sector 257 * @nr_sects: number of sectors to write 258 * @gfp_mask: memory allocation flags (for bio_alloc) 259 * @biop: pointer to anchor bio 260 * @flags: controls detailed behavior 261 * 262 * Description: 263 * Zero-fill a block range, either using hardware offload or by explicitly 264 * writing zeroes to the device. 265 * 266 * If a device is using logical block provisioning, the underlying space will 267 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP. 268 * 269 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return 270 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided. 271 */ 272 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 273 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 274 unsigned flags) 275 { 276 sector_t limit = bio_write_zeroes_limit(bdev); 277 278 if (bdev_read_only(bdev)) 279 return -EPERM; 280 281 if (limit) { 282 __blkdev_issue_write_zeroes(bdev, sector, nr_sects, 283 gfp_mask, biop, flags, limit); 284 } else { 285 if (flags & BLKDEV_ZERO_NOFALLBACK) 286 return -EOPNOTSUPP; 287 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, 288 biop, flags); 289 } 290 return 0; 291 } 292 EXPORT_SYMBOL(__blkdev_issue_zeroout); 293 294 /** 295 * blkdev_issue_zeroout - zero-fill a block range 296 * @bdev: blockdev to write 297 * @sector: start sector 298 * @nr_sects: number of sectors to write 299 * @gfp_mask: memory allocation flags (for bio_alloc) 300 * @flags: controls detailed behavior 301 * 302 * Description: 303 * Zero-fill a block range, either using hardware offload or by explicitly 304 * writing zeroes to the device. See __blkdev_issue_zeroout() for the 305 * valid values for %flags. 306 */ 307 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 308 sector_t nr_sects, gfp_t gfp_mask, unsigned flags) 309 { 310 int ret; 311 312 if ((sector | nr_sects) & ((bdev_logical_block_size(bdev) >> 9) - 1)) 313 return -EINVAL; 314 if (bdev_read_only(bdev)) 315 return -EPERM; 316 317 if (bdev_write_zeroes_sectors(bdev)) { 318 ret = blkdev_issue_write_zeroes(bdev, sector, nr_sects, 319 gfp_mask, flags); 320 if (ret != -EOPNOTSUPP) 321 return ret; 322 } 323 324 return blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, flags); 325 } 326 EXPORT_SYMBOL(blkdev_issue_zeroout); 327 328 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 329 sector_t nr_sects, gfp_t gfp) 330 { 331 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 332 unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev); 333 struct bio *bio = NULL; 334 struct blk_plug plug; 335 int ret = 0; 336 337 /* make sure that "len << SECTOR_SHIFT" doesn't overflow */ 338 if (max_sectors > BIO_MAX_SECTORS) 339 max_sectors = BIO_MAX_SECTORS; 340 max_sectors &= ~bs_mask; 341 342 if (max_sectors == 0) 343 return -EOPNOTSUPP; 344 if ((sector | nr_sects) & bs_mask) 345 return -EINVAL; 346 if (bdev_read_only(bdev)) 347 return -EPERM; 348 349 blk_start_plug(&plug); 350 while (nr_sects) { 351 unsigned int len = min_t(sector_t, nr_sects, max_sectors); 352 353 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp); 354 bio->bi_iter.bi_sector = sector; 355 bio->bi_iter.bi_size = len << SECTOR_SHIFT; 356 357 sector += len; 358 nr_sects -= len; 359 cond_resched(); 360 } 361 if (bio) { 362 ret = submit_bio_wait(bio); 363 bio_put(bio); 364 } 365 blk_finish_plug(&plug); 366 367 return ret; 368 } 369 EXPORT_SYMBOL(blkdev_issue_secure_erase); 370