1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to generic helpers functions 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/scatterlist.h> 10 11 #include "blk.h" 12 13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector) 14 { 15 unsigned int discard_granularity = bdev_discard_granularity(bdev); 16 sector_t granularity_aligned_sector; 17 18 if (bdev_is_partition(bdev)) 19 sector += bdev->bd_start_sect; 20 21 granularity_aligned_sector = 22 round_up(sector, discard_granularity >> SECTOR_SHIFT); 23 24 /* 25 * Make sure subsequent bios start aligned to the discard granularity if 26 * it needs to be split. 27 */ 28 if (granularity_aligned_sector != sector) 29 return granularity_aligned_sector - sector; 30 31 /* 32 * Align the bio size to the discard granularity to make splitting the bio 33 * at discard granularity boundaries easier in the driver if needed. 34 */ 35 return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT; 36 } 37 38 static void await_bio_endio(struct bio *bio) 39 { 40 complete(bio->bi_private); 41 bio_put(bio); 42 } 43 44 /* 45 * await_bio_chain - ends @bio and waits for every chained bio to complete 46 */ 47 static void await_bio_chain(struct bio *bio) 48 { 49 DECLARE_COMPLETION_ONSTACK_MAP(done, 50 bio->bi_bdev->bd_disk->lockdep_map); 51 52 bio->bi_private = &done; 53 bio->bi_end_io = await_bio_endio; 54 bio_endio(bio); 55 blk_wait_io(&done); 56 } 57 58 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 59 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop) 60 { 61 struct bio *bio = *biop; 62 sector_t bs_mask; 63 64 if (bdev_read_only(bdev)) 65 return -EPERM; 66 if (!bdev_max_discard_sectors(bdev)) 67 return -EOPNOTSUPP; 68 69 /* In case the discard granularity isn't set by buggy device driver */ 70 if (WARN_ON_ONCE(!bdev_discard_granularity(bdev))) { 71 pr_err_ratelimited("%pg: Error: discard_granularity is 0.\n", 72 bdev); 73 return -EOPNOTSUPP; 74 } 75 76 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 77 if ((sector | nr_sects) & bs_mask) 78 return -EINVAL; 79 80 if (!nr_sects) 81 return -EINVAL; 82 83 while (nr_sects) { 84 sector_t req_sects = 85 min(nr_sects, bio_discard_limit(bdev, sector)); 86 87 bio = blk_next_bio(bio, bdev, 0, REQ_OP_DISCARD, gfp_mask); 88 bio->bi_iter.bi_sector = sector; 89 bio->bi_iter.bi_size = req_sects << 9; 90 sector += req_sects; 91 nr_sects -= req_sects; 92 93 /* 94 * We can loop for a long time in here, if someone does 95 * full device discards (like mkfs). Be nice and allow 96 * us to schedule out to avoid softlocking if preempt 97 * is disabled. 98 */ 99 cond_resched(); 100 if (fatal_signal_pending(current)) { 101 await_bio_chain(bio); 102 return -EINTR; 103 } 104 } 105 106 *biop = bio; 107 return 0; 108 } 109 EXPORT_SYMBOL(__blkdev_issue_discard); 110 111 /** 112 * blkdev_issue_discard - queue a discard 113 * @bdev: blockdev to issue discard for 114 * @sector: start sector 115 * @nr_sects: number of sectors to discard 116 * @gfp_mask: memory allocation flags (for bio_alloc) 117 * 118 * Description: 119 * Issue a discard request for the sectors in question. 120 */ 121 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 122 sector_t nr_sects, gfp_t gfp_mask) 123 { 124 struct bio *bio = NULL; 125 struct blk_plug plug; 126 int ret; 127 128 blk_start_plug(&plug); 129 ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio); 130 if (!ret && bio) { 131 ret = submit_bio_wait(bio); 132 if (ret == -EOPNOTSUPP) 133 ret = 0; 134 bio_put(bio); 135 } 136 blk_finish_plug(&plug); 137 138 return ret; 139 } 140 EXPORT_SYMBOL(blkdev_issue_discard); 141 142 static int __blkdev_issue_write_zeroes(struct block_device *bdev, 143 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 144 struct bio **biop, unsigned flags) 145 { 146 struct bio *bio = *biop; 147 unsigned int max_sectors; 148 149 if (bdev_read_only(bdev)) 150 return -EPERM; 151 152 /* Ensure that max_sectors doesn't overflow bi_size */ 153 max_sectors = bdev_write_zeroes_sectors(bdev); 154 155 if (max_sectors == 0) 156 return -EOPNOTSUPP; 157 158 while (nr_sects) { 159 unsigned int len = min_t(sector_t, nr_sects, max_sectors); 160 161 bio = blk_next_bio(bio, bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask); 162 bio->bi_iter.bi_sector = sector; 163 if (flags & BLKDEV_ZERO_NOUNMAP) 164 bio->bi_opf |= REQ_NOUNMAP; 165 166 bio->bi_iter.bi_size = len << SECTOR_SHIFT; 167 nr_sects -= len; 168 sector += len; 169 cond_resched(); 170 if (fatal_signal_pending(current)) { 171 await_bio_chain(bio); 172 return -EINTR; 173 } 174 } 175 176 *biop = bio; 177 return 0; 178 } 179 180 /* 181 * Convert a number of 512B sectors to a number of pages. 182 * The result is limited to a number of pages that can fit into a BIO. 183 * Also make sure that the result is always at least 1 (page) for the cases 184 * where nr_sects is lower than the number of sectors in a page. 185 */ 186 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) 187 { 188 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512); 189 190 return min(pages, (sector_t)BIO_MAX_VECS); 191 } 192 193 static int __blkdev_issue_zero_pages(struct block_device *bdev, 194 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 195 struct bio **biop) 196 { 197 struct bio *bio = *biop; 198 int bi_size = 0; 199 unsigned int sz; 200 201 if (bdev_read_only(bdev)) 202 return -EPERM; 203 204 while (nr_sects != 0) { 205 bio = blk_next_bio(bio, bdev, __blkdev_sectors_to_bio_pages(nr_sects), 206 REQ_OP_WRITE, gfp_mask); 207 bio->bi_iter.bi_sector = sector; 208 209 while (nr_sects != 0) { 210 sz = min((sector_t) PAGE_SIZE, nr_sects << 9); 211 bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0); 212 nr_sects -= bi_size >> 9; 213 sector += bi_size >> 9; 214 if (bi_size < sz) 215 break; 216 } 217 cond_resched(); 218 if (fatal_signal_pending(current)) { 219 await_bio_chain(bio); 220 return -EINTR; 221 } 222 } 223 224 *biop = bio; 225 return 0; 226 } 227 228 /** 229 * __blkdev_issue_zeroout - generate number of zero filed write bios 230 * @bdev: blockdev to issue 231 * @sector: start sector 232 * @nr_sects: number of sectors to write 233 * @gfp_mask: memory allocation flags (for bio_alloc) 234 * @biop: pointer to anchor bio 235 * @flags: controls detailed behavior 236 * 237 * Description: 238 * Zero-fill a block range, either using hardware offload or by explicitly 239 * writing zeroes to the device. 240 * 241 * If a device is using logical block provisioning, the underlying space will 242 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP. 243 * 244 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return 245 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided. 246 */ 247 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 248 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 249 unsigned flags) 250 { 251 int ret; 252 sector_t bs_mask; 253 254 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 255 if ((sector | nr_sects) & bs_mask) 256 return -EINVAL; 257 258 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask, 259 biop, flags); 260 if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK)) 261 return ret; 262 263 return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, 264 biop); 265 } 266 EXPORT_SYMBOL(__blkdev_issue_zeroout); 267 268 /** 269 * blkdev_issue_zeroout - zero-fill a block range 270 * @bdev: blockdev to write 271 * @sector: start sector 272 * @nr_sects: number of sectors to write 273 * @gfp_mask: memory allocation flags (for bio_alloc) 274 * @flags: controls detailed behavior 275 * 276 * Description: 277 * Zero-fill a block range, either using hardware offload or by explicitly 278 * writing zeroes to the device. See __blkdev_issue_zeroout() for the 279 * valid values for %flags. 280 */ 281 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 282 sector_t nr_sects, gfp_t gfp_mask, unsigned flags) 283 { 284 int ret = 0; 285 sector_t bs_mask; 286 struct bio *bio; 287 struct blk_plug plug; 288 bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev); 289 290 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 291 if ((sector | nr_sects) & bs_mask) 292 return -EINVAL; 293 294 retry: 295 bio = NULL; 296 blk_start_plug(&plug); 297 if (try_write_zeroes) { 298 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, 299 gfp_mask, &bio, flags); 300 } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { 301 ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects, 302 gfp_mask, &bio); 303 } else { 304 /* No zeroing offload support */ 305 ret = -EOPNOTSUPP; 306 } 307 if (ret == 0 && bio) { 308 ret = submit_bio_wait(bio); 309 bio_put(bio); 310 } 311 blk_finish_plug(&plug); 312 if (ret && ret != -EINTR && try_write_zeroes) { 313 if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { 314 try_write_zeroes = false; 315 goto retry; 316 } 317 if (!bdev_write_zeroes_sectors(bdev)) { 318 /* 319 * Zeroing offload support was indicated, but the 320 * device reported ILLEGAL REQUEST (for some devices 321 * there is no non-destructive way to verify whether 322 * WRITE ZEROES is actually supported). 323 */ 324 ret = -EOPNOTSUPP; 325 } 326 } 327 328 return ret; 329 } 330 EXPORT_SYMBOL(blkdev_issue_zeroout); 331 332 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 333 sector_t nr_sects, gfp_t gfp) 334 { 335 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 336 unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev); 337 struct bio *bio = NULL; 338 struct blk_plug plug; 339 int ret = 0; 340 341 /* make sure that "len << SECTOR_SHIFT" doesn't overflow */ 342 if (max_sectors > UINT_MAX >> SECTOR_SHIFT) 343 max_sectors = UINT_MAX >> SECTOR_SHIFT; 344 max_sectors &= ~bs_mask; 345 346 if (max_sectors == 0) 347 return -EOPNOTSUPP; 348 if ((sector | nr_sects) & bs_mask) 349 return -EINVAL; 350 if (bdev_read_only(bdev)) 351 return -EPERM; 352 353 blk_start_plug(&plug); 354 while (nr_sects) { 355 unsigned int len = min_t(sector_t, nr_sects, max_sectors); 356 357 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp); 358 bio->bi_iter.bi_sector = sector; 359 bio->bi_iter.bi_size = len << SECTOR_SHIFT; 360 361 sector += len; 362 nr_sects -= len; 363 cond_resched(); 364 if (fatal_signal_pending(current)) { 365 await_bio_chain(bio); 366 ret = -EINTR; 367 bio = NULL; 368 break; 369 } 370 } 371 if (bio) { 372 ret = submit_bio_wait(bio); 373 bio_put(bio); 374 } 375 blk_finish_plug(&plug); 376 377 return ret; 378 } 379 EXPORT_SYMBOL(blkdev_issue_secure_erase); 380