1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * bio-integrity.c - bio data integrity extensions 4 * 5 * Copyright (C) 2007, 2008, 2009 Oracle Corporation 6 * Written by: Martin K. Petersen <martin.petersen@oracle.com> 7 */ 8 9 #include <linux/blk-integrity.h> 10 #include <linux/mempool.h> 11 #include <linux/export.h> 12 #include <linux/bio.h> 13 #include <linux/workqueue.h> 14 #include <linux/slab.h> 15 #include "blk.h" 16 17 static struct kmem_cache *bip_slab; 18 static struct workqueue_struct *kintegrityd_wq; 19 20 void blk_flush_integrity(void) 21 { 22 flush_workqueue(kintegrityd_wq); 23 } 24 25 /** 26 * bio_integrity_free - Free bio integrity payload 27 * @bio: bio containing bip to be freed 28 * 29 * Description: Free the integrity portion of a bio. 30 */ 31 void bio_integrity_free(struct bio *bio) 32 { 33 struct bio_integrity_payload *bip = bio_integrity(bio); 34 struct bio_set *bs = bio->bi_pool; 35 36 if (bs && mempool_initialized(&bs->bio_integrity_pool)) { 37 if (bip->bip_vec) 38 bvec_free(&bs->bvec_integrity_pool, bip->bip_vec, 39 bip->bip_max_vcnt); 40 mempool_free(bip, &bs->bio_integrity_pool); 41 } else { 42 kfree(bip); 43 } 44 bio->bi_integrity = NULL; 45 bio->bi_opf &= ~REQ_INTEGRITY; 46 } 47 48 /** 49 * bio_integrity_alloc - Allocate integrity payload and attach it to bio 50 * @bio: bio to attach integrity metadata to 51 * @gfp_mask: Memory allocation mask 52 * @nr_vecs: Number of integrity metadata scatter-gather elements 53 * 54 * Description: This function prepares a bio for attaching integrity 55 * metadata. nr_vecs specifies the maximum number of pages containing 56 * integrity metadata that can be attached. 57 */ 58 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 59 gfp_t gfp_mask, 60 unsigned int nr_vecs) 61 { 62 struct bio_integrity_payload *bip; 63 struct bio_set *bs = bio->bi_pool; 64 unsigned inline_vecs; 65 66 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) 67 return ERR_PTR(-EOPNOTSUPP); 68 69 if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) { 70 bip = kmalloc(struct_size(bip, bip_inline_vecs, nr_vecs), gfp_mask); 71 inline_vecs = nr_vecs; 72 } else { 73 bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask); 74 inline_vecs = BIO_INLINE_VECS; 75 } 76 77 if (unlikely(!bip)) 78 return ERR_PTR(-ENOMEM); 79 80 memset(bip, 0, sizeof(*bip)); 81 82 /* always report as many vecs as asked explicitly, not inline vecs */ 83 bip->bip_max_vcnt = nr_vecs; 84 if (nr_vecs > inline_vecs) { 85 bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool, 86 &bip->bip_max_vcnt, gfp_mask); 87 if (!bip->bip_vec) 88 goto err; 89 } else if (nr_vecs) { 90 bip->bip_vec = bip->bip_inline_vecs; 91 } 92 93 bip->bip_bio = bio; 94 bio->bi_integrity = bip; 95 bio->bi_opf |= REQ_INTEGRITY; 96 97 return bip; 98 err: 99 if (bs && mempool_initialized(&bs->bio_integrity_pool)) 100 mempool_free(bip, &bs->bio_integrity_pool); 101 else 102 kfree(bip); 103 return ERR_PTR(-ENOMEM); 104 } 105 EXPORT_SYMBOL(bio_integrity_alloc); 106 107 static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs, 108 bool dirty) 109 { 110 int i; 111 112 for (i = 0; i < nr_vecs; i++) { 113 if (dirty && !PageCompound(bv[i].bv_page)) 114 set_page_dirty_lock(bv[i].bv_page); 115 unpin_user_page(bv[i].bv_page); 116 } 117 } 118 119 static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip) 120 { 121 unsigned short nr_vecs = bip->bip_max_vcnt - 1; 122 struct bio_vec *copy = &bip->bip_vec[1]; 123 size_t bytes = bip->bip_iter.bi_size; 124 struct iov_iter iter; 125 int ret; 126 127 iov_iter_bvec(&iter, ITER_DEST, copy, nr_vecs, bytes); 128 ret = copy_to_iter(bvec_virt(bip->bip_vec), bytes, &iter); 129 WARN_ON_ONCE(ret != bytes); 130 131 bio_integrity_unpin_bvec(copy, nr_vecs, true); 132 } 133 134 /** 135 * bio_integrity_unmap_user - Unmap user integrity payload 136 * @bio: bio containing bip to be unmapped 137 * 138 * Unmap the user mapped integrity portion of a bio. 139 */ 140 void bio_integrity_unmap_user(struct bio *bio) 141 { 142 struct bio_integrity_payload *bip = bio_integrity(bio); 143 144 if (bip->bip_flags & BIP_COPY_USER) { 145 if (bio_data_dir(bio) == READ) 146 bio_integrity_uncopy_user(bip); 147 kfree(bvec_virt(bip->bip_vec)); 148 return; 149 } 150 151 bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt, 152 bio_data_dir(bio) == READ); 153 } 154 155 /** 156 * bio_integrity_add_page - Attach integrity metadata 157 * @bio: bio to update 158 * @page: page containing integrity metadata 159 * @len: number of bytes of integrity metadata in page 160 * @offset: start offset within page 161 * 162 * Description: Attach a page containing integrity metadata to bio. 163 */ 164 int bio_integrity_add_page(struct bio *bio, struct page *page, 165 unsigned int len, unsigned int offset) 166 { 167 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 168 struct bio_integrity_payload *bip = bio_integrity(bio); 169 170 if (bip->bip_vcnt > 0) { 171 struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1]; 172 bool same_page = false; 173 174 if (bvec_try_merge_hw_page(q, bv, page, len, offset, 175 &same_page)) { 176 bip->bip_iter.bi_size += len; 177 return len; 178 } 179 180 if (bip->bip_vcnt >= 181 min(bip->bip_max_vcnt, queue_max_integrity_segments(q))) 182 return 0; 183 184 /* 185 * If the queue doesn't support SG gaps and adding this segment 186 * would create a gap, disallow it. 187 */ 188 if (bvec_gap_to_prev(&q->limits, bv, offset)) 189 return 0; 190 } 191 192 bvec_set_page(&bip->bip_vec[bip->bip_vcnt], page, len, offset); 193 bip->bip_vcnt++; 194 bip->bip_iter.bi_size += len; 195 196 return len; 197 } 198 EXPORT_SYMBOL(bio_integrity_add_page); 199 200 static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec, 201 int nr_vecs, unsigned int len, 202 unsigned int direction, u32 seed) 203 { 204 bool write = direction == ITER_SOURCE; 205 struct bio_integrity_payload *bip; 206 struct iov_iter iter; 207 void *buf; 208 int ret; 209 210 buf = kmalloc(len, GFP_KERNEL); 211 if (!buf) 212 return -ENOMEM; 213 214 if (write) { 215 iov_iter_bvec(&iter, direction, bvec, nr_vecs, len); 216 if (!copy_from_iter_full(buf, len, &iter)) { 217 ret = -EFAULT; 218 goto free_buf; 219 } 220 221 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); 222 } else { 223 memset(buf, 0, len); 224 225 /* 226 * We need to preserve the original bvec and the number of vecs 227 * in it for completion handling 228 */ 229 bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs + 1); 230 } 231 232 if (IS_ERR(bip)) { 233 ret = PTR_ERR(bip); 234 goto free_buf; 235 } 236 237 if (write) 238 bio_integrity_unpin_bvec(bvec, nr_vecs, false); 239 else 240 memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec)); 241 242 ret = bio_integrity_add_page(bio, virt_to_page(buf), len, 243 offset_in_page(buf)); 244 if (ret != len) { 245 ret = -ENOMEM; 246 goto free_bip; 247 } 248 249 bip->bip_flags |= BIP_COPY_USER; 250 bip->bip_iter.bi_sector = seed; 251 bip->bip_vcnt = nr_vecs; 252 return 0; 253 free_bip: 254 bio_integrity_free(bio); 255 free_buf: 256 kfree(buf); 257 return ret; 258 } 259 260 static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec, 261 int nr_vecs, unsigned int len, u32 seed) 262 { 263 struct bio_integrity_payload *bip; 264 265 bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs); 266 if (IS_ERR(bip)) 267 return PTR_ERR(bip); 268 269 memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec)); 270 bip->bip_iter.bi_sector = seed; 271 bip->bip_iter.bi_size = len; 272 bip->bip_vcnt = nr_vecs; 273 return 0; 274 } 275 276 static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages, 277 int nr_vecs, ssize_t bytes, ssize_t offset) 278 { 279 unsigned int nr_bvecs = 0; 280 int i, j; 281 282 for (i = 0; i < nr_vecs; i = j) { 283 size_t size = min_t(size_t, bytes, PAGE_SIZE - offset); 284 struct folio *folio = page_folio(pages[i]); 285 286 bytes -= size; 287 for (j = i + 1; j < nr_vecs; j++) { 288 size_t next = min_t(size_t, PAGE_SIZE, bytes); 289 290 if (page_folio(pages[j]) != folio || 291 pages[j] != pages[j - 1] + 1) 292 break; 293 unpin_user_page(pages[j]); 294 size += next; 295 bytes -= next; 296 } 297 298 bvec_set_page(&bvec[nr_bvecs], pages[i], size, offset); 299 offset = 0; 300 nr_bvecs++; 301 } 302 303 return nr_bvecs; 304 } 305 306 int bio_integrity_map_user(struct bio *bio, void __user *ubuf, ssize_t bytes, 307 u32 seed) 308 { 309 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 310 unsigned int align = blk_lim_dma_alignment_and_pad(&q->limits); 311 struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages; 312 struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec; 313 unsigned int direction, nr_bvecs; 314 struct iov_iter iter; 315 int ret, nr_vecs; 316 size_t offset; 317 bool copy; 318 319 if (bio_integrity(bio)) 320 return -EINVAL; 321 if (bytes >> SECTOR_SHIFT > queue_max_hw_sectors(q)) 322 return -E2BIG; 323 324 if (bio_data_dir(bio) == READ) 325 direction = ITER_DEST; 326 else 327 direction = ITER_SOURCE; 328 329 iov_iter_ubuf(&iter, direction, ubuf, bytes); 330 nr_vecs = iov_iter_npages(&iter, BIO_MAX_VECS + 1); 331 if (nr_vecs > BIO_MAX_VECS) 332 return -E2BIG; 333 if (nr_vecs > UIO_FASTIOV) { 334 bvec = kcalloc(nr_vecs, sizeof(*bvec), GFP_KERNEL); 335 if (!bvec) 336 return -ENOMEM; 337 pages = NULL; 338 } 339 340 copy = !iov_iter_is_aligned(&iter, align, align); 341 ret = iov_iter_extract_pages(&iter, &pages, bytes, nr_vecs, 0, &offset); 342 if (unlikely(ret < 0)) 343 goto free_bvec; 344 345 nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset); 346 if (pages != stack_pages) 347 kvfree(pages); 348 if (nr_bvecs > queue_max_integrity_segments(q)) 349 copy = true; 350 351 if (copy) 352 ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes, 353 direction, seed); 354 else 355 ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes, seed); 356 if (ret) 357 goto release_pages; 358 if (bvec != stack_vec) 359 kfree(bvec); 360 361 return 0; 362 363 release_pages: 364 bio_integrity_unpin_bvec(bvec, nr_bvecs, false); 365 free_bvec: 366 if (bvec != stack_vec) 367 kfree(bvec); 368 return ret; 369 } 370 EXPORT_SYMBOL_GPL(bio_integrity_map_user); 371 372 /** 373 * bio_integrity_prep - Prepare bio for integrity I/O 374 * @bio: bio to prepare 375 * 376 * Description: Checks if the bio already has an integrity payload attached. 377 * If it does, the payload has been generated by another kernel subsystem, 378 * and we just pass it through. Otherwise allocates integrity payload. 379 * The bio must have data direction, target device and start sector set priot 380 * to calling. In the WRITE case, integrity metadata will be generated using 381 * the block device's integrity function. In the READ case, the buffer 382 * will be prepared for DMA and a suitable end_io handler set up. 383 */ 384 bool bio_integrity_prep(struct bio *bio) 385 { 386 struct bio_integrity_payload *bip; 387 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 388 unsigned int len; 389 void *buf; 390 gfp_t gfp = GFP_NOIO; 391 392 if (!bi) 393 return true; 394 395 if (!bio_sectors(bio)) 396 return true; 397 398 /* Already protected? */ 399 if (bio_integrity(bio)) 400 return true; 401 402 switch (bio_op(bio)) { 403 case REQ_OP_READ: 404 if (bi->flags & BLK_INTEGRITY_NOVERIFY) 405 return true; 406 break; 407 case REQ_OP_WRITE: 408 if (bi->flags & BLK_INTEGRITY_NOGENERATE) 409 return true; 410 411 /* 412 * Zero the memory allocated to not leak uninitialized kernel 413 * memory to disk for non-integrity metadata where nothing else 414 * initializes the memory. 415 */ 416 if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE) 417 gfp |= __GFP_ZERO; 418 break; 419 default: 420 return true; 421 } 422 423 /* Allocate kernel buffer for protection data */ 424 len = bio_integrity_bytes(bi, bio_sectors(bio)); 425 buf = kmalloc(len, gfp); 426 if (unlikely(buf == NULL)) { 427 goto err_end_io; 428 } 429 430 bip = bio_integrity_alloc(bio, GFP_NOIO, 1); 431 if (IS_ERR(bip)) { 432 kfree(buf); 433 goto err_end_io; 434 } 435 436 bip->bip_flags |= BIP_BLOCK_INTEGRITY; 437 bip_set_seed(bip, bio->bi_iter.bi_sector); 438 439 if (bi->csum_type == BLK_INTEGRITY_CSUM_IP) 440 bip->bip_flags |= BIP_IP_CHECKSUM; 441 442 if (bio_integrity_add_page(bio, virt_to_page(buf), len, 443 offset_in_page(buf)) < len) { 444 printk(KERN_ERR "could not attach integrity payload\n"); 445 goto err_end_io; 446 } 447 448 /* Auto-generate integrity metadata if this is a write */ 449 if (bio_data_dir(bio) == WRITE) 450 blk_integrity_generate(bio); 451 else 452 bip->bio_iter = bio->bi_iter; 453 return true; 454 455 err_end_io: 456 bio->bi_status = BLK_STS_RESOURCE; 457 bio_endio(bio); 458 return false; 459 } 460 EXPORT_SYMBOL(bio_integrity_prep); 461 462 /** 463 * bio_integrity_verify_fn - Integrity I/O completion worker 464 * @work: Work struct stored in bio to be verified 465 * 466 * Description: This workqueue function is called to complete a READ 467 * request. The function verifies the transferred integrity metadata 468 * and then calls the original bio end_io function. 469 */ 470 static void bio_integrity_verify_fn(struct work_struct *work) 471 { 472 struct bio_integrity_payload *bip = 473 container_of(work, struct bio_integrity_payload, bip_work); 474 struct bio *bio = bip->bip_bio; 475 476 blk_integrity_verify(bio); 477 478 kfree(bvec_virt(bip->bip_vec)); 479 bio_integrity_free(bio); 480 bio_endio(bio); 481 } 482 483 /** 484 * __bio_integrity_endio - Integrity I/O completion function 485 * @bio: Protected bio 486 * 487 * Description: Completion for integrity I/O 488 * 489 * Normally I/O completion is done in interrupt context. However, 490 * verifying I/O integrity is a time-consuming task which must be run 491 * in process context. This function postpones completion 492 * accordingly. 493 */ 494 bool __bio_integrity_endio(struct bio *bio) 495 { 496 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 497 struct bio_integrity_payload *bip = bio_integrity(bio); 498 499 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && bi->csum_type) { 500 INIT_WORK(&bip->bip_work, bio_integrity_verify_fn); 501 queue_work(kintegrityd_wq, &bip->bip_work); 502 return false; 503 } 504 505 kfree(bvec_virt(bip->bip_vec)); 506 bio_integrity_free(bio); 507 return true; 508 } 509 510 /** 511 * bio_integrity_advance - Advance integrity vector 512 * @bio: bio whose integrity vector to update 513 * @bytes_done: number of data bytes that have been completed 514 * 515 * Description: This function calculates how many integrity bytes the 516 * number of completed data bytes correspond to and advances the 517 * integrity vector accordingly. 518 */ 519 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) 520 { 521 struct bio_integrity_payload *bip = bio_integrity(bio); 522 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 523 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); 524 525 bip->bip_iter.bi_sector += bio_integrity_intervals(bi, bytes_done >> 9); 526 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes); 527 } 528 529 /** 530 * bio_integrity_trim - Trim integrity vector 531 * @bio: bio whose integrity vector to update 532 * 533 * Description: Used to trim the integrity vector in a cloned bio. 534 */ 535 void bio_integrity_trim(struct bio *bio) 536 { 537 struct bio_integrity_payload *bip = bio_integrity(bio); 538 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 539 540 bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio)); 541 } 542 EXPORT_SYMBOL(bio_integrity_trim); 543 544 /** 545 * bio_integrity_clone - Callback for cloning bios with integrity metadata 546 * @bio: New bio 547 * @bio_src: Original bio 548 * @gfp_mask: Memory allocation mask 549 * 550 * Description: Called to allocate a bip when cloning a bio 551 */ 552 int bio_integrity_clone(struct bio *bio, struct bio *bio_src, 553 gfp_t gfp_mask) 554 { 555 struct bio_integrity_payload *bip_src = bio_integrity(bio_src); 556 struct bio_integrity_payload *bip; 557 558 BUG_ON(bip_src == NULL); 559 560 bip = bio_integrity_alloc(bio, gfp_mask, 0); 561 if (IS_ERR(bip)) 562 return PTR_ERR(bip); 563 564 bip->bip_vec = bip_src->bip_vec; 565 bip->bip_iter = bip_src->bip_iter; 566 bip->bip_flags = bip_src->bip_flags & ~BIP_BLOCK_INTEGRITY; 567 568 return 0; 569 } 570 571 int bioset_integrity_create(struct bio_set *bs, int pool_size) 572 { 573 if (mempool_initialized(&bs->bio_integrity_pool)) 574 return 0; 575 576 if (mempool_init_slab_pool(&bs->bio_integrity_pool, 577 pool_size, bip_slab)) 578 return -1; 579 580 if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) { 581 mempool_exit(&bs->bio_integrity_pool); 582 return -1; 583 } 584 585 return 0; 586 } 587 EXPORT_SYMBOL(bioset_integrity_create); 588 589 void bioset_integrity_free(struct bio_set *bs) 590 { 591 mempool_exit(&bs->bio_integrity_pool); 592 mempool_exit(&bs->bvec_integrity_pool); 593 } 594 595 void __init bio_integrity_init(void) 596 { 597 /* 598 * kintegrityd won't block much but may burn a lot of CPU cycles. 599 * Make it highpri CPU intensive wq with max concurrency of 1. 600 */ 601 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | 602 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1); 603 if (!kintegrityd_wq) 604 panic("Failed to create kintegrityd\n"); 605 606 bip_slab = kmem_cache_create("bio_integrity_payload", 607 sizeof(struct bio_integrity_payload) + 608 sizeof(struct bio_vec) * BIO_INLINE_VECS, 609 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 610 } 611