1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * bio-integrity.c - bio data integrity extensions 4 * 5 * Copyright (C) 2007, 2008, 2009 Oracle Corporation 6 * Written by: Martin K. Petersen <martin.petersen@oracle.com> 7 */ 8 9 #include <linux/blk-integrity.h> 10 #include <linux/mempool.h> 11 #include <linux/export.h> 12 #include <linux/bio.h> 13 #include <linux/workqueue.h> 14 #include <linux/slab.h> 15 #include "blk.h" 16 17 static struct kmem_cache *bip_slab; 18 static struct workqueue_struct *kintegrityd_wq; 19 20 void blk_flush_integrity(void) 21 { 22 flush_workqueue(kintegrityd_wq); 23 } 24 25 static void __bio_integrity_free(struct bio_set *bs, 26 struct bio_integrity_payload *bip) 27 { 28 if (bs && mempool_initialized(&bs->bio_integrity_pool)) { 29 if (bip->bip_vec) 30 bvec_free(&bs->bvec_integrity_pool, bip->bip_vec, 31 bip->bip_max_vcnt); 32 mempool_free(bip, &bs->bio_integrity_pool); 33 } else { 34 kfree(bip); 35 } 36 } 37 38 /** 39 * bio_integrity_alloc - Allocate integrity payload and attach it to bio 40 * @bio: bio to attach integrity metadata to 41 * @gfp_mask: Memory allocation mask 42 * @nr_vecs: Number of integrity metadata scatter-gather elements 43 * 44 * Description: This function prepares a bio for attaching integrity 45 * metadata. nr_vecs specifies the maximum number of pages containing 46 * integrity metadata that can be attached. 47 */ 48 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 49 gfp_t gfp_mask, 50 unsigned int nr_vecs) 51 { 52 struct bio_integrity_payload *bip; 53 struct bio_set *bs = bio->bi_pool; 54 unsigned inline_vecs; 55 56 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) 57 return ERR_PTR(-EOPNOTSUPP); 58 59 if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) { 60 bip = kmalloc(struct_size(bip, bip_inline_vecs, nr_vecs), gfp_mask); 61 inline_vecs = nr_vecs; 62 } else { 63 bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask); 64 inline_vecs = BIO_INLINE_VECS; 65 } 66 67 if (unlikely(!bip)) 68 return ERR_PTR(-ENOMEM); 69 70 memset(bip, 0, sizeof(*bip)); 71 72 if (nr_vecs > inline_vecs) { 73 bip->bip_max_vcnt = nr_vecs; 74 bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool, 75 &bip->bip_max_vcnt, gfp_mask); 76 if (!bip->bip_vec) 77 goto err; 78 } else { 79 bip->bip_vec = bip->bip_inline_vecs; 80 bip->bip_max_vcnt = inline_vecs; 81 } 82 83 bip->bip_bio = bio; 84 bio->bi_integrity = bip; 85 bio->bi_opf |= REQ_INTEGRITY; 86 87 return bip; 88 err: 89 __bio_integrity_free(bs, bip); 90 return ERR_PTR(-ENOMEM); 91 } 92 EXPORT_SYMBOL(bio_integrity_alloc); 93 94 /** 95 * bio_integrity_free - Free bio integrity payload 96 * @bio: bio containing bip to be freed 97 * 98 * Description: Used to free the integrity portion of a bio. Usually 99 * called from bio_free(). 100 */ 101 void bio_integrity_free(struct bio *bio) 102 { 103 struct bio_integrity_payload *bip = bio_integrity(bio); 104 struct bio_set *bs = bio->bi_pool; 105 106 if (bip->bip_flags & BIP_BLOCK_INTEGRITY) 107 kfree(bvec_virt(bip->bip_vec)); 108 109 __bio_integrity_free(bs, bip); 110 bio->bi_integrity = NULL; 111 bio->bi_opf &= ~REQ_INTEGRITY; 112 } 113 114 /** 115 * bio_integrity_add_page - Attach integrity metadata 116 * @bio: bio to update 117 * @page: page containing integrity metadata 118 * @len: number of bytes of integrity metadata in page 119 * @offset: start offset within page 120 * 121 * Description: Attach a page containing integrity metadata to bio. 122 */ 123 int bio_integrity_add_page(struct bio *bio, struct page *page, 124 unsigned int len, unsigned int offset) 125 { 126 struct bio_integrity_payload *bip = bio_integrity(bio); 127 128 if (bip->bip_vcnt >= bip->bip_max_vcnt) { 129 printk(KERN_ERR "%s: bip_vec full\n", __func__); 130 return 0; 131 } 132 133 if (bip->bip_vcnt && 134 bvec_gap_to_prev(&bdev_get_queue(bio->bi_bdev)->limits, 135 &bip->bip_vec[bip->bip_vcnt - 1], offset)) 136 return 0; 137 138 bvec_set_page(&bip->bip_vec[bip->bip_vcnt], page, len, offset); 139 bip->bip_vcnt++; 140 141 return len; 142 } 143 EXPORT_SYMBOL(bio_integrity_add_page); 144 145 /** 146 * bio_integrity_process - Process integrity metadata for a bio 147 * @bio: bio to generate/verify integrity metadata for 148 * @proc_iter: iterator to process 149 * @proc_fn: Pointer to the relevant processing function 150 */ 151 static blk_status_t bio_integrity_process(struct bio *bio, 152 struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn) 153 { 154 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 155 struct blk_integrity_iter iter; 156 struct bvec_iter bviter; 157 struct bio_vec bv; 158 struct bio_integrity_payload *bip = bio_integrity(bio); 159 blk_status_t ret = BLK_STS_OK; 160 161 iter.disk_name = bio->bi_bdev->bd_disk->disk_name; 162 iter.interval = 1 << bi->interval_exp; 163 iter.tuple_size = bi->tuple_size; 164 iter.seed = proc_iter->bi_sector; 165 iter.prot_buf = bvec_virt(bip->bip_vec); 166 167 __bio_for_each_segment(bv, bio, bviter, *proc_iter) { 168 void *kaddr = bvec_kmap_local(&bv); 169 170 iter.data_buf = kaddr; 171 iter.data_size = bv.bv_len; 172 ret = proc_fn(&iter); 173 kunmap_local(kaddr); 174 175 if (ret) 176 break; 177 178 } 179 return ret; 180 } 181 182 /** 183 * bio_integrity_prep - Prepare bio for integrity I/O 184 * @bio: bio to prepare 185 * 186 * Description: Checks if the bio already has an integrity payload attached. 187 * If it does, the payload has been generated by another kernel subsystem, 188 * and we just pass it through. Otherwise allocates integrity payload. 189 * The bio must have data direction, target device and start sector set priot 190 * to calling. In the WRITE case, integrity metadata will be generated using 191 * the block device's integrity function. In the READ case, the buffer 192 * will be prepared for DMA and a suitable end_io handler set up. 193 */ 194 bool bio_integrity_prep(struct bio *bio) 195 { 196 struct bio_integrity_payload *bip; 197 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 198 void *buf; 199 unsigned long start, end; 200 unsigned int len, nr_pages; 201 unsigned int bytes, offset, i; 202 unsigned int intervals; 203 blk_status_t status; 204 205 if (!bi) 206 return true; 207 208 if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE) 209 return true; 210 211 if (!bio_sectors(bio)) 212 return true; 213 214 /* Already protected? */ 215 if (bio_integrity(bio)) 216 return true; 217 218 if (bio_data_dir(bio) == READ) { 219 if (!bi->profile->verify_fn || 220 !(bi->flags & BLK_INTEGRITY_VERIFY)) 221 return true; 222 } else { 223 if (!bi->profile->generate_fn || 224 !(bi->flags & BLK_INTEGRITY_GENERATE)) 225 return true; 226 } 227 intervals = bio_integrity_intervals(bi, bio_sectors(bio)); 228 229 /* Allocate kernel buffer for protection data */ 230 len = intervals * bi->tuple_size; 231 buf = kmalloc(len, GFP_NOIO); 232 status = BLK_STS_RESOURCE; 233 if (unlikely(buf == NULL)) { 234 printk(KERN_ERR "could not allocate integrity buffer\n"); 235 goto err_end_io; 236 } 237 238 end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 239 start = ((unsigned long) buf) >> PAGE_SHIFT; 240 nr_pages = end - start; 241 242 /* Allocate bio integrity payload and integrity vectors */ 243 bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages); 244 if (IS_ERR(bip)) { 245 printk(KERN_ERR "could not allocate data integrity bioset\n"); 246 kfree(buf); 247 status = BLK_STS_RESOURCE; 248 goto err_end_io; 249 } 250 251 bip->bip_flags |= BIP_BLOCK_INTEGRITY; 252 bip->bip_iter.bi_size = len; 253 bip_set_seed(bip, bio->bi_iter.bi_sector); 254 255 if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) 256 bip->bip_flags |= BIP_IP_CHECKSUM; 257 258 /* Map it */ 259 offset = offset_in_page(buf); 260 for (i = 0 ; i < nr_pages ; i++) { 261 int ret; 262 bytes = PAGE_SIZE - offset; 263 264 if (len <= 0) 265 break; 266 267 if (bytes > len) 268 bytes = len; 269 270 ret = bio_integrity_add_page(bio, virt_to_page(buf), 271 bytes, offset); 272 273 if (ret == 0) { 274 printk(KERN_ERR "could not attach integrity payload\n"); 275 status = BLK_STS_RESOURCE; 276 goto err_end_io; 277 } 278 279 if (ret < bytes) 280 break; 281 282 buf += bytes; 283 len -= bytes; 284 offset = 0; 285 } 286 287 /* Auto-generate integrity metadata if this is a write */ 288 if (bio_data_dir(bio) == WRITE) { 289 bio_integrity_process(bio, &bio->bi_iter, 290 bi->profile->generate_fn); 291 } else { 292 bip->bio_iter = bio->bi_iter; 293 } 294 return true; 295 296 err_end_io: 297 bio->bi_status = status; 298 bio_endio(bio); 299 return false; 300 301 } 302 EXPORT_SYMBOL(bio_integrity_prep); 303 304 /** 305 * bio_integrity_verify_fn - Integrity I/O completion worker 306 * @work: Work struct stored in bio to be verified 307 * 308 * Description: This workqueue function is called to complete a READ 309 * request. The function verifies the transferred integrity metadata 310 * and then calls the original bio end_io function. 311 */ 312 static void bio_integrity_verify_fn(struct work_struct *work) 313 { 314 struct bio_integrity_payload *bip = 315 container_of(work, struct bio_integrity_payload, bip_work); 316 struct bio *bio = bip->bip_bio; 317 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 318 319 /* 320 * At the moment verify is called bio's iterator was advanced 321 * during split and completion, we need to rewind iterator to 322 * it's original position. 323 */ 324 bio->bi_status = bio_integrity_process(bio, &bip->bio_iter, 325 bi->profile->verify_fn); 326 bio_integrity_free(bio); 327 bio_endio(bio); 328 } 329 330 /** 331 * __bio_integrity_endio - Integrity I/O completion function 332 * @bio: Protected bio 333 * 334 * Description: Completion for integrity I/O 335 * 336 * Normally I/O completion is done in interrupt context. However, 337 * verifying I/O integrity is a time-consuming task which must be run 338 * in process context. This function postpones completion 339 * accordingly. 340 */ 341 bool __bio_integrity_endio(struct bio *bio) 342 { 343 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 344 struct bio_integrity_payload *bip = bio_integrity(bio); 345 346 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && 347 (bip->bip_flags & BIP_BLOCK_INTEGRITY) && bi->profile->verify_fn) { 348 INIT_WORK(&bip->bip_work, bio_integrity_verify_fn); 349 queue_work(kintegrityd_wq, &bip->bip_work); 350 return false; 351 } 352 353 bio_integrity_free(bio); 354 return true; 355 } 356 357 /** 358 * bio_integrity_advance - Advance integrity vector 359 * @bio: bio whose integrity vector to update 360 * @bytes_done: number of data bytes that have been completed 361 * 362 * Description: This function calculates how many integrity bytes the 363 * number of completed data bytes correspond to and advances the 364 * integrity vector accordingly. 365 */ 366 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) 367 { 368 struct bio_integrity_payload *bip = bio_integrity(bio); 369 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 370 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); 371 372 bip->bip_iter.bi_sector += bio_integrity_intervals(bi, bytes_done >> 9); 373 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes); 374 } 375 376 /** 377 * bio_integrity_trim - Trim integrity vector 378 * @bio: bio whose integrity vector to update 379 * 380 * Description: Used to trim the integrity vector in a cloned bio. 381 */ 382 void bio_integrity_trim(struct bio *bio) 383 { 384 struct bio_integrity_payload *bip = bio_integrity(bio); 385 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 386 387 bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio)); 388 } 389 EXPORT_SYMBOL(bio_integrity_trim); 390 391 /** 392 * bio_integrity_clone - Callback for cloning bios with integrity metadata 393 * @bio: New bio 394 * @bio_src: Original bio 395 * @gfp_mask: Memory allocation mask 396 * 397 * Description: Called to allocate a bip when cloning a bio 398 */ 399 int bio_integrity_clone(struct bio *bio, struct bio *bio_src, 400 gfp_t gfp_mask) 401 { 402 struct bio_integrity_payload *bip_src = bio_integrity(bio_src); 403 struct bio_integrity_payload *bip; 404 405 BUG_ON(bip_src == NULL); 406 407 bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt); 408 if (IS_ERR(bip)) 409 return PTR_ERR(bip); 410 411 memcpy(bip->bip_vec, bip_src->bip_vec, 412 bip_src->bip_vcnt * sizeof(struct bio_vec)); 413 414 bip->bip_vcnt = bip_src->bip_vcnt; 415 bip->bip_iter = bip_src->bip_iter; 416 bip->bip_flags = bip_src->bip_flags & ~BIP_BLOCK_INTEGRITY; 417 418 return 0; 419 } 420 421 int bioset_integrity_create(struct bio_set *bs, int pool_size) 422 { 423 if (mempool_initialized(&bs->bio_integrity_pool)) 424 return 0; 425 426 if (mempool_init_slab_pool(&bs->bio_integrity_pool, 427 pool_size, bip_slab)) 428 return -1; 429 430 if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) { 431 mempool_exit(&bs->bio_integrity_pool); 432 return -1; 433 } 434 435 return 0; 436 } 437 EXPORT_SYMBOL(bioset_integrity_create); 438 439 void bioset_integrity_free(struct bio_set *bs) 440 { 441 mempool_exit(&bs->bio_integrity_pool); 442 mempool_exit(&bs->bvec_integrity_pool); 443 } 444 445 void __init bio_integrity_init(void) 446 { 447 /* 448 * kintegrityd won't block much but may burn a lot of CPU cycles. 449 * Make it highpri CPU intensive wq with max concurrency of 1. 450 */ 451 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | 452 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1); 453 if (!kintegrityd_wq) 454 panic("Failed to create kintegrityd\n"); 455 456 bip_slab = kmem_cache_create("bio_integrity_payload", 457 sizeof(struct bio_integrity_payload) + 458 sizeof(struct bio_vec) * BIO_INLINE_VECS, 459 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 460 } 461