1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * bio-integrity.c - bio data integrity extensions 4 * 5 * Copyright (C) 2007, 2008, 2009 Oracle Corporation 6 * Written by: Martin K. Petersen <martin.petersen@oracle.com> 7 */ 8 9 #include <linux/blkdev.h> 10 #include <linux/mempool.h> 11 #include <linux/export.h> 12 #include <linux/bio.h> 13 #include <linux/workqueue.h> 14 #include <linux/slab.h> 15 #include "blk.h" 16 17 #define BIP_INLINE_VECS 4 18 19 static struct kmem_cache *bip_slab; 20 static struct workqueue_struct *kintegrityd_wq; 21 22 void blk_flush_integrity(void) 23 { 24 flush_workqueue(kintegrityd_wq); 25 } 26 27 /** 28 * bio_integrity_alloc - Allocate integrity payload and attach it to bio 29 * @bio: bio to attach integrity metadata to 30 * @gfp_mask: Memory allocation mask 31 * @nr_vecs: Number of integrity metadata scatter-gather elements 32 * 33 * Description: This function prepares a bio for attaching integrity 34 * metadata. nr_vecs specifies the maximum number of pages containing 35 * integrity metadata that can be attached. 36 */ 37 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 38 gfp_t gfp_mask, 39 unsigned int nr_vecs) 40 { 41 struct bio_integrity_payload *bip; 42 struct bio_set *bs = bio->bi_pool; 43 unsigned inline_vecs; 44 45 if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) { 46 bip = kmalloc(struct_size(bip, bip_inline_vecs, nr_vecs), gfp_mask); 47 inline_vecs = nr_vecs; 48 } else { 49 bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask); 50 inline_vecs = BIP_INLINE_VECS; 51 } 52 53 if (unlikely(!bip)) 54 return ERR_PTR(-ENOMEM); 55 56 memset(bip, 0, sizeof(*bip)); 57 58 if (nr_vecs > inline_vecs) { 59 unsigned long idx = 0; 60 61 bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx, 62 &bs->bvec_integrity_pool); 63 if (!bip->bip_vec) 64 goto err; 65 bip->bip_max_vcnt = bvec_nr_vecs(idx); 66 bip->bip_slab = idx; 67 } else { 68 bip->bip_vec = bip->bip_inline_vecs; 69 bip->bip_max_vcnt = inline_vecs; 70 } 71 72 bip->bip_bio = bio; 73 bio->bi_integrity = bip; 74 bio->bi_opf |= REQ_INTEGRITY; 75 76 return bip; 77 err: 78 mempool_free(bip, &bs->bio_integrity_pool); 79 return ERR_PTR(-ENOMEM); 80 } 81 EXPORT_SYMBOL(bio_integrity_alloc); 82 83 /** 84 * bio_integrity_free - Free bio integrity payload 85 * @bio: bio containing bip to be freed 86 * 87 * Description: Used to free the integrity portion of a bio. Usually 88 * called from bio_free(). 89 */ 90 static void bio_integrity_free(struct bio *bio) 91 { 92 struct bio_integrity_payload *bip = bio_integrity(bio); 93 struct bio_set *bs = bio->bi_pool; 94 95 if (bip->bip_flags & BIP_BLOCK_INTEGRITY) 96 kfree(page_address(bip->bip_vec->bv_page) + 97 bip->bip_vec->bv_offset); 98 99 if (bs && mempool_initialized(&bs->bio_integrity_pool)) { 100 bvec_free(&bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab); 101 102 mempool_free(bip, &bs->bio_integrity_pool); 103 } else { 104 kfree(bip); 105 } 106 107 bio->bi_integrity = NULL; 108 bio->bi_opf &= ~REQ_INTEGRITY; 109 } 110 111 /** 112 * bio_integrity_add_page - Attach integrity metadata 113 * @bio: bio to update 114 * @page: page containing integrity metadata 115 * @len: number of bytes of integrity metadata in page 116 * @offset: start offset within page 117 * 118 * Description: Attach a page containing integrity metadata to bio. 119 */ 120 int bio_integrity_add_page(struct bio *bio, struct page *page, 121 unsigned int len, unsigned int offset) 122 { 123 struct bio_integrity_payload *bip = bio_integrity(bio); 124 struct bio_vec *iv; 125 126 if (bip->bip_vcnt >= bip->bip_max_vcnt) { 127 printk(KERN_ERR "%s: bip_vec full\n", __func__); 128 return 0; 129 } 130 131 iv = bip->bip_vec + bip->bip_vcnt; 132 133 if (bip->bip_vcnt && 134 bvec_gap_to_prev(bio->bi_disk->queue, 135 &bip->bip_vec[bip->bip_vcnt - 1], offset)) 136 return 0; 137 138 iv->bv_page = page; 139 iv->bv_len = len; 140 iv->bv_offset = offset; 141 bip->bip_vcnt++; 142 143 return len; 144 } 145 EXPORT_SYMBOL(bio_integrity_add_page); 146 147 /** 148 * bio_integrity_process - Process integrity metadata for a bio 149 * @bio: bio to generate/verify integrity metadata for 150 * @proc_iter: iterator to process 151 * @proc_fn: Pointer to the relevant processing function 152 */ 153 static blk_status_t bio_integrity_process(struct bio *bio, 154 struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn) 155 { 156 struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); 157 struct blk_integrity_iter iter; 158 struct bvec_iter bviter; 159 struct bio_vec bv; 160 struct bio_integrity_payload *bip = bio_integrity(bio); 161 blk_status_t ret = BLK_STS_OK; 162 void *prot_buf = page_address(bip->bip_vec->bv_page) + 163 bip->bip_vec->bv_offset; 164 165 iter.disk_name = bio->bi_disk->disk_name; 166 iter.interval = 1 << bi->interval_exp; 167 iter.seed = proc_iter->bi_sector; 168 iter.prot_buf = prot_buf; 169 170 __bio_for_each_segment(bv, bio, bviter, *proc_iter) { 171 void *kaddr = kmap_atomic(bv.bv_page); 172 173 iter.data_buf = kaddr + bv.bv_offset; 174 iter.data_size = bv.bv_len; 175 176 ret = proc_fn(&iter); 177 if (ret) { 178 kunmap_atomic(kaddr); 179 return ret; 180 } 181 182 kunmap_atomic(kaddr); 183 } 184 return ret; 185 } 186 187 /** 188 * bio_integrity_prep - Prepare bio for integrity I/O 189 * @bio: bio to prepare 190 * 191 * Description: Checks if the bio already has an integrity payload attached. 192 * If it does, the payload has been generated by another kernel subsystem, 193 * and we just pass it through. Otherwise allocates integrity payload. 194 * The bio must have data direction, target device and start sector set priot 195 * to calling. In the WRITE case, integrity metadata will be generated using 196 * the block device's integrity function. In the READ case, the buffer 197 * will be prepared for DMA and a suitable end_io handler set up. 198 */ 199 bool bio_integrity_prep(struct bio *bio) 200 { 201 struct bio_integrity_payload *bip; 202 struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); 203 struct request_queue *q = bio->bi_disk->queue; 204 void *buf; 205 unsigned long start, end; 206 unsigned int len, nr_pages; 207 unsigned int bytes, offset, i; 208 unsigned int intervals; 209 blk_status_t status; 210 211 if (!bi) 212 return true; 213 214 if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE) 215 return true; 216 217 if (!bio_sectors(bio)) 218 return true; 219 220 /* Already protected? */ 221 if (bio_integrity(bio)) 222 return true; 223 224 if (bio_data_dir(bio) == READ) { 225 if (!bi->profile->verify_fn || 226 !(bi->flags & BLK_INTEGRITY_VERIFY)) 227 return true; 228 } else { 229 if (!bi->profile->generate_fn || 230 !(bi->flags & BLK_INTEGRITY_GENERATE)) 231 return true; 232 } 233 intervals = bio_integrity_intervals(bi, bio_sectors(bio)); 234 235 /* Allocate kernel buffer for protection data */ 236 len = intervals * bi->tuple_size; 237 buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); 238 status = BLK_STS_RESOURCE; 239 if (unlikely(buf == NULL)) { 240 printk(KERN_ERR "could not allocate integrity buffer\n"); 241 goto err_end_io; 242 } 243 244 end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 245 start = ((unsigned long) buf) >> PAGE_SHIFT; 246 nr_pages = end - start; 247 248 /* Allocate bio integrity payload and integrity vectors */ 249 bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages); 250 if (IS_ERR(bip)) { 251 printk(KERN_ERR "could not allocate data integrity bioset\n"); 252 kfree(buf); 253 status = BLK_STS_RESOURCE; 254 goto err_end_io; 255 } 256 257 bip->bip_flags |= BIP_BLOCK_INTEGRITY; 258 bip->bip_iter.bi_size = len; 259 bip_set_seed(bip, bio->bi_iter.bi_sector); 260 261 if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) 262 bip->bip_flags |= BIP_IP_CHECKSUM; 263 264 /* Map it */ 265 offset = offset_in_page(buf); 266 for (i = 0 ; i < nr_pages ; i++) { 267 int ret; 268 bytes = PAGE_SIZE - offset; 269 270 if (len <= 0) 271 break; 272 273 if (bytes > len) 274 bytes = len; 275 276 ret = bio_integrity_add_page(bio, virt_to_page(buf), 277 bytes, offset); 278 279 if (ret == 0) 280 return false; 281 282 if (ret < bytes) 283 break; 284 285 buf += bytes; 286 len -= bytes; 287 offset = 0; 288 } 289 290 /* Auto-generate integrity metadata if this is a write */ 291 if (bio_data_dir(bio) == WRITE) { 292 bio_integrity_process(bio, &bio->bi_iter, 293 bi->profile->generate_fn); 294 } else { 295 bip->bio_iter = bio->bi_iter; 296 } 297 return true; 298 299 err_end_io: 300 bio->bi_status = status; 301 bio_endio(bio); 302 return false; 303 304 } 305 EXPORT_SYMBOL(bio_integrity_prep); 306 307 /** 308 * bio_integrity_verify_fn - Integrity I/O completion worker 309 * @work: Work struct stored in bio to be verified 310 * 311 * Description: This workqueue function is called to complete a READ 312 * request. The function verifies the transferred integrity metadata 313 * and then calls the original bio end_io function. 314 */ 315 static void bio_integrity_verify_fn(struct work_struct *work) 316 { 317 struct bio_integrity_payload *bip = 318 container_of(work, struct bio_integrity_payload, bip_work); 319 struct bio *bio = bip->bip_bio; 320 struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); 321 322 /* 323 * At the moment verify is called bio's iterator was advanced 324 * during split and completion, we need to rewind iterator to 325 * it's original position. 326 */ 327 bio->bi_status = bio_integrity_process(bio, &bip->bio_iter, 328 bi->profile->verify_fn); 329 bio_integrity_free(bio); 330 bio_endio(bio); 331 } 332 333 /** 334 * __bio_integrity_endio - Integrity I/O completion function 335 * @bio: Protected bio 336 * 337 * Description: Completion for integrity I/O 338 * 339 * Normally I/O completion is done in interrupt context. However, 340 * verifying I/O integrity is a time-consuming task which must be run 341 * in process context. This function postpones completion 342 * accordingly. 343 */ 344 bool __bio_integrity_endio(struct bio *bio) 345 { 346 struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); 347 struct bio_integrity_payload *bip = bio_integrity(bio); 348 349 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && 350 (bip->bip_flags & BIP_BLOCK_INTEGRITY) && bi->profile->verify_fn) { 351 INIT_WORK(&bip->bip_work, bio_integrity_verify_fn); 352 queue_work(kintegrityd_wq, &bip->bip_work); 353 return false; 354 } 355 356 bio_integrity_free(bio); 357 return true; 358 } 359 360 /** 361 * bio_integrity_advance - Advance integrity vector 362 * @bio: bio whose integrity vector to update 363 * @bytes_done: number of data bytes that have been completed 364 * 365 * Description: This function calculates how many integrity bytes the 366 * number of completed data bytes correspond to and advances the 367 * integrity vector accordingly. 368 */ 369 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) 370 { 371 struct bio_integrity_payload *bip = bio_integrity(bio); 372 struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); 373 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); 374 375 bip->bip_iter.bi_sector += bytes_done >> 9; 376 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes); 377 } 378 379 /** 380 * bio_integrity_trim - Trim integrity vector 381 * @bio: bio whose integrity vector to update 382 * 383 * Description: Used to trim the integrity vector in a cloned bio. 384 */ 385 void bio_integrity_trim(struct bio *bio) 386 { 387 struct bio_integrity_payload *bip = bio_integrity(bio); 388 struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); 389 390 bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio)); 391 } 392 EXPORT_SYMBOL(bio_integrity_trim); 393 394 /** 395 * bio_integrity_clone - Callback for cloning bios with integrity metadata 396 * @bio: New bio 397 * @bio_src: Original bio 398 * @gfp_mask: Memory allocation mask 399 * 400 * Description: Called to allocate a bip when cloning a bio 401 */ 402 int bio_integrity_clone(struct bio *bio, struct bio *bio_src, 403 gfp_t gfp_mask) 404 { 405 struct bio_integrity_payload *bip_src = bio_integrity(bio_src); 406 struct bio_integrity_payload *bip; 407 408 BUG_ON(bip_src == NULL); 409 410 bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt); 411 if (IS_ERR(bip)) 412 return PTR_ERR(bip); 413 414 memcpy(bip->bip_vec, bip_src->bip_vec, 415 bip_src->bip_vcnt * sizeof(struct bio_vec)); 416 417 bip->bip_vcnt = bip_src->bip_vcnt; 418 bip->bip_iter = bip_src->bip_iter; 419 420 return 0; 421 } 422 EXPORT_SYMBOL(bio_integrity_clone); 423 424 int bioset_integrity_create(struct bio_set *bs, int pool_size) 425 { 426 if (mempool_initialized(&bs->bio_integrity_pool)) 427 return 0; 428 429 if (mempool_init_slab_pool(&bs->bio_integrity_pool, 430 pool_size, bip_slab)) 431 return -1; 432 433 if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) { 434 mempool_exit(&bs->bio_integrity_pool); 435 return -1; 436 } 437 438 return 0; 439 } 440 EXPORT_SYMBOL(bioset_integrity_create); 441 442 void bioset_integrity_free(struct bio_set *bs) 443 { 444 mempool_exit(&bs->bio_integrity_pool); 445 mempool_exit(&bs->bvec_integrity_pool); 446 } 447 448 void __init bio_integrity_init(void) 449 { 450 /* 451 * kintegrityd won't block much but may burn a lot of CPU cycles. 452 * Make it highpri CPU intensive wq with max concurrency of 1. 453 */ 454 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | 455 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1); 456 if (!kintegrityd_wq) 457 panic("Failed to create kintegrityd\n"); 458 459 bip_slab = kmem_cache_create("bio_integrity_payload", 460 sizeof(struct bio_integrity_payload) + 461 sizeof(struct bio_vec) * BIP_INLINE_VECS, 462 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 463 } 464