1 /* 2 * bio-integrity.c - bio data integrity extensions 3 * 4 * Copyright (C) 2007, 2008, 2009 Oracle Corporation 5 * Written by: Martin K. Petersen <martin.petersen@oracle.com> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License version 9 * 2 as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, but 12 * WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; see the file COPYING. If not, write to 18 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, 19 * USA. 20 * 21 */ 22 23 #include <linux/blkdev.h> 24 #include <linux/mempool.h> 25 #include <linux/export.h> 26 #include <linux/bio.h> 27 #include <linux/workqueue.h> 28 #include <linux/slab.h> 29 30 #define BIP_INLINE_VECS 4 31 32 static struct kmem_cache *bip_slab; 33 static struct workqueue_struct *kintegrityd_wq; 34 35 /** 36 * bio_integrity_alloc - Allocate integrity payload and attach it to bio 37 * @bio: bio to attach integrity metadata to 38 * @gfp_mask: Memory allocation mask 39 * @nr_vecs: Number of integrity metadata scatter-gather elements 40 * 41 * Description: This function prepares a bio for attaching integrity 42 * metadata. nr_vecs specifies the maximum number of pages containing 43 * integrity metadata that can be attached. 44 */ 45 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 46 gfp_t gfp_mask, 47 unsigned int nr_vecs) 48 { 49 struct bio_integrity_payload *bip; 50 struct bio_set *bs = bio->bi_pool; 51 unsigned long idx = BIO_POOL_NONE; 52 unsigned inline_vecs; 53 54 if (!bs) { 55 bip = kmalloc(sizeof(struct bio_integrity_payload) + 56 sizeof(struct bio_vec) * nr_vecs, gfp_mask); 57 inline_vecs = nr_vecs; 58 } else { 59 bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask); 60 inline_vecs = BIP_INLINE_VECS; 61 } 62 63 if (unlikely(!bip)) 64 return NULL; 65 66 memset(bip, 0, sizeof(*bip)); 67 68 if (nr_vecs > inline_vecs) { 69 bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx, 70 bs->bvec_integrity_pool); 71 if (!bip->bip_vec) 72 goto err; 73 bip->bip_max_vcnt = bvec_nr_vecs(idx); 74 } else { 75 bip->bip_vec = bip->bip_inline_vecs; 76 bip->bip_max_vcnt = inline_vecs; 77 } 78 79 bip->bip_slab = idx; 80 bip->bip_bio = bio; 81 bio->bi_integrity = bip; 82 bio->bi_rw |= REQ_INTEGRITY; 83 84 return bip; 85 err: 86 mempool_free(bip, bs->bio_integrity_pool); 87 return NULL; 88 } 89 EXPORT_SYMBOL(bio_integrity_alloc); 90 91 /** 92 * bio_integrity_free - Free bio integrity payload 93 * @bio: bio containing bip to be freed 94 * 95 * Description: Used to free the integrity portion of a bio. Usually 96 * called from bio_free(). 97 */ 98 void bio_integrity_free(struct bio *bio) 99 { 100 struct bio_integrity_payload *bip = bio_integrity(bio); 101 struct bio_set *bs = bio->bi_pool; 102 103 if (bip->bip_flags & BIP_BLOCK_INTEGRITY) 104 kfree(page_address(bip->bip_vec->bv_page) + 105 bip->bip_vec->bv_offset); 106 107 if (bs) { 108 if (bip->bip_slab != BIO_POOL_NONE) 109 bvec_free(bs->bvec_integrity_pool, bip->bip_vec, 110 bip->bip_slab); 111 112 mempool_free(bip, bs->bio_integrity_pool); 113 } else { 114 kfree(bip); 115 } 116 117 bio->bi_integrity = NULL; 118 } 119 EXPORT_SYMBOL(bio_integrity_free); 120 121 /** 122 * bio_integrity_add_page - Attach integrity metadata 123 * @bio: bio to update 124 * @page: page containing integrity metadata 125 * @len: number of bytes of integrity metadata in page 126 * @offset: start offset within page 127 * 128 * Description: Attach a page containing integrity metadata to bio. 129 */ 130 int bio_integrity_add_page(struct bio *bio, struct page *page, 131 unsigned int len, unsigned int offset) 132 { 133 struct bio_integrity_payload *bip = bio_integrity(bio); 134 struct bio_vec *iv; 135 136 if (bip->bip_vcnt >= bip->bip_max_vcnt) { 137 printk(KERN_ERR "%s: bip_vec full\n", __func__); 138 return 0; 139 } 140 141 iv = bip->bip_vec + bip->bip_vcnt; 142 143 iv->bv_page = page; 144 iv->bv_len = len; 145 iv->bv_offset = offset; 146 bip->bip_vcnt++; 147 148 return len; 149 } 150 EXPORT_SYMBOL(bio_integrity_add_page); 151 152 /** 153 * bio_integrity_enabled - Check whether integrity can be passed 154 * @bio: bio to check 155 * 156 * Description: Determines whether bio_integrity_prep() can be called 157 * on this bio or not. bio data direction and target device must be 158 * set prior to calling. The functions honors the write_generate and 159 * read_verify flags in sysfs. 160 */ 161 bool bio_integrity_enabled(struct bio *bio) 162 { 163 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 164 165 if (!bio_is_rw(bio)) 166 return false; 167 168 /* Already protected? */ 169 if (bio_integrity(bio)) 170 return false; 171 172 if (bi == NULL) 173 return false; 174 175 if (bio_data_dir(bio) == READ && bi->verify_fn != NULL && 176 (bi->flags & BLK_INTEGRITY_VERIFY)) 177 return true; 178 179 if (bio_data_dir(bio) == WRITE && bi->generate_fn != NULL && 180 (bi->flags & BLK_INTEGRITY_GENERATE)) 181 return true; 182 183 return false; 184 } 185 EXPORT_SYMBOL(bio_integrity_enabled); 186 187 /** 188 * bio_integrity_intervals - Return number of integrity intervals for a bio 189 * @bi: blk_integrity profile for device 190 * @sectors: Size of the bio in 512-byte sectors 191 * 192 * Description: The block layer calculates everything in 512 byte 193 * sectors but integrity metadata is done in terms of the data integrity 194 * interval size of the storage device. Convert the block layer sectors 195 * to the appropriate number of integrity intervals. 196 */ 197 static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, 198 unsigned int sectors) 199 { 200 return sectors >> (ilog2(bi->interval) - 9); 201 } 202 203 static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, 204 unsigned int sectors) 205 { 206 return bio_integrity_intervals(bi, sectors) * bi->tuple_size; 207 } 208 209 /** 210 * bio_integrity_process - Process integrity metadata for a bio 211 * @bio: bio to generate/verify integrity metadata for 212 * @proc_fn: Pointer to the relevant processing function 213 */ 214 static int bio_integrity_process(struct bio *bio, 215 integrity_processing_fn *proc_fn) 216 { 217 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 218 struct blk_integrity_iter iter; 219 struct bio_vec *bv; 220 struct bio_integrity_payload *bip = bio_integrity(bio); 221 unsigned int i, ret = 0; 222 void *prot_buf = page_address(bip->bip_vec->bv_page) + 223 bip->bip_vec->bv_offset; 224 225 iter.disk_name = bio->bi_bdev->bd_disk->disk_name; 226 iter.interval = bi->interval; 227 iter.seed = bip_get_seed(bip); 228 iter.prot_buf = prot_buf; 229 230 bio_for_each_segment_all(bv, bio, i) { 231 void *kaddr = kmap_atomic(bv->bv_page); 232 233 iter.data_buf = kaddr + bv->bv_offset; 234 iter.data_size = bv->bv_len; 235 236 ret = proc_fn(&iter); 237 if (ret) { 238 kunmap_atomic(kaddr); 239 return ret; 240 } 241 242 kunmap_atomic(kaddr); 243 } 244 return ret; 245 } 246 247 /** 248 * bio_integrity_prep - Prepare bio for integrity I/O 249 * @bio: bio to prepare 250 * 251 * Description: Allocates a buffer for integrity metadata, maps the 252 * pages and attaches them to a bio. The bio must have data 253 * direction, target device and start sector set priot to calling. In 254 * the WRITE case, integrity metadata will be generated using the 255 * block device's integrity function. In the READ case, the buffer 256 * will be prepared for DMA and a suitable end_io handler set up. 257 */ 258 int bio_integrity_prep(struct bio *bio) 259 { 260 struct bio_integrity_payload *bip; 261 struct blk_integrity *bi; 262 struct request_queue *q; 263 void *buf; 264 unsigned long start, end; 265 unsigned int len, nr_pages; 266 unsigned int bytes, offset, i; 267 unsigned int intervals; 268 269 bi = bdev_get_integrity(bio->bi_bdev); 270 q = bdev_get_queue(bio->bi_bdev); 271 BUG_ON(bi == NULL); 272 BUG_ON(bio_integrity(bio)); 273 274 intervals = bio_integrity_intervals(bi, bio_sectors(bio)); 275 276 /* Allocate kernel buffer for protection data */ 277 len = intervals * bi->tuple_size; 278 buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); 279 if (unlikely(buf == NULL)) { 280 printk(KERN_ERR "could not allocate integrity buffer\n"); 281 return -ENOMEM; 282 } 283 284 end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 285 start = ((unsigned long) buf) >> PAGE_SHIFT; 286 nr_pages = end - start; 287 288 /* Allocate bio integrity payload and integrity vectors */ 289 bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages); 290 if (unlikely(bip == NULL)) { 291 printk(KERN_ERR "could not allocate data integrity bioset\n"); 292 kfree(buf); 293 return -EIO; 294 } 295 296 bip->bip_flags |= BIP_BLOCK_INTEGRITY; 297 bip->bip_iter.bi_size = len; 298 bip_set_seed(bip, bio->bi_iter.bi_sector); 299 300 if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) 301 bip->bip_flags |= BIP_IP_CHECKSUM; 302 303 /* Map it */ 304 offset = offset_in_page(buf); 305 for (i = 0 ; i < nr_pages ; i++) { 306 int ret; 307 bytes = PAGE_SIZE - offset; 308 309 if (len <= 0) 310 break; 311 312 if (bytes > len) 313 bytes = len; 314 315 ret = bio_integrity_add_page(bio, virt_to_page(buf), 316 bytes, offset); 317 318 if (ret == 0) 319 return 0; 320 321 if (ret < bytes) 322 break; 323 324 buf += bytes; 325 len -= bytes; 326 offset = 0; 327 } 328 329 /* Install custom I/O completion handler if read verify is enabled */ 330 if (bio_data_dir(bio) == READ) { 331 bip->bip_end_io = bio->bi_end_io; 332 bio->bi_end_io = bio_integrity_endio; 333 } 334 335 /* Auto-generate integrity metadata if this is a write */ 336 if (bio_data_dir(bio) == WRITE) 337 bio_integrity_process(bio, bi->generate_fn); 338 339 return 0; 340 } 341 EXPORT_SYMBOL(bio_integrity_prep); 342 343 /** 344 * bio_integrity_verify_fn - Integrity I/O completion worker 345 * @work: Work struct stored in bio to be verified 346 * 347 * Description: This workqueue function is called to complete a READ 348 * request. The function verifies the transferred integrity metadata 349 * and then calls the original bio end_io function. 350 */ 351 static void bio_integrity_verify_fn(struct work_struct *work) 352 { 353 struct bio_integrity_payload *bip = 354 container_of(work, struct bio_integrity_payload, bip_work); 355 struct bio *bio = bip->bip_bio; 356 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 357 int error; 358 359 error = bio_integrity_process(bio, bi->verify_fn); 360 361 /* Restore original bio completion handler */ 362 bio->bi_end_io = bip->bip_end_io; 363 bio_endio_nodec(bio, error); 364 } 365 366 /** 367 * bio_integrity_endio - Integrity I/O completion function 368 * @bio: Protected bio 369 * @error: Pointer to errno 370 * 371 * Description: Completion for integrity I/O 372 * 373 * Normally I/O completion is done in interrupt context. However, 374 * verifying I/O integrity is a time-consuming task which must be run 375 * in process context. This function postpones completion 376 * accordingly. 377 */ 378 void bio_integrity_endio(struct bio *bio, int error) 379 { 380 struct bio_integrity_payload *bip = bio_integrity(bio); 381 382 BUG_ON(bip->bip_bio != bio); 383 384 /* In case of an I/O error there is no point in verifying the 385 * integrity metadata. Restore original bio end_io handler 386 * and run it. 387 */ 388 if (error) { 389 bio->bi_end_io = bip->bip_end_io; 390 bio_endio_nodec(bio, error); 391 392 return; 393 } 394 395 INIT_WORK(&bip->bip_work, bio_integrity_verify_fn); 396 queue_work(kintegrityd_wq, &bip->bip_work); 397 } 398 EXPORT_SYMBOL(bio_integrity_endio); 399 400 /** 401 * bio_integrity_advance - Advance integrity vector 402 * @bio: bio whose integrity vector to update 403 * @bytes_done: number of data bytes that have been completed 404 * 405 * Description: This function calculates how many integrity bytes the 406 * number of completed data bytes correspond to and advances the 407 * integrity vector accordingly. 408 */ 409 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) 410 { 411 struct bio_integrity_payload *bip = bio_integrity(bio); 412 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 413 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); 414 415 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes); 416 } 417 EXPORT_SYMBOL(bio_integrity_advance); 418 419 /** 420 * bio_integrity_trim - Trim integrity vector 421 * @bio: bio whose integrity vector to update 422 * @offset: offset to first data sector 423 * @sectors: number of data sectors 424 * 425 * Description: Used to trim the integrity vector in a cloned bio. 426 * The ivec will be advanced corresponding to 'offset' data sectors 427 * and the length will be truncated corresponding to 'len' data 428 * sectors. 429 */ 430 void bio_integrity_trim(struct bio *bio, unsigned int offset, 431 unsigned int sectors) 432 { 433 struct bio_integrity_payload *bip = bio_integrity(bio); 434 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 435 436 bio_integrity_advance(bio, offset << 9); 437 bip->bip_iter.bi_size = bio_integrity_bytes(bi, sectors); 438 } 439 EXPORT_SYMBOL(bio_integrity_trim); 440 441 /** 442 * bio_integrity_clone - Callback for cloning bios with integrity metadata 443 * @bio: New bio 444 * @bio_src: Original bio 445 * @gfp_mask: Memory allocation mask 446 * 447 * Description: Called to allocate a bip when cloning a bio 448 */ 449 int bio_integrity_clone(struct bio *bio, struct bio *bio_src, 450 gfp_t gfp_mask) 451 { 452 struct bio_integrity_payload *bip_src = bio_integrity(bio_src); 453 struct bio_integrity_payload *bip; 454 455 BUG_ON(bip_src == NULL); 456 457 bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt); 458 459 if (bip == NULL) 460 return -EIO; 461 462 memcpy(bip->bip_vec, bip_src->bip_vec, 463 bip_src->bip_vcnt * sizeof(struct bio_vec)); 464 465 bip->bip_vcnt = bip_src->bip_vcnt; 466 bip->bip_iter = bip_src->bip_iter; 467 468 return 0; 469 } 470 EXPORT_SYMBOL(bio_integrity_clone); 471 472 int bioset_integrity_create(struct bio_set *bs, int pool_size) 473 { 474 if (bs->bio_integrity_pool) 475 return 0; 476 477 bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab); 478 if (!bs->bio_integrity_pool) 479 return -1; 480 481 bs->bvec_integrity_pool = biovec_create_pool(pool_size); 482 if (!bs->bvec_integrity_pool) { 483 mempool_destroy(bs->bio_integrity_pool); 484 return -1; 485 } 486 487 return 0; 488 } 489 EXPORT_SYMBOL(bioset_integrity_create); 490 491 void bioset_integrity_free(struct bio_set *bs) 492 { 493 if (bs->bio_integrity_pool) 494 mempool_destroy(bs->bio_integrity_pool); 495 496 if (bs->bvec_integrity_pool) 497 mempool_destroy(bs->bvec_integrity_pool); 498 } 499 EXPORT_SYMBOL(bioset_integrity_free); 500 501 void __init bio_integrity_init(void) 502 { 503 /* 504 * kintegrityd won't block much but may burn a lot of CPU cycles. 505 * Make it highpri CPU intensive wq with max concurrency of 1. 506 */ 507 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | 508 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1); 509 if (!kintegrityd_wq) 510 panic("Failed to create kintegrityd\n"); 511 512 bip_slab = kmem_cache_create("bio_integrity_payload", 513 sizeof(struct bio_integrity_payload) + 514 sizeof(struct bio_vec) * BIP_INLINE_VECS, 515 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 516 } 517