1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007, 2008, 2009 Oracle Corporation 4 * Written by: Martin K. Petersen <martin.petersen@oracle.com> 5 * 6 * Automatically generate and verify integrity data on PI capable devices if the 7 * bio submitter didn't provide PI itself. This ensures that kernel verifies 8 * data integrity even if the file system (or other user of the block device) is 9 * not aware of PI. 10 */ 11 #include <linux/blk-integrity.h> 12 #include <linux/t10-pi.h> 13 #include <linux/workqueue.h> 14 #include "blk.h" 15 16 struct bio_integrity_data { 17 struct bio *bio; 18 struct bvec_iter saved_bio_iter; 19 struct work_struct work; 20 struct bio_integrity_payload bip; 21 struct bio_vec bvec; 22 }; 23 24 static struct kmem_cache *bid_slab; 25 static mempool_t bid_pool; 26 static struct workqueue_struct *kintegrityd_wq; 27 28 static void bio_integrity_finish(struct bio_integrity_data *bid) 29 { 30 bid->bio->bi_integrity = NULL; 31 bid->bio->bi_opf &= ~REQ_INTEGRITY; 32 bio_integrity_free_buf(&bid->bip); 33 mempool_free(bid, &bid_pool); 34 } 35 36 static void bio_integrity_verify_fn(struct work_struct *work) 37 { 38 struct bio_integrity_data *bid = 39 container_of(work, struct bio_integrity_data, work); 40 struct bio *bio = bid->bio; 41 42 bio->bi_status = bio_integrity_verify(bio, &bid->saved_bio_iter); 43 bio_integrity_finish(bid); 44 bio_endio(bio); 45 } 46 47 /** 48 * __bio_integrity_endio - Integrity I/O completion function 49 * @bio: Protected bio 50 * 51 * Normally I/O completion is done in interrupt context. However, verifying I/O 52 * integrity is a time-consuming task which must be run in process context. 53 * 54 * This function postpones completion accordingly. 55 */ 56 bool __bio_integrity_endio(struct bio *bio) 57 { 58 struct bio_integrity_payload *bip = bio_integrity(bio); 59 struct bio_integrity_data *bid = 60 container_of(bip, struct bio_integrity_data, bip); 61 62 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && 63 (bip->bip_flags & BIP_CHECK_FLAGS)) { 64 INIT_WORK(&bid->work, bio_integrity_verify_fn); 65 queue_work(kintegrityd_wq, &bid->work); 66 return false; 67 } 68 69 bio_integrity_finish(bid); 70 return true; 71 } 72 73 /** 74 * bio_integrity_prep - Prepare bio for integrity I/O 75 * @bio: bio to prepare 76 * @action: preparation action needed (BI_ACT_*) 77 * 78 * Allocate the integrity payload. For writes, generate the integrity metadata 79 * and for reads, setup the completion handler to verify the metadata. 80 * 81 * This is used for bios that do not have user integrity payloads attached. 82 */ 83 void bio_integrity_prep(struct bio *bio, unsigned int action) 84 { 85 struct bio_integrity_data *bid; 86 87 bid = mempool_alloc(&bid_pool, GFP_NOIO); 88 bio_integrity_init(bio, &bid->bip, &bid->bvec, 1); 89 bid->bio = bio; 90 bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY; 91 bio_integrity_alloc_buf(bio, GFP_NOIO, action & BI_ACT_ZERO); 92 if (action & BI_ACT_CHECK) 93 bio_integrity_setup_default(bio); 94 95 /* Auto-generate integrity metadata if this is a write */ 96 if (bio_data_dir(bio) == WRITE && (bid->bip.bip_flags & BIP_CHECK_FLAGS)) 97 bio_integrity_generate(bio); 98 else 99 bid->saved_bio_iter = bio->bi_iter; 100 } 101 EXPORT_SYMBOL(bio_integrity_prep); 102 103 void blk_flush_integrity(void) 104 { 105 flush_workqueue(kintegrityd_wq); 106 } 107 108 static int __init blk_integrity_auto_init(void) 109 { 110 bid_slab = kmem_cache_create("bio_integrity_data", 111 sizeof(struct bio_integrity_data), 0, 112 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); 113 114 if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab)) 115 panic("bio: can't create integrity pool\n"); 116 117 /* 118 * kintegrityd won't block much but may burn a lot of CPU cycles. 119 * Make it highpri CPU intensive wq with max concurrency of 1. 120 */ 121 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | 122 WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_PERCPU, 1); 123 if (!kintegrityd_wq) 124 panic("Failed to create kintegrityd\n"); 125 return 0; 126 } 127 subsys_initcall(blk_integrity_auto_init); 128