1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007, 2008, 2009 Oracle Corporation 4 * Written by: Martin K. Petersen <martin.petersen@oracle.com> 5 * 6 * Automatically generate and verify integrity data on PI capable devices if the 7 * bio submitter didn't provide PI itself. This ensures that kernel verifies 8 * data integrity even if the file system (or other user of the block device) is 9 * not aware of PI. 10 */ 11 #include <linux/blk-integrity.h> 12 #include <linux/workqueue.h> 13 #include "blk.h" 14 15 struct bio_integrity_data { 16 struct bio *bio; 17 struct bvec_iter saved_bio_iter; 18 struct work_struct work; 19 struct bio_integrity_payload bip; 20 struct bio_vec bvec; 21 }; 22 23 static struct kmem_cache *bid_slab; 24 static mempool_t bid_pool; 25 static struct workqueue_struct *kintegrityd_wq; 26 27 static void bio_integrity_finish(struct bio_integrity_data *bid) 28 { 29 bid->bio->bi_integrity = NULL; 30 bid->bio->bi_opf &= ~REQ_INTEGRITY; 31 kfree(bvec_virt(bid->bip.bip_vec)); 32 mempool_free(bid, &bid_pool); 33 } 34 35 static void bio_integrity_verify_fn(struct work_struct *work) 36 { 37 struct bio_integrity_data *bid = 38 container_of(work, struct bio_integrity_data, work); 39 struct bio *bio = bid->bio; 40 41 blk_integrity_verify_iter(bio, &bid->saved_bio_iter); 42 bio_integrity_finish(bid); 43 bio_endio(bio); 44 } 45 46 /** 47 * __bio_integrity_endio - Integrity I/O completion function 48 * @bio: Protected bio 49 * 50 * Normally I/O completion is done in interrupt context. However, verifying I/O 51 * integrity is a time-consuming task which must be run in process context. 52 * 53 * This function postpones completion accordingly. 54 */ 55 bool __bio_integrity_endio(struct bio *bio) 56 { 57 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 58 struct bio_integrity_payload *bip = bio_integrity(bio); 59 struct bio_integrity_data *bid = 60 container_of(bip, struct bio_integrity_data, bip); 61 62 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && bi->csum_type) { 63 INIT_WORK(&bid->work, bio_integrity_verify_fn); 64 queue_work(kintegrityd_wq, &bid->work); 65 return false; 66 } 67 68 bio_integrity_finish(bid); 69 return true; 70 } 71 72 /** 73 * bio_integrity_prep - Prepare bio for integrity I/O 74 * @bio: bio to prepare 75 * 76 * Checks if the bio already has an integrity payload attached. If it does, the 77 * payload has been generated by another kernel subsystem, and we just pass it 78 * through. 79 * Otherwise allocates integrity payload and for writes the integrity metadata 80 * will be generated. For reads, the completion handler will verify the 81 * metadata. 82 */ 83 bool bio_integrity_prep(struct bio *bio) 84 { 85 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 86 struct bio_integrity_data *bid; 87 gfp_t gfp = GFP_NOIO; 88 unsigned int len; 89 void *buf; 90 91 if (!bi) 92 return true; 93 94 if (!bio_sectors(bio)) 95 return true; 96 97 /* Already protected? */ 98 if (bio_integrity(bio)) 99 return true; 100 101 switch (bio_op(bio)) { 102 case REQ_OP_READ: 103 if (bi->flags & BLK_INTEGRITY_NOVERIFY) 104 return true; 105 break; 106 case REQ_OP_WRITE: 107 if (bi->flags & BLK_INTEGRITY_NOGENERATE) 108 return true; 109 110 /* 111 * Zero the memory allocated to not leak uninitialized kernel 112 * memory to disk for non-integrity metadata where nothing else 113 * initializes the memory. 114 */ 115 if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE) 116 gfp |= __GFP_ZERO; 117 break; 118 default: 119 return true; 120 } 121 122 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) 123 return true; 124 125 /* Allocate kernel buffer for protection data */ 126 len = bio_integrity_bytes(bi, bio_sectors(bio)); 127 buf = kmalloc(len, gfp); 128 if (!buf) 129 goto err_end_io; 130 bid = mempool_alloc(&bid_pool, GFP_NOIO); 131 if (!bid) 132 goto err_free_buf; 133 bio_integrity_init(bio, &bid->bip, &bid->bvec, 1); 134 135 bid->bio = bio; 136 137 bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY; 138 bip_set_seed(&bid->bip, bio->bi_iter.bi_sector); 139 140 if (bi->csum_type == BLK_INTEGRITY_CSUM_IP) 141 bid->bip.bip_flags |= BIP_IP_CHECKSUM; 142 if (bi->csum_type) 143 bid->bip.bip_flags |= BIP_CHECK_GUARD; 144 if (bi->flags & BLK_INTEGRITY_REF_TAG) 145 bid->bip.bip_flags |= BIP_CHECK_REFTAG; 146 147 if (bio_integrity_add_page(bio, virt_to_page(buf), len, 148 offset_in_page(buf)) < len) 149 goto err_end_io; 150 151 /* Auto-generate integrity metadata if this is a write */ 152 if (bio_data_dir(bio) == WRITE) 153 blk_integrity_generate(bio); 154 else 155 bid->saved_bio_iter = bio->bi_iter; 156 return true; 157 158 err_free_buf: 159 kfree(buf); 160 err_end_io: 161 bio->bi_status = BLK_STS_RESOURCE; 162 bio_endio(bio); 163 return false; 164 } 165 EXPORT_SYMBOL(bio_integrity_prep); 166 167 void blk_flush_integrity(void) 168 { 169 flush_workqueue(kintegrityd_wq); 170 } 171 172 static int __init blk_integrity_auto_init(void) 173 { 174 bid_slab = kmem_cache_create("bio_integrity_data", 175 sizeof(struct bio_integrity_data), 0, 176 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); 177 178 if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab)) 179 panic("bio: can't create integrity pool\n"); 180 181 /* 182 * kintegrityd won't block much but may burn a lot of CPU cycles. 183 * Make it highpri CPU intensive wq with max concurrency of 1. 184 */ 185 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | 186 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1); 187 if (!kintegrityd_wq) 188 panic("Failed to create kintegrityd\n"); 189 return 0; 190 } 191 subsys_initcall(blk_integrity_auto_init); 192