1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007, 2008, 2009 Oracle Corporation 4 * Written by: Martin K. Petersen <martin.petersen@oracle.com> 5 * 6 * Automatically generate and verify integrity data on PI capable devices if the 7 * bio submitter didn't provide PI itself. This ensures that kernel verifies 8 * data integrity even if the file system (or other user of the block device) is 9 * not aware of PI. 10 */ 11 #include <linux/blk-integrity.h> 12 #include <linux/t10-pi.h> 13 #include <linux/workqueue.h> 14 #include "blk.h" 15 16 struct bio_integrity_data { 17 struct bio *bio; 18 struct bvec_iter saved_bio_iter; 19 struct work_struct work; 20 struct bio_integrity_payload bip; 21 struct bio_vec bvec; 22 }; 23 24 static struct kmem_cache *bid_slab; 25 static mempool_t bid_pool; 26 static struct workqueue_struct *kintegrityd_wq; 27 28 static void bio_integrity_finish(struct bio_integrity_data *bid) 29 { 30 bid->bio->bi_integrity = NULL; 31 bid->bio->bi_opf &= ~REQ_INTEGRITY; 32 kfree(bvec_virt(bid->bip.bip_vec)); 33 mempool_free(bid, &bid_pool); 34 } 35 36 static void bio_integrity_verify_fn(struct work_struct *work) 37 { 38 struct bio_integrity_data *bid = 39 container_of(work, struct bio_integrity_data, work); 40 struct bio *bio = bid->bio; 41 42 blk_integrity_verify_iter(bio, &bid->saved_bio_iter); 43 bio_integrity_finish(bid); 44 bio_endio(bio); 45 } 46 47 #define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) 48 static bool bip_should_check(struct bio_integrity_payload *bip) 49 { 50 return bip->bip_flags & BIP_CHECK_FLAGS; 51 } 52 53 static bool bi_offload_capable(struct blk_integrity *bi) 54 { 55 switch (bi->csum_type) { 56 case BLK_INTEGRITY_CSUM_CRC64: 57 return bi->metadata_size == sizeof(struct crc64_pi_tuple); 58 case BLK_INTEGRITY_CSUM_CRC: 59 case BLK_INTEGRITY_CSUM_IP: 60 return bi->metadata_size == sizeof(struct t10_pi_tuple); 61 default: 62 pr_warn_once("%s: unknown integrity checksum type:%d\n", 63 __func__, bi->csum_type); 64 fallthrough; 65 case BLK_INTEGRITY_CSUM_NONE: 66 return false; 67 } 68 } 69 70 /** 71 * __bio_integrity_endio - Integrity I/O completion function 72 * @bio: Protected bio 73 * 74 * Normally I/O completion is done in interrupt context. However, verifying I/O 75 * integrity is a time-consuming task which must be run in process context. 76 * 77 * This function postpones completion accordingly. 78 */ 79 bool __bio_integrity_endio(struct bio *bio) 80 { 81 struct bio_integrity_payload *bip = bio_integrity(bio); 82 struct bio_integrity_data *bid = 83 container_of(bip, struct bio_integrity_data, bip); 84 85 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && 86 bip_should_check(bip)) { 87 INIT_WORK(&bid->work, bio_integrity_verify_fn); 88 queue_work(kintegrityd_wq, &bid->work); 89 return false; 90 } 91 92 bio_integrity_finish(bid); 93 return true; 94 } 95 96 /** 97 * bio_integrity_prep - Prepare bio for integrity I/O 98 * @bio: bio to prepare 99 * 100 * Checks if the bio already has an integrity payload attached. If it does, the 101 * payload has been generated by another kernel subsystem, and we just pass it 102 * through. 103 * Otherwise allocates integrity payload and for writes the integrity metadata 104 * will be generated. For reads, the completion handler will verify the 105 * metadata. 106 */ 107 bool bio_integrity_prep(struct bio *bio) 108 { 109 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); 110 struct bio_integrity_data *bid; 111 bool set_flags = true; 112 gfp_t gfp = GFP_NOIO; 113 unsigned int len; 114 void *buf; 115 116 if (!bi) 117 return true; 118 119 if (!bio_sectors(bio)) 120 return true; 121 122 /* Already protected? */ 123 if (bio_integrity(bio)) 124 return true; 125 126 switch (bio_op(bio)) { 127 case REQ_OP_READ: 128 if (bi->flags & BLK_INTEGRITY_NOVERIFY) { 129 if (bi_offload_capable(bi)) 130 return true; 131 set_flags = false; 132 } 133 break; 134 case REQ_OP_WRITE: 135 /* 136 * Zero the memory allocated to not leak uninitialized kernel 137 * memory to disk for non-integrity metadata where nothing else 138 * initializes the memory. 139 */ 140 if (bi->flags & BLK_INTEGRITY_NOGENERATE) { 141 if (bi_offload_capable(bi)) 142 return true; 143 set_flags = false; 144 gfp |= __GFP_ZERO; 145 } else if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE) 146 gfp |= __GFP_ZERO; 147 break; 148 default: 149 return true; 150 } 151 152 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) 153 return true; 154 155 /* Allocate kernel buffer for protection data */ 156 len = bio_integrity_bytes(bi, bio_sectors(bio)); 157 buf = kmalloc(len, gfp); 158 if (!buf) 159 goto err_end_io; 160 bid = mempool_alloc(&bid_pool, GFP_NOIO); 161 if (!bid) 162 goto err_free_buf; 163 bio_integrity_init(bio, &bid->bip, &bid->bvec, 1); 164 165 bid->bio = bio; 166 167 bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY; 168 bip_set_seed(&bid->bip, bio->bi_iter.bi_sector); 169 170 if (set_flags) { 171 if (bi->csum_type == BLK_INTEGRITY_CSUM_IP) 172 bid->bip.bip_flags |= BIP_IP_CHECKSUM; 173 if (bi->csum_type) 174 bid->bip.bip_flags |= BIP_CHECK_GUARD; 175 if (bi->flags & BLK_INTEGRITY_REF_TAG) 176 bid->bip.bip_flags |= BIP_CHECK_REFTAG; 177 } 178 179 if (bio_integrity_add_page(bio, virt_to_page(buf), len, 180 offset_in_page(buf)) < len) 181 goto err_end_io; 182 183 /* Auto-generate integrity metadata if this is a write */ 184 if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip)) 185 blk_integrity_generate(bio); 186 else 187 bid->saved_bio_iter = bio->bi_iter; 188 return true; 189 190 err_free_buf: 191 kfree(buf); 192 err_end_io: 193 bio->bi_status = BLK_STS_RESOURCE; 194 bio_endio(bio); 195 return false; 196 } 197 EXPORT_SYMBOL(bio_integrity_prep); 198 199 void blk_flush_integrity(void) 200 { 201 flush_workqueue(kintegrityd_wq); 202 } 203 204 static int __init blk_integrity_auto_init(void) 205 { 206 bid_slab = kmem_cache_create("bio_integrity_data", 207 sizeof(struct bio_integrity_data), 0, 208 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); 209 210 if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab)) 211 panic("bio: can't create integrity pool\n"); 212 213 /* 214 * kintegrityd won't block much but may burn a lot of CPU cycles. 215 * Make it highpri CPU intensive wq with max concurrency of 1. 216 */ 217 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | 218 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1); 219 if (!kintegrityd_wq) 220 panic("Failed to create kintegrityd\n"); 221 return 0; 222 } 223 subsys_initcall(blk_integrity_auto_init); 224