1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
4 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
5 *
6 * Automatically generate and verify integrity data on PI capable devices if the
7 * bio submitter didn't provide PI itself. This ensures that kernel verifies
8 * data integrity even if the file system (or other user of the block device) is
9 * not aware of PI.
10 */
11 #include <linux/blk-integrity.h>
12 #include <linux/t10-pi.h>
13 #include <linux/workqueue.h>
14 #include "blk.h"
15
16 struct bio_integrity_data {
17 struct bio *bio;
18 struct bvec_iter saved_bio_iter;
19 struct work_struct work;
20 struct bio_integrity_payload bip;
21 struct bio_vec bvec;
22 };
23
24 static struct kmem_cache *bid_slab;
25 static mempool_t bid_pool;
26 static struct workqueue_struct *kintegrityd_wq;
27
bio_integrity_finish(struct bio_integrity_data * bid)28 static void bio_integrity_finish(struct bio_integrity_data *bid)
29 {
30 bid->bio->bi_integrity = NULL;
31 bid->bio->bi_opf &= ~REQ_INTEGRITY;
32 kfree(bvec_virt(bid->bip.bip_vec));
33 mempool_free(bid, &bid_pool);
34 }
35
bio_integrity_verify_fn(struct work_struct * work)36 static void bio_integrity_verify_fn(struct work_struct *work)
37 {
38 struct bio_integrity_data *bid =
39 container_of(work, struct bio_integrity_data, work);
40 struct bio *bio = bid->bio;
41
42 blk_integrity_verify_iter(bio, &bid->saved_bio_iter);
43 bio_integrity_finish(bid);
44 bio_endio(bio);
45 }
46
47 #define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG)
bip_should_check(struct bio_integrity_payload * bip)48 static bool bip_should_check(struct bio_integrity_payload *bip)
49 {
50 return bip->bip_flags & BIP_CHECK_FLAGS;
51 }
52
bi_offload_capable(struct blk_integrity * bi)53 static bool bi_offload_capable(struct blk_integrity *bi)
54 {
55 switch (bi->csum_type) {
56 case BLK_INTEGRITY_CSUM_CRC64:
57 return bi->tuple_size == sizeof(struct crc64_pi_tuple);
58 case BLK_INTEGRITY_CSUM_CRC:
59 case BLK_INTEGRITY_CSUM_IP:
60 return bi->tuple_size == sizeof(struct t10_pi_tuple);
61 default:
62 pr_warn_once("%s: unknown integrity checksum type:%d\n",
63 __func__, bi->csum_type);
64 fallthrough;
65 case BLK_INTEGRITY_CSUM_NONE:
66 return false;
67 }
68 }
69
70 /**
71 * __bio_integrity_endio - Integrity I/O completion function
72 * @bio: Protected bio
73 *
74 * Normally I/O completion is done in interrupt context. However, verifying I/O
75 * integrity is a time-consuming task which must be run in process context.
76 *
77 * This function postpones completion accordingly.
78 */
__bio_integrity_endio(struct bio * bio)79 bool __bio_integrity_endio(struct bio *bio)
80 {
81 struct bio_integrity_payload *bip = bio_integrity(bio);
82 struct bio_integrity_data *bid =
83 container_of(bip, struct bio_integrity_data, bip);
84
85 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
86 bip_should_check(bip)) {
87 INIT_WORK(&bid->work, bio_integrity_verify_fn);
88 queue_work(kintegrityd_wq, &bid->work);
89 return false;
90 }
91
92 bio_integrity_finish(bid);
93 return true;
94 }
95
96 /**
97 * bio_integrity_prep - Prepare bio for integrity I/O
98 * @bio: bio to prepare
99 *
100 * Checks if the bio already has an integrity payload attached. If it does, the
101 * payload has been generated by another kernel subsystem, and we just pass it
102 * through.
103 * Otherwise allocates integrity payload and for writes the integrity metadata
104 * will be generated. For reads, the completion handler will verify the
105 * metadata.
106 */
bio_integrity_prep(struct bio * bio)107 bool bio_integrity_prep(struct bio *bio)
108 {
109 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
110 struct bio_integrity_data *bid;
111 bool set_flags = true;
112 gfp_t gfp = GFP_NOIO;
113 unsigned int len;
114 void *buf;
115
116 if (!bi)
117 return true;
118
119 if (!bio_sectors(bio))
120 return true;
121
122 /* Already protected? */
123 if (bio_integrity(bio))
124 return true;
125
126 switch (bio_op(bio)) {
127 case REQ_OP_READ:
128 if (bi->flags & BLK_INTEGRITY_NOVERIFY) {
129 if (bi_offload_capable(bi))
130 return true;
131 set_flags = false;
132 }
133 break;
134 case REQ_OP_WRITE:
135 /*
136 * Zero the memory allocated to not leak uninitialized kernel
137 * memory to disk for non-integrity metadata where nothing else
138 * initializes the memory.
139 */
140 if (bi->flags & BLK_INTEGRITY_NOGENERATE) {
141 if (bi_offload_capable(bi))
142 return true;
143 set_flags = false;
144 gfp |= __GFP_ZERO;
145 } else if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE)
146 gfp |= __GFP_ZERO;
147 break;
148 default:
149 return true;
150 }
151
152 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
153 return true;
154
155 /* Allocate kernel buffer for protection data */
156 len = bio_integrity_bytes(bi, bio_sectors(bio));
157 buf = kmalloc(len, gfp);
158 if (!buf)
159 goto err_end_io;
160 bid = mempool_alloc(&bid_pool, GFP_NOIO);
161 if (!bid)
162 goto err_free_buf;
163 bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
164
165 bid->bio = bio;
166
167 bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
168 bip_set_seed(&bid->bip, bio->bi_iter.bi_sector);
169
170 if (set_flags) {
171 if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
172 bid->bip.bip_flags |= BIP_IP_CHECKSUM;
173 if (bi->csum_type)
174 bid->bip.bip_flags |= BIP_CHECK_GUARD;
175 if (bi->flags & BLK_INTEGRITY_REF_TAG)
176 bid->bip.bip_flags |= BIP_CHECK_REFTAG;
177 }
178
179 if (bio_integrity_add_page(bio, virt_to_page(buf), len,
180 offset_in_page(buf)) < len)
181 goto err_end_io;
182
183 /* Auto-generate integrity metadata if this is a write */
184 if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip))
185 blk_integrity_generate(bio);
186 else
187 bid->saved_bio_iter = bio->bi_iter;
188 return true;
189
190 err_free_buf:
191 kfree(buf);
192 err_end_io:
193 bio->bi_status = BLK_STS_RESOURCE;
194 bio_endio(bio);
195 return false;
196 }
197 EXPORT_SYMBOL(bio_integrity_prep);
198
blk_flush_integrity(void)199 void blk_flush_integrity(void)
200 {
201 flush_workqueue(kintegrityd_wq);
202 }
203
blk_integrity_auto_init(void)204 static int __init blk_integrity_auto_init(void)
205 {
206 bid_slab = kmem_cache_create("bio_integrity_data",
207 sizeof(struct bio_integrity_data), 0,
208 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
209
210 if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab))
211 panic("bio: can't create integrity pool\n");
212
213 /*
214 * kintegrityd won't block much but may burn a lot of CPU cycles.
215 * Make it highpri CPU intensive wq with max concurrency of 1.
216 */
217 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
218 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);
219 if (!kintegrityd_wq)
220 panic("Failed to create kintegrityd\n");
221 return 0;
222 }
223 subsys_initcall(blk_integrity_auto_init);
224