1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
4 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
5 *
6 * Automatically generate and verify integrity data on PI capable devices if the
7 * bio submitter didn't provide PI itself. This ensures that kernel verifies
8 * data integrity even if the file system (or other user of the block device) is
9 * not aware of PI.
10 */
11 #include <linux/blk-integrity.h>
12 #include <linux/t10-pi.h>
13 #include <linux/workqueue.h>
14 #include "blk.h"
15
16 struct bio_integrity_data {
17 struct bio *bio;
18 struct bvec_iter saved_bio_iter;
19 struct work_struct work;
20 struct bio_integrity_payload bip;
21 struct bio_vec bvec;
22 };
23
24 static struct kmem_cache *bid_slab;
25 static mempool_t bid_pool;
26 static struct workqueue_struct *kintegrityd_wq;
27
bio_integrity_finish(struct bio_integrity_data * bid)28 static void bio_integrity_finish(struct bio_integrity_data *bid)
29 {
30 bid->bio->bi_integrity = NULL;
31 bid->bio->bi_opf &= ~REQ_INTEGRITY;
32 bio_integrity_free_buf(&bid->bip);
33 mempool_free(bid, &bid_pool);
34 }
35
bio_integrity_verify_fn(struct work_struct * work)36 static void bio_integrity_verify_fn(struct work_struct *work)
37 {
38 struct bio_integrity_data *bid =
39 container_of(work, struct bio_integrity_data, work);
40 struct bio *bio = bid->bio;
41
42 bio->bi_status = bio_integrity_verify(bio, &bid->saved_bio_iter);
43 bio_integrity_finish(bid);
44 bio_endio(bio);
45 }
46
47 /**
48 * __bio_integrity_endio - Integrity I/O completion function
49 * @bio: Protected bio
50 *
51 * Normally I/O completion is done in interrupt context. However, verifying I/O
52 * integrity is a time-consuming task which must be run in process context.
53 *
54 * This function postpones completion accordingly.
55 */
__bio_integrity_endio(struct bio * bio)56 bool __bio_integrity_endio(struct bio *bio)
57 {
58 struct bio_integrity_payload *bip = bio_integrity(bio);
59 struct bio_integrity_data *bid =
60 container_of(bip, struct bio_integrity_data, bip);
61
62 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
63 (bip->bip_flags & BIP_CHECK_FLAGS)) {
64 INIT_WORK(&bid->work, bio_integrity_verify_fn);
65 queue_work(kintegrityd_wq, &bid->work);
66 return false;
67 }
68
69 bio_integrity_finish(bid);
70 return true;
71 }
72
73 /**
74 * bio_integrity_prep - Prepare bio for integrity I/O
75 * @bio: bio to prepare
76 * @action: preparation action needed (BI_ACT_*)
77 *
78 * Allocate the integrity payload. For writes, generate the integrity metadata
79 * and for reads, setup the completion handler to verify the metadata.
80 *
81 * This is used for bios that do not have user integrity payloads attached.
82 */
bio_integrity_prep(struct bio * bio,unsigned int action)83 void bio_integrity_prep(struct bio *bio, unsigned int action)
84 {
85 struct bio_integrity_data *bid;
86
87 bid = mempool_alloc(&bid_pool, GFP_NOIO);
88 bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
89 bid->bio = bio;
90 bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
91 bio_integrity_alloc_buf(bio, GFP_NOIO, action & BI_ACT_ZERO);
92 if (action & BI_ACT_CHECK)
93 bio_integrity_setup_default(bio);
94
95 /* Auto-generate integrity metadata if this is a write */
96 if (bio_data_dir(bio) == WRITE && (bid->bip.bip_flags & BIP_CHECK_FLAGS))
97 bio_integrity_generate(bio);
98 else
99 bid->saved_bio_iter = bio->bi_iter;
100 }
101 EXPORT_SYMBOL(bio_integrity_prep);
102
blk_flush_integrity(void)103 void blk_flush_integrity(void)
104 {
105 flush_workqueue(kintegrityd_wq);
106 }
107
blk_integrity_auto_init(void)108 static int __init blk_integrity_auto_init(void)
109 {
110 bid_slab = kmem_cache_create("bio_integrity_data",
111 sizeof(struct bio_integrity_data), 0,
112 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
113
114 if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab))
115 panic("bio: can't create integrity pool\n");
116
117 /*
118 * kintegrityd won't block much but may burn a lot of CPU cycles.
119 * Make it highpri CPU intensive wq with max concurrency of 1.
120 */
121 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
122 WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_PERCPU, 1);
123 if (!kintegrityd_wq)
124 panic("Failed to create kintegrityd\n");
125 return 0;
126 }
127 subsys_initcall(blk_integrity_auto_init);
128