xref: /linux/block/bio-integrity.c (revision 6d8854216ebb60959ddb6f4ea4123bd449ba6cf6)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * bio-integrity.c - bio data integrity extensions
4  *
5  * Copyright (C) 2007, 2008, 2009 Oracle Corporation
6  * Written by: Martin K. Petersen <martin.petersen@oracle.com>
7  */
8 
9 #include <linux/blk-integrity.h>
10 #include "blk.h"
11 
12 struct bio_integrity_alloc {
13 	struct bio_integrity_payload	bip;
14 	struct bio_vec			bvecs[];
15 };
16 
17 /**
18  * bio_integrity_free - Free bio integrity payload
19  * @bio:	bio containing bip to be freed
20  *
21  * Description: Free the integrity portion of a bio.
22  */
bio_integrity_free(struct bio * bio)23 void bio_integrity_free(struct bio *bio)
24 {
25 	kfree(bio_integrity(bio));
26 	bio->bi_integrity = NULL;
27 	bio->bi_opf &= ~REQ_INTEGRITY;
28 }
29 
bio_integrity_init(struct bio * bio,struct bio_integrity_payload * bip,struct bio_vec * bvecs,unsigned int nr_vecs)30 void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
31 		struct bio_vec *bvecs, unsigned int nr_vecs)
32 {
33 	memset(bip, 0, sizeof(*bip));
34 	bip->bip_max_vcnt = nr_vecs;
35 	if (nr_vecs)
36 		bip->bip_vec = bvecs;
37 
38 	bio->bi_integrity = bip;
39 	bio->bi_opf |= REQ_INTEGRITY;
40 }
41 
42 /**
43  * bio_integrity_alloc - Allocate integrity payload and attach it to bio
44  * @bio:	bio to attach integrity metadata to
45  * @gfp_mask:	Memory allocation mask
46  * @nr_vecs:	Number of integrity metadata scatter-gather elements
47  *
48  * Description: This function prepares a bio for attaching integrity
49  * metadata.  nr_vecs specifies the maximum number of pages containing
50  * integrity metadata that can be attached.
51  */
bio_integrity_alloc(struct bio * bio,gfp_t gfp_mask,unsigned int nr_vecs)52 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
53 						  gfp_t gfp_mask,
54 						  unsigned int nr_vecs)
55 {
56 	struct bio_integrity_alloc *bia;
57 
58 	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
59 		return ERR_PTR(-EOPNOTSUPP);
60 
61 	bia = kmalloc(struct_size(bia, bvecs, nr_vecs), gfp_mask);
62 	if (unlikely(!bia))
63 		return ERR_PTR(-ENOMEM);
64 	bio_integrity_init(bio, &bia->bip, bia->bvecs, nr_vecs);
65 	return &bia->bip;
66 }
67 EXPORT_SYMBOL(bio_integrity_alloc);
68 
bio_integrity_unpin_bvec(struct bio_vec * bv,int nr_vecs)69 static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs)
70 {
71 	int i;
72 
73 	for (i = 0; i < nr_vecs; i++)
74 		unpin_user_page(bv[i].bv_page);
75 }
76 
bio_integrity_uncopy_user(struct bio_integrity_payload * bip)77 static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip)
78 {
79 	unsigned short orig_nr_vecs = bip->bip_max_vcnt - 1;
80 	struct bio_vec *orig_bvecs = &bip->bip_vec[1];
81 	struct bio_vec *bounce_bvec = &bip->bip_vec[0];
82 	size_t bytes = bounce_bvec->bv_len;
83 	struct iov_iter orig_iter;
84 	int ret;
85 
86 	iov_iter_bvec(&orig_iter, ITER_DEST, orig_bvecs, orig_nr_vecs, bytes);
87 	ret = copy_to_iter(bvec_virt(bounce_bvec), bytes, &orig_iter);
88 	WARN_ON_ONCE(ret != bytes);
89 
90 	bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs);
91 }
92 
93 /**
94  * bio_integrity_unmap_user - Unmap user integrity payload
95  * @bio:	bio containing bip to be unmapped
96  *
97  * Unmap the user mapped integrity portion of a bio.
98  */
bio_integrity_unmap_user(struct bio * bio)99 void bio_integrity_unmap_user(struct bio *bio)
100 {
101 	struct bio_integrity_payload *bip = bio_integrity(bio);
102 
103 	if (bip->bip_flags & BIP_COPY_USER) {
104 		if (bio_data_dir(bio) == READ)
105 			bio_integrity_uncopy_user(bip);
106 		kfree(bvec_virt(bip->bip_vec));
107 		return;
108 	}
109 
110 	bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt);
111 }
112 
113 /**
114  * bio_integrity_add_page - Attach integrity metadata
115  * @bio:	bio to update
116  * @page:	page containing integrity metadata
117  * @len:	number of bytes of integrity metadata in page
118  * @offset:	start offset within page
119  *
120  * Description: Attach a page containing integrity metadata to bio.
121  */
bio_integrity_add_page(struct bio * bio,struct page * page,unsigned int len,unsigned int offset)122 int bio_integrity_add_page(struct bio *bio, struct page *page,
123 			   unsigned int len, unsigned int offset)
124 {
125 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
126 	struct bio_integrity_payload *bip = bio_integrity(bio);
127 
128 	if (bip->bip_vcnt > 0) {
129 		struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1];
130 
131 		if (bvec_try_merge_hw_page(q, bv, page, len, offset)) {
132 			bip->bip_iter.bi_size += len;
133 			return len;
134 		}
135 
136 		if (bip->bip_vcnt >=
137 		    min(bip->bip_max_vcnt, queue_max_integrity_segments(q)))
138 			return 0;
139 
140 		/*
141 		 * If the queue doesn't support SG gaps and adding this segment
142 		 * would create a gap, disallow it.
143 		 */
144 		if (bvec_gap_to_prev(&q->limits, bv, offset))
145 			return 0;
146 	}
147 
148 	bvec_set_page(&bip->bip_vec[bip->bip_vcnt], page, len, offset);
149 	bip->bip_vcnt++;
150 	bip->bip_iter.bi_size += len;
151 
152 	return len;
153 }
154 EXPORT_SYMBOL(bio_integrity_add_page);
155 
bio_integrity_copy_user(struct bio * bio,struct bio_vec * bvec,int nr_vecs,unsigned int len)156 static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
157 				   int nr_vecs, unsigned int len)
158 {
159 	bool write = op_is_write(bio_op(bio));
160 	struct bio_integrity_payload *bip;
161 	struct iov_iter iter;
162 	void *buf;
163 	int ret;
164 
165 	buf = kmalloc(len, GFP_KERNEL);
166 	if (!buf)
167 		return -ENOMEM;
168 
169 	if (write) {
170 		iov_iter_bvec(&iter, ITER_SOURCE, bvec, nr_vecs, len);
171 		if (!copy_from_iter_full(buf, len, &iter)) {
172 			ret = -EFAULT;
173 			goto free_buf;
174 		}
175 
176 		bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
177 	} else {
178 		memset(buf, 0, len);
179 
180 		/*
181 		 * We need to preserve the original bvec and the number of vecs
182 		 * in it for completion handling
183 		 */
184 		bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs + 1);
185 	}
186 
187 	if (IS_ERR(bip)) {
188 		ret = PTR_ERR(bip);
189 		goto free_buf;
190 	}
191 
192 	if (write)
193 		bio_integrity_unpin_bvec(bvec, nr_vecs);
194 	else
195 		memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
196 
197 	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
198 				     offset_in_page(buf));
199 	if (ret != len) {
200 		ret = -ENOMEM;
201 		goto free_bip;
202 	}
203 
204 	bip->bip_flags |= BIP_COPY_USER;
205 	bip->bip_vcnt = nr_vecs;
206 	return 0;
207 free_bip:
208 	bio_integrity_free(bio);
209 free_buf:
210 	kfree(buf);
211 	return ret;
212 }
213 
bio_integrity_init_user(struct bio * bio,struct bio_vec * bvec,int nr_vecs,unsigned int len)214 static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec,
215 				   int nr_vecs, unsigned int len)
216 {
217 	struct bio_integrity_payload *bip;
218 
219 	bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs);
220 	if (IS_ERR(bip))
221 		return PTR_ERR(bip);
222 
223 	memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec));
224 	bip->bip_iter.bi_size = len;
225 	bip->bip_vcnt = nr_vecs;
226 	return 0;
227 }
228 
bvec_from_pages(struct bio_vec * bvec,struct page ** pages,int nr_vecs,ssize_t bytes,ssize_t offset)229 static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
230 				    int nr_vecs, ssize_t bytes, ssize_t offset)
231 {
232 	unsigned int nr_bvecs = 0;
233 	int i, j;
234 
235 	for (i = 0; i < nr_vecs; i = j) {
236 		size_t size = min_t(size_t, bytes, PAGE_SIZE - offset);
237 		struct folio *folio = page_folio(pages[i]);
238 
239 		bytes -= size;
240 		for (j = i + 1; j < nr_vecs; j++) {
241 			size_t next = min_t(size_t, PAGE_SIZE, bytes);
242 
243 			if (page_folio(pages[j]) != folio ||
244 			    pages[j] != pages[j - 1] + 1)
245 				break;
246 			unpin_user_page(pages[j]);
247 			size += next;
248 			bytes -= next;
249 		}
250 
251 		bvec_set_page(&bvec[nr_bvecs], pages[i], size, offset);
252 		offset = 0;
253 		nr_bvecs++;
254 	}
255 
256 	return nr_bvecs;
257 }
258 
bio_integrity_map_user(struct bio * bio,struct iov_iter * iter)259 int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
260 {
261 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
262 	unsigned int align = blk_lim_dma_alignment_and_pad(&q->limits);
263 	struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages;
264 	struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec;
265 	size_t offset, bytes = iter->count;
266 	unsigned int nr_bvecs;
267 	int ret, nr_vecs;
268 	bool copy;
269 
270 	if (bio_integrity(bio))
271 		return -EINVAL;
272 	if (bytes >> SECTOR_SHIFT > queue_max_hw_sectors(q))
273 		return -E2BIG;
274 
275 	nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS + 1);
276 	if (nr_vecs > BIO_MAX_VECS)
277 		return -E2BIG;
278 	if (nr_vecs > UIO_FASTIOV) {
279 		bvec = kcalloc(nr_vecs, sizeof(*bvec), GFP_KERNEL);
280 		if (!bvec)
281 			return -ENOMEM;
282 		pages = NULL;
283 	}
284 
285 	copy = !iov_iter_is_aligned(iter, align, align);
286 	ret = iov_iter_extract_pages(iter, &pages, bytes, nr_vecs, 0, &offset);
287 	if (unlikely(ret < 0))
288 		goto free_bvec;
289 
290 	nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset);
291 	if (pages != stack_pages)
292 		kvfree(pages);
293 	if (nr_bvecs > queue_max_integrity_segments(q))
294 		copy = true;
295 
296 	if (copy)
297 		ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes);
298 	else
299 		ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes);
300 	if (ret)
301 		goto release_pages;
302 	if (bvec != stack_vec)
303 		kfree(bvec);
304 
305 	return 0;
306 
307 release_pages:
308 	bio_integrity_unpin_bvec(bvec, nr_bvecs);
309 free_bvec:
310 	if (bvec != stack_vec)
311 		kfree(bvec);
312 	return ret;
313 }
314 
bio_uio_meta_to_bip(struct bio * bio,struct uio_meta * meta)315 static void bio_uio_meta_to_bip(struct bio *bio, struct uio_meta *meta)
316 {
317 	struct bio_integrity_payload *bip = bio_integrity(bio);
318 
319 	if (meta->flags & IO_INTEGRITY_CHK_GUARD)
320 		bip->bip_flags |= BIP_CHECK_GUARD;
321 	if (meta->flags & IO_INTEGRITY_CHK_APPTAG)
322 		bip->bip_flags |= BIP_CHECK_APPTAG;
323 	if (meta->flags & IO_INTEGRITY_CHK_REFTAG)
324 		bip->bip_flags |= BIP_CHECK_REFTAG;
325 
326 	bip->app_tag = meta->app_tag;
327 }
328 
bio_integrity_map_iter(struct bio * bio,struct uio_meta * meta)329 int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta)
330 {
331 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
332 	unsigned int integrity_bytes;
333 	int ret;
334 	struct iov_iter it;
335 
336 	if (!bi)
337 		return -EINVAL;
338 	/*
339 	 * original meta iterator can be bigger.
340 	 * process integrity info corresponding to current data buffer only.
341 	 */
342 	it = meta->iter;
343 	integrity_bytes = bio_integrity_bytes(bi, bio_sectors(bio));
344 	if (it.count < integrity_bytes)
345 		return -EINVAL;
346 
347 	/* should fit into two bytes */
348 	BUILD_BUG_ON(IO_INTEGRITY_VALID_FLAGS >= (1 << 16));
349 
350 	if (meta->flags && (meta->flags & ~IO_INTEGRITY_VALID_FLAGS))
351 		return -EINVAL;
352 
353 	it.count = integrity_bytes;
354 	ret = bio_integrity_map_user(bio, &it);
355 	if (!ret) {
356 		bio_uio_meta_to_bip(bio, meta);
357 		bip_set_seed(bio_integrity(bio), meta->seed);
358 		iov_iter_advance(&meta->iter, integrity_bytes);
359 		meta->seed += bio_integrity_intervals(bi, bio_sectors(bio));
360 	}
361 	return ret;
362 }
363 
364 /**
365  * bio_integrity_advance - Advance integrity vector
366  * @bio:	bio whose integrity vector to update
367  * @bytes_done:	number of data bytes that have been completed
368  *
369  * Description: This function calculates how many integrity bytes the
370  * number of completed data bytes correspond to and advances the
371  * integrity vector accordingly.
372  */
bio_integrity_advance(struct bio * bio,unsigned int bytes_done)373 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
374 {
375 	struct bio_integrity_payload *bip = bio_integrity(bio);
376 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
377 	unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
378 
379 	bip->bip_iter.bi_sector += bio_integrity_intervals(bi, bytes_done >> 9);
380 	bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
381 }
382 
383 /**
384  * bio_integrity_trim - Trim integrity vector
385  * @bio:	bio whose integrity vector to update
386  *
387  * Description: Used to trim the integrity vector in a cloned bio.
388  */
bio_integrity_trim(struct bio * bio)389 void bio_integrity_trim(struct bio *bio)
390 {
391 	struct bio_integrity_payload *bip = bio_integrity(bio);
392 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
393 
394 	bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
395 }
396 EXPORT_SYMBOL(bio_integrity_trim);
397 
398 /**
399  * bio_integrity_clone - Callback for cloning bios with integrity metadata
400  * @bio:	New bio
401  * @bio_src:	Original bio
402  * @gfp_mask:	Memory allocation mask
403  *
404  * Description:	Called to allocate a bip when cloning a bio
405  */
bio_integrity_clone(struct bio * bio,struct bio * bio_src,gfp_t gfp_mask)406 int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
407 			gfp_t gfp_mask)
408 {
409 	struct bio_integrity_payload *bip_src = bio_integrity(bio_src);
410 	struct bio_integrity_payload *bip;
411 
412 	BUG_ON(bip_src == NULL);
413 
414 	bip = bio_integrity_alloc(bio, gfp_mask, 0);
415 	if (IS_ERR(bip))
416 		return PTR_ERR(bip);
417 
418 	bip->bip_vec = bip_src->bip_vec;
419 	bip->bip_iter = bip_src->bip_iter;
420 	bip->bip_flags = bip_src->bip_flags & BIP_CLONE_FLAGS;
421 	bip->app_tag = bip_src->app_tag;
422 
423 	return 0;
424 }
425