xref: /linux/block/bio-integrity.c (revision a1c8bdbbd72564cebb0d02948c1ed57b80b2e773) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * bio-integrity.c - bio data integrity extensions
4  *
5  * Copyright (C) 2007, 2008, 2009 Oracle Corporation
6  * Written by: Martin K. Petersen <martin.petersen@oracle.com>
7  */
8 
9 #include <linux/blk-integrity.h>
10 #include <linux/t10-pi.h>
11 #include "blk.h"
12 
13 struct bio_integrity_alloc {
14 	struct bio_integrity_payload	bip;
15 	struct bio_vec			bvecs[];
16 };
17 
18 static mempool_t integrity_buf_pool;
19 
20 static bool bi_offload_capable(struct blk_integrity *bi)
21 {
22 	return bi->metadata_size == bi->pi_tuple_size;
23 }
24 
25 unsigned int __bio_integrity_action(struct bio *bio)
26 {
27 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
28 
29 	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
30 		return 0;
31 
32 	switch (bio_op(bio)) {
33 	case REQ_OP_READ:
34 		if (bi->flags & BLK_INTEGRITY_NOVERIFY) {
35 			if (bi_offload_capable(bi))
36 				return 0;
37 			return BI_ACT_BUFFER;
38 		}
39 		return BI_ACT_BUFFER | BI_ACT_CHECK;
40 	case REQ_OP_WRITE:
41 	case REQ_OP_ZONE_APPEND:
42 		/*
43 		 * Flush masquerading as write?
44 		 */
45 		if (!bio_sectors(bio))
46 			return 0;
47 
48 		/*
49 		 * Zero the memory allocated to not leak uninitialized kernel
50 		 * memory to disk for non-integrity metadata where nothing else
51 		 * initializes the memory.
52 		 */
53 		if (bi->flags & BLK_INTEGRITY_NOGENERATE) {
54 			if (bi_offload_capable(bi))
55 				return 0;
56 			return BI_ACT_BUFFER | BI_ACT_ZERO;
57 		}
58 
59 		if (bi->metadata_size > bi->pi_tuple_size)
60 			return BI_ACT_BUFFER | BI_ACT_CHECK | BI_ACT_ZERO;
61 		return BI_ACT_BUFFER | BI_ACT_CHECK;
62 	default:
63 		return 0;
64 	}
65 }
66 EXPORT_SYMBOL_GPL(__bio_integrity_action);
67 
68 void bio_integrity_alloc_buf(struct bio *bio, gfp_t gfp, bool zero_buffer)
69 {
70 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
71 	struct bio_integrity_payload *bip = bio_integrity(bio);
72 	unsigned int len = bio_integrity_bytes(bi, bio_sectors(bio));
73 	void *buf;
74 
75 	buf = kmalloc(len, gfp | __GFP_NOWARN | (zero_buffer ? __GFP_ZERO : 0));
76 	if (unlikely(!buf)) {
77 		struct page *page;
78 
79 		page = mempool_alloc(&integrity_buf_pool, gfp);
80 		if (zero_buffer)
81 			memset(page_address(page), 0, len);
82 		bvec_set_page(&bip->bip_vec[0], page, len, 0);
83 		bip->bip_flags |= BIP_MEMPOOL;
84 	} else {
85 		bvec_set_page(&bip->bip_vec[0], virt_to_page(buf), len,
86 				offset_in_page(buf));
87 	}
88 
89 	bip->bip_vcnt = 1;
90 	bip->bip_iter.bi_size = len;
91 }
92 
93 void bio_integrity_free_buf(struct bio_integrity_payload *bip)
94 {
95 	struct bio_vec *bv = &bip->bip_vec[0];
96 
97 	if (bip->bip_flags & BIP_MEMPOOL)
98 		mempool_free(bv->bv_page, &integrity_buf_pool);
99 	else
100 		kfree(bvec_virt(bv));
101 }
102 
103 void bio_integrity_setup_default(struct bio *bio)
104 {
105 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
106 	struct bio_integrity_payload *bip = bio_integrity(bio);
107 
108 	bip_set_seed(bip, bio->bi_iter.bi_sector);
109 
110 	if (bi->csum_type) {
111 		bip->bip_flags |= BIP_CHECK_GUARD;
112 		if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
113 			bip->bip_flags |= BIP_IP_CHECKSUM;
114 	}
115 	if (bi->flags & BLK_INTEGRITY_REF_TAG)
116 		bip->bip_flags |= BIP_CHECK_REFTAG;
117 }
118 
119 /**
120  * bio_integrity_free - Free bio integrity payload
121  * @bio:	bio containing bip to be freed
122  *
123  * Description: Free the integrity portion of a bio.
124  */
125 void bio_integrity_free(struct bio *bio)
126 {
127 	kfree(bio_integrity(bio));
128 	bio->bi_integrity = NULL;
129 	bio->bi_opf &= ~REQ_INTEGRITY;
130 }
131 
132 void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
133 		struct bio_vec *bvecs, unsigned int nr_vecs)
134 {
135 	memset(bip, 0, sizeof(*bip));
136 	bip->bip_max_vcnt = nr_vecs;
137 	if (nr_vecs)
138 		bip->bip_vec = bvecs;
139 
140 	bio->bi_integrity = bip;
141 	bio->bi_opf |= REQ_INTEGRITY;
142 }
143 
144 /**
145  * bio_integrity_alloc - Allocate integrity payload and attach it to bio
146  * @bio:	bio to attach integrity metadata to
147  * @gfp_mask:	Memory allocation mask
148  * @nr_vecs:	Number of integrity metadata scatter-gather elements
149  *
150  * Description: This function prepares a bio for attaching integrity
151  * metadata.  nr_vecs specifies the maximum number of pages containing
152  * integrity metadata that can be attached.
153  */
154 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
155 						  gfp_t gfp_mask,
156 						  unsigned int nr_vecs)
157 {
158 	struct bio_integrity_alloc *bia;
159 
160 	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
161 		return ERR_PTR(-EOPNOTSUPP);
162 
163 	bia = kmalloc_flex(*bia, bvecs, nr_vecs, gfp_mask);
164 	if (unlikely(!bia))
165 		return ERR_PTR(-ENOMEM);
166 	bio_integrity_init(bio, &bia->bip, bia->bvecs, nr_vecs);
167 	return &bia->bip;
168 }
169 EXPORT_SYMBOL(bio_integrity_alloc);
170 
171 static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs)
172 {
173 	int i;
174 
175 	for (i = 0; i < nr_vecs; i++)
176 		unpin_user_page(bv[i].bv_page);
177 }
178 
179 static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip)
180 {
181 	unsigned short orig_nr_vecs = bip->bip_max_vcnt - 1;
182 	struct bio_vec *orig_bvecs = &bip->bip_vec[1];
183 	struct bio_vec *bounce_bvec = &bip->bip_vec[0];
184 	size_t bytes = bounce_bvec->bv_len;
185 	struct iov_iter orig_iter;
186 	int ret;
187 
188 	iov_iter_bvec(&orig_iter, ITER_DEST, orig_bvecs, orig_nr_vecs, bytes);
189 	ret = copy_to_iter(bvec_virt(bounce_bvec), bytes, &orig_iter);
190 	WARN_ON_ONCE(ret != bytes);
191 
192 	bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs);
193 }
194 
195 /**
196  * bio_integrity_unmap_user - Unmap user integrity payload
197  * @bio:	bio containing bip to be unmapped
198  *
199  * Unmap the user mapped integrity portion of a bio.
200  */
201 void bio_integrity_unmap_user(struct bio *bio)
202 {
203 	struct bio_integrity_payload *bip = bio_integrity(bio);
204 
205 	if (bip->bip_flags & BIP_COPY_USER) {
206 		if (bio_data_dir(bio) == READ)
207 			bio_integrity_uncopy_user(bip);
208 		kfree(bvec_virt(bip->bip_vec));
209 		return;
210 	}
211 
212 	bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt);
213 }
214 
215 /**
216  * bio_integrity_add_page - Attach integrity metadata
217  * @bio:	bio to update
218  * @page:	page containing integrity metadata
219  * @len:	number of bytes of integrity metadata in page
220  * @offset:	start offset within page
221  *
222  * Description: Attach a page containing integrity metadata to bio.
223  */
224 int bio_integrity_add_page(struct bio *bio, struct page *page,
225 			   unsigned int len, unsigned int offset)
226 {
227 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
228 	struct bio_integrity_payload *bip = bio_integrity(bio);
229 
230 	if (bip->bip_vcnt > 0) {
231 		struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1];
232 
233 		if (!zone_device_pages_compatible(bv->bv_page, page))
234 			return 0;
235 		if (zone_device_pages_have_same_pgmap(bv->bv_page, page) &&
236 		    bvec_try_merge_hw_page(q, bv, page, len, offset)) {
237 			bip->bip_iter.bi_size += len;
238 			return len;
239 		}
240 
241 		if (bip->bip_vcnt >=
242 		    min(bip->bip_max_vcnt, queue_max_integrity_segments(q)))
243 			return 0;
244 
245 		/*
246 		 * If the queue doesn't support SG gaps and adding this segment
247 		 * would create a gap, disallow it.
248 		 */
249 		if (bvec_gap_to_prev(&q->limits, bv, offset))
250 			return 0;
251 	}
252 
253 	bvec_set_page(&bip->bip_vec[bip->bip_vcnt], page, len, offset);
254 	bip->bip_vcnt++;
255 	bip->bip_iter.bi_size += len;
256 
257 	return len;
258 }
259 EXPORT_SYMBOL(bio_integrity_add_page);
260 
261 static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
262 				   int nr_vecs, unsigned int len)
263 {
264 	bool write = op_is_write(bio_op(bio));
265 	struct bio_integrity_payload *bip;
266 	struct iov_iter iter;
267 	void *buf;
268 	int ret;
269 
270 	buf = kmalloc(len, GFP_KERNEL);
271 	if (!buf)
272 		return -ENOMEM;
273 
274 	if (write) {
275 		iov_iter_bvec(&iter, ITER_SOURCE, bvec, nr_vecs, len);
276 		if (!copy_from_iter_full(buf, len, &iter)) {
277 			ret = -EFAULT;
278 			goto free_buf;
279 		}
280 
281 		bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
282 	} else {
283 		memset(buf, 0, len);
284 
285 		/*
286 		 * We need to preserve the original bvec and the number of vecs
287 		 * in it for completion handling
288 		 */
289 		bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs + 1);
290 	}
291 
292 	if (IS_ERR(bip)) {
293 		ret = PTR_ERR(bip);
294 		goto free_buf;
295 	}
296 
297 	if (write)
298 		bio_integrity_unpin_bvec(bvec, nr_vecs);
299 	else
300 		memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
301 
302 	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
303 				     offset_in_page(buf));
304 	if (ret != len) {
305 		ret = -ENOMEM;
306 		goto free_bip;
307 	}
308 
309 	bip->bip_flags |= BIP_COPY_USER;
310 	return 0;
311 free_bip:
312 	bio_integrity_free(bio);
313 free_buf:
314 	kfree(buf);
315 	return ret;
316 }
317 
318 static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec,
319 				   int nr_vecs, unsigned int len)
320 {
321 	struct bio_integrity_payload *bip;
322 
323 	bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs);
324 	if (IS_ERR(bip))
325 		return PTR_ERR(bip);
326 
327 	memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec));
328 	bip->bip_iter.bi_size = len;
329 	bip->bip_vcnt = nr_vecs;
330 	return 0;
331 }
332 
333 static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
334 				    int nr_vecs, ssize_t bytes, ssize_t offset,
335 				    bool *is_p2p)
336 {
337 	unsigned int nr_bvecs = 0;
338 	int i, j;
339 
340 	for (i = 0; i < nr_vecs; i = j) {
341 		size_t size = min_t(size_t, bytes, PAGE_SIZE - offset);
342 		struct folio *folio = page_folio(pages[i]);
343 
344 		bytes -= size;
345 		for (j = i + 1; j < nr_vecs; j++) {
346 			size_t next = min_t(size_t, PAGE_SIZE, bytes);
347 
348 			if (page_folio(pages[j]) != folio ||
349 			    pages[j] != pages[j - 1] + 1)
350 				break;
351 			unpin_user_page(pages[j]);
352 			size += next;
353 			bytes -= next;
354 		}
355 
356 		if (is_pci_p2pdma_page(pages[i]))
357 			*is_p2p = true;
358 
359 		bvec_set_page(&bvec[nr_bvecs], pages[i], size, offset);
360 		offset = 0;
361 		nr_bvecs++;
362 	}
363 
364 	return nr_bvecs;
365 }
366 
367 int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
368 {
369 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
370 	struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages;
371 	struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec;
372 	iov_iter_extraction_t extraction_flags = 0;
373 	size_t offset, bytes = iter->count;
374 	bool copy, is_p2p = false;
375 	unsigned int nr_bvecs;
376 	int ret, nr_vecs;
377 
378 	if (bio_integrity(bio))
379 		return -EINVAL;
380 	if (bytes >> SECTOR_SHIFT > queue_max_hw_sectors(q))
381 		return -E2BIG;
382 
383 	nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS + 1);
384 	if (nr_vecs > BIO_MAX_VECS)
385 		return -E2BIG;
386 	if (nr_vecs > UIO_FASTIOV) {
387 		bvec = kzalloc_objs(*bvec, nr_vecs);
388 		if (!bvec)
389 			return -ENOMEM;
390 		pages = NULL;
391 	}
392 
393 	copy = iov_iter_alignment(iter) &
394 			blk_lim_dma_alignment_and_pad(&q->limits);
395 
396 	if (blk_queue_pci_p2pdma(q))
397 		extraction_flags |= ITER_ALLOW_P2PDMA;
398 
399 	ret = iov_iter_extract_pages(iter, &pages, bytes, nr_vecs,
400 					extraction_flags, &offset);
401 	if (unlikely(ret < 0))
402 		goto free_bvec;
403 
404 	/*
405 	 * Handle partial pinning. This can happen when pin_user_pages_fast()
406 	 * returns fewer pages than requested.
407 	 */
408 	if (user_backed_iter(iter) && unlikely(ret != bytes)) {
409 		if (ret > 0) {
410 			int npinned = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
411 			int i;
412 
413 			for (i = 0; i < npinned; i++)
414 				unpin_user_page(pages[i]);
415 		}
416 		if (pages != stack_pages)
417 			kvfree(pages);
418 		ret = -EFAULT;
419 		goto free_bvec;
420 	}
421 
422 	nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset,
423 				   &is_p2p);
424 	if (pages != stack_pages)
425 		kvfree(pages);
426 	if (nr_bvecs > queue_max_integrity_segments(q))
427 		copy = true;
428 	if (is_p2p)
429 		bio->bi_opf |= REQ_NOMERGE;
430 
431 	if (copy)
432 		ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes);
433 	else
434 		ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes);
435 	if (ret)
436 		goto release_pages;
437 	if (bvec != stack_vec)
438 		kfree(bvec);
439 
440 	return 0;
441 
442 release_pages:
443 	bio_integrity_unpin_bvec(bvec, nr_bvecs);
444 free_bvec:
445 	if (bvec != stack_vec)
446 		kfree(bvec);
447 	return ret;
448 }
449 
450 static void bio_uio_meta_to_bip(struct bio *bio, struct uio_meta *meta)
451 {
452 	struct bio_integrity_payload *bip = bio_integrity(bio);
453 
454 	if (meta->flags & IO_INTEGRITY_CHK_GUARD)
455 		bip->bip_flags |= BIP_CHECK_GUARD;
456 	if (meta->flags & IO_INTEGRITY_CHK_APPTAG)
457 		bip->bip_flags |= BIP_CHECK_APPTAG;
458 	if (meta->flags & IO_INTEGRITY_CHK_REFTAG)
459 		bip->bip_flags |= BIP_CHECK_REFTAG;
460 
461 	bip->app_tag = meta->app_tag;
462 }
463 
464 int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta)
465 {
466 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
467 	unsigned int integrity_bytes;
468 	int ret;
469 	struct iov_iter it;
470 
471 	if (!bi)
472 		return -EINVAL;
473 	/*
474 	 * original meta iterator can be bigger.
475 	 * process integrity info corresponding to current data buffer only.
476 	 */
477 	it = meta->iter;
478 	integrity_bytes = bio_integrity_bytes(bi, bio_sectors(bio));
479 	if (it.count < integrity_bytes)
480 		return -EINVAL;
481 
482 	/* should fit into two bytes */
483 	BUILD_BUG_ON(IO_INTEGRITY_VALID_FLAGS >= (1 << 16));
484 
485 	if (meta->flags && (meta->flags & ~IO_INTEGRITY_VALID_FLAGS))
486 		return -EINVAL;
487 
488 	it.count = integrity_bytes;
489 	ret = bio_integrity_map_user(bio, &it);
490 	if (!ret) {
491 		bio_uio_meta_to_bip(bio, meta);
492 		bip_set_seed(bio_integrity(bio), meta->seed);
493 		iov_iter_advance(&meta->iter, integrity_bytes);
494 		meta->seed += bio_integrity_intervals(bi, bio_sectors(bio));
495 	}
496 	return ret;
497 }
498 
499 /**
500  * bio_integrity_advance - Advance integrity vector
501  * @bio:	bio whose integrity vector to update
502  * @bytes_done:	number of data bytes that have been completed
503  *
504  * Description: This function calculates how many integrity bytes the
505  * number of completed data bytes correspond to and advances the
506  * integrity vector accordingly.
507  */
508 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
509 {
510 	struct bio_integrity_payload *bip = bio_integrity(bio);
511 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
512 	unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
513 
514 	bip->bip_iter.bi_sector += bio_integrity_intervals(bi, bytes_done >> 9);
515 	bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
516 }
517 
518 /**
519  * bio_integrity_trim - Trim integrity vector
520  * @bio:	bio whose integrity vector to update
521  *
522  * Description: Used to trim the integrity vector in a cloned bio.
523  */
524 void bio_integrity_trim(struct bio *bio)
525 {
526 	struct bio_integrity_payload *bip = bio_integrity(bio);
527 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
528 
529 	bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
530 }
531 EXPORT_SYMBOL(bio_integrity_trim);
532 
533 /**
534  * bio_integrity_clone - Callback for cloning bios with integrity metadata
535  * @bio:	New bio
536  * @bio_src:	Original bio
537  * @gfp_mask:	Memory allocation mask
538  *
539  * Description:	Called to allocate a bip when cloning a bio
540  */
541 int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
542 			gfp_t gfp_mask)
543 {
544 	struct bio_integrity_payload *bip_src = bio_integrity(bio_src);
545 	struct bio_integrity_payload *bip;
546 
547 	BUG_ON(bip_src == NULL);
548 
549 	bip = bio_integrity_alloc(bio, gfp_mask, 0);
550 	if (IS_ERR(bip))
551 		return PTR_ERR(bip);
552 
553 	bip->bip_vec = bip_src->bip_vec;
554 	bip->bip_iter = bip_src->bip_iter;
555 	bip->bip_flags = bip_src->bip_flags & BIP_CLONE_FLAGS;
556 	bip->app_tag = bip_src->app_tag;
557 
558 	return 0;
559 }
560 
561 static int __init bio_integrity_initfn(void)
562 {
563 	if (mempool_init_page_pool(&integrity_buf_pool, BIO_POOL_SIZE,
564 			get_order(BLK_INTEGRITY_MAX_SIZE)))
565 		panic("bio: can't create integrity buf pool\n");
566 	return 0;
567 }
568 subsys_initcall(bio_integrity_initfn);
569