xref: /linux/block/bio-integrity.c (revision 6e11664f148454a127dd89e8698c3e3e80e5f62f)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * bio-integrity.c - bio data integrity extensions
4  *
5  * Copyright (C) 2007, 2008, 2009 Oracle Corporation
6  * Written by: Martin K. Petersen <martin.petersen@oracle.com>
7  */
8 
9 #include <linux/blk-integrity.h>
10 #include "blk.h"
11 
12 struct bio_integrity_alloc {
13 	struct bio_integrity_payload	bip;
14 	struct bio_vec			bvecs[];
15 };
16 
17 /**
18  * bio_integrity_free - Free bio integrity payload
19  * @bio:	bio containing bip to be freed
20  *
21  * Description: Free the integrity portion of a bio.
22  */
bio_integrity_free(struct bio * bio)23 void bio_integrity_free(struct bio *bio)
24 {
25 	kfree(bio_integrity(bio));
26 	bio->bi_integrity = NULL;
27 	bio->bi_opf &= ~REQ_INTEGRITY;
28 }
29 
bio_integrity_init(struct bio * bio,struct bio_integrity_payload * bip,struct bio_vec * bvecs,unsigned int nr_vecs)30 void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
31 		struct bio_vec *bvecs, unsigned int nr_vecs)
32 {
33 	memset(bip, 0, sizeof(*bip));
34 	bip->bip_max_vcnt = nr_vecs;
35 	if (nr_vecs)
36 		bip->bip_vec = bvecs;
37 
38 	bio->bi_integrity = bip;
39 	bio->bi_opf |= REQ_INTEGRITY;
40 }
41 
42 /**
43  * bio_integrity_alloc - Allocate integrity payload and attach it to bio
44  * @bio:	bio to attach integrity metadata to
45  * @gfp_mask:	Memory allocation mask
46  * @nr_vecs:	Number of integrity metadata scatter-gather elements
47  *
48  * Description: This function prepares a bio for attaching integrity
49  * metadata.  nr_vecs specifies the maximum number of pages containing
50  * integrity metadata that can be attached.
51  */
bio_integrity_alloc(struct bio * bio,gfp_t gfp_mask,unsigned int nr_vecs)52 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
53 						  gfp_t gfp_mask,
54 						  unsigned int nr_vecs)
55 {
56 	struct bio_integrity_alloc *bia;
57 
58 	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
59 		return ERR_PTR(-EOPNOTSUPP);
60 
61 	bia = kmalloc(struct_size(bia, bvecs, nr_vecs), gfp_mask);
62 	if (unlikely(!bia))
63 		return ERR_PTR(-ENOMEM);
64 	bio_integrity_init(bio, &bia->bip, bia->bvecs, nr_vecs);
65 	return &bia->bip;
66 }
67 EXPORT_SYMBOL(bio_integrity_alloc);
68 
bio_integrity_unpin_bvec(struct bio_vec * bv,int nr_vecs)69 static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs)
70 {
71 	int i;
72 
73 	for (i = 0; i < nr_vecs; i++)
74 		unpin_user_page(bv[i].bv_page);
75 }
76 
bio_integrity_uncopy_user(struct bio_integrity_payload * bip)77 static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip)
78 {
79 	unsigned short orig_nr_vecs = bip->bip_max_vcnt - 1;
80 	struct bio_vec *orig_bvecs = &bip->bip_vec[1];
81 	struct bio_vec *bounce_bvec = &bip->bip_vec[0];
82 	size_t bytes = bounce_bvec->bv_len;
83 	struct iov_iter orig_iter;
84 	int ret;
85 
86 	iov_iter_bvec(&orig_iter, ITER_DEST, orig_bvecs, orig_nr_vecs, bytes);
87 	ret = copy_to_iter(bvec_virt(bounce_bvec), bytes, &orig_iter);
88 	WARN_ON_ONCE(ret != bytes);
89 
90 	bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs);
91 }
92 
93 /**
94  * bio_integrity_unmap_user - Unmap user integrity payload
95  * @bio:	bio containing bip to be unmapped
96  *
97  * Unmap the user mapped integrity portion of a bio.
98  */
bio_integrity_unmap_user(struct bio * bio)99 void bio_integrity_unmap_user(struct bio *bio)
100 {
101 	struct bio_integrity_payload *bip = bio_integrity(bio);
102 
103 	if (bip->bip_flags & BIP_COPY_USER) {
104 		if (bio_data_dir(bio) == READ)
105 			bio_integrity_uncopy_user(bip);
106 		kfree(bvec_virt(bip->bip_vec));
107 		return;
108 	}
109 
110 	bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt);
111 }
112 
113 /**
114  * bio_integrity_add_page - Attach integrity metadata
115  * @bio:	bio to update
116  * @page:	page containing integrity metadata
117  * @len:	number of bytes of integrity metadata in page
118  * @offset:	start offset within page
119  *
120  * Description: Attach a page containing integrity metadata to bio.
121  */
bio_integrity_add_page(struct bio * bio,struct page * page,unsigned int len,unsigned int offset)122 int bio_integrity_add_page(struct bio *bio, struct page *page,
123 			   unsigned int len, unsigned int offset)
124 {
125 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
126 	struct bio_integrity_payload *bip = bio_integrity(bio);
127 
128 	if (bip->bip_vcnt > 0) {
129 		struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1];
130 
131 		if (!zone_device_pages_have_same_pgmap(bv->bv_page, page))
132 			return 0;
133 
134 		if (bvec_try_merge_hw_page(q, bv, page, len, offset)) {
135 			bip->bip_iter.bi_size += len;
136 			return len;
137 		}
138 
139 		if (bip->bip_vcnt >=
140 		    min(bip->bip_max_vcnt, queue_max_integrity_segments(q)))
141 			return 0;
142 
143 		/*
144 		 * If the queue doesn't support SG gaps and adding this segment
145 		 * would create a gap, disallow it.
146 		 */
147 		if (bvec_gap_to_prev(&q->limits, bv, offset))
148 			return 0;
149 	}
150 
151 	bvec_set_page(&bip->bip_vec[bip->bip_vcnt], page, len, offset);
152 	bip->bip_vcnt++;
153 	bip->bip_iter.bi_size += len;
154 
155 	return len;
156 }
157 EXPORT_SYMBOL(bio_integrity_add_page);
158 
bio_integrity_copy_user(struct bio * bio,struct bio_vec * bvec,int nr_vecs,unsigned int len)159 static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
160 				   int nr_vecs, unsigned int len)
161 {
162 	bool write = op_is_write(bio_op(bio));
163 	struct bio_integrity_payload *bip;
164 	struct iov_iter iter;
165 	void *buf;
166 	int ret;
167 
168 	buf = kmalloc(len, GFP_KERNEL);
169 	if (!buf)
170 		return -ENOMEM;
171 
172 	if (write) {
173 		iov_iter_bvec(&iter, ITER_SOURCE, bvec, nr_vecs, len);
174 		if (!copy_from_iter_full(buf, len, &iter)) {
175 			ret = -EFAULT;
176 			goto free_buf;
177 		}
178 
179 		bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
180 	} else {
181 		memset(buf, 0, len);
182 
183 		/*
184 		 * We need to preserve the original bvec and the number of vecs
185 		 * in it for completion handling
186 		 */
187 		bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs + 1);
188 	}
189 
190 	if (IS_ERR(bip)) {
191 		ret = PTR_ERR(bip);
192 		goto free_buf;
193 	}
194 
195 	if (write)
196 		bio_integrity_unpin_bvec(bvec, nr_vecs);
197 	else
198 		memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
199 
200 	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
201 				     offset_in_page(buf));
202 	if (ret != len) {
203 		ret = -ENOMEM;
204 		goto free_bip;
205 	}
206 
207 	bip->bip_flags |= BIP_COPY_USER;
208 	bip->bip_vcnt = nr_vecs;
209 	return 0;
210 free_bip:
211 	bio_integrity_free(bio);
212 free_buf:
213 	kfree(buf);
214 	return ret;
215 }
216 
bio_integrity_init_user(struct bio * bio,struct bio_vec * bvec,int nr_vecs,unsigned int len)217 static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec,
218 				   int nr_vecs, unsigned int len)
219 {
220 	struct bio_integrity_payload *bip;
221 
222 	bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs);
223 	if (IS_ERR(bip))
224 		return PTR_ERR(bip);
225 
226 	memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec));
227 	bip->bip_iter.bi_size = len;
228 	bip->bip_vcnt = nr_vecs;
229 	return 0;
230 }
231 
bvec_from_pages(struct bio_vec * bvec,struct page ** pages,int nr_vecs,ssize_t bytes,ssize_t offset)232 static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
233 				    int nr_vecs, ssize_t bytes, ssize_t offset)
234 {
235 	unsigned int nr_bvecs = 0;
236 	int i, j;
237 
238 	for (i = 0; i < nr_vecs; i = j) {
239 		size_t size = min_t(size_t, bytes, PAGE_SIZE - offset);
240 		struct folio *folio = page_folio(pages[i]);
241 
242 		bytes -= size;
243 		for (j = i + 1; j < nr_vecs; j++) {
244 			size_t next = min_t(size_t, PAGE_SIZE, bytes);
245 
246 			if (page_folio(pages[j]) != folio ||
247 			    pages[j] != pages[j - 1] + 1)
248 				break;
249 			unpin_user_page(pages[j]);
250 			size += next;
251 			bytes -= next;
252 		}
253 
254 		bvec_set_page(&bvec[nr_bvecs], pages[i], size, offset);
255 		offset = 0;
256 		nr_bvecs++;
257 	}
258 
259 	return nr_bvecs;
260 }
261 
bio_integrity_map_user(struct bio * bio,struct iov_iter * iter)262 int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
263 {
264 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
265 	unsigned int align = blk_lim_dma_alignment_and_pad(&q->limits);
266 	struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages;
267 	struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec;
268 	size_t offset, bytes = iter->count;
269 	unsigned int nr_bvecs;
270 	int ret, nr_vecs;
271 	bool copy;
272 
273 	if (bio_integrity(bio))
274 		return -EINVAL;
275 	if (bytes >> SECTOR_SHIFT > queue_max_hw_sectors(q))
276 		return -E2BIG;
277 
278 	nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS + 1);
279 	if (nr_vecs > BIO_MAX_VECS)
280 		return -E2BIG;
281 	if (nr_vecs > UIO_FASTIOV) {
282 		bvec = kcalloc(nr_vecs, sizeof(*bvec), GFP_KERNEL);
283 		if (!bvec)
284 			return -ENOMEM;
285 		pages = NULL;
286 	}
287 
288 	copy = !iov_iter_is_aligned(iter, align, align);
289 	ret = iov_iter_extract_pages(iter, &pages, bytes, nr_vecs, 0, &offset);
290 	if (unlikely(ret < 0))
291 		goto free_bvec;
292 
293 	nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset);
294 	if (pages != stack_pages)
295 		kvfree(pages);
296 	if (nr_bvecs > queue_max_integrity_segments(q))
297 		copy = true;
298 
299 	if (copy)
300 		ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes);
301 	else
302 		ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes);
303 	if (ret)
304 		goto release_pages;
305 	if (bvec != stack_vec)
306 		kfree(bvec);
307 
308 	return 0;
309 
310 release_pages:
311 	bio_integrity_unpin_bvec(bvec, nr_bvecs);
312 free_bvec:
313 	if (bvec != stack_vec)
314 		kfree(bvec);
315 	return ret;
316 }
317 
bio_uio_meta_to_bip(struct bio * bio,struct uio_meta * meta)318 static void bio_uio_meta_to_bip(struct bio *bio, struct uio_meta *meta)
319 {
320 	struct bio_integrity_payload *bip = bio_integrity(bio);
321 
322 	if (meta->flags & IO_INTEGRITY_CHK_GUARD)
323 		bip->bip_flags |= BIP_CHECK_GUARD;
324 	if (meta->flags & IO_INTEGRITY_CHK_APPTAG)
325 		bip->bip_flags |= BIP_CHECK_APPTAG;
326 	if (meta->flags & IO_INTEGRITY_CHK_REFTAG)
327 		bip->bip_flags |= BIP_CHECK_REFTAG;
328 
329 	bip->app_tag = meta->app_tag;
330 }
331 
bio_integrity_map_iter(struct bio * bio,struct uio_meta * meta)332 int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta)
333 {
334 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
335 	unsigned int integrity_bytes;
336 	int ret;
337 	struct iov_iter it;
338 
339 	if (!bi)
340 		return -EINVAL;
341 	/*
342 	 * original meta iterator can be bigger.
343 	 * process integrity info corresponding to current data buffer only.
344 	 */
345 	it = meta->iter;
346 	integrity_bytes = bio_integrity_bytes(bi, bio_sectors(bio));
347 	if (it.count < integrity_bytes)
348 		return -EINVAL;
349 
350 	/* should fit into two bytes */
351 	BUILD_BUG_ON(IO_INTEGRITY_VALID_FLAGS >= (1 << 16));
352 
353 	if (meta->flags && (meta->flags & ~IO_INTEGRITY_VALID_FLAGS))
354 		return -EINVAL;
355 
356 	it.count = integrity_bytes;
357 	ret = bio_integrity_map_user(bio, &it);
358 	if (!ret) {
359 		bio_uio_meta_to_bip(bio, meta);
360 		bip_set_seed(bio_integrity(bio), meta->seed);
361 		iov_iter_advance(&meta->iter, integrity_bytes);
362 		meta->seed += bio_integrity_intervals(bi, bio_sectors(bio));
363 	}
364 	return ret;
365 }
366 
367 /**
368  * bio_integrity_advance - Advance integrity vector
369  * @bio:	bio whose integrity vector to update
370  * @bytes_done:	number of data bytes that have been completed
371  *
372  * Description: This function calculates how many integrity bytes the
373  * number of completed data bytes correspond to and advances the
374  * integrity vector accordingly.
375  */
bio_integrity_advance(struct bio * bio,unsigned int bytes_done)376 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
377 {
378 	struct bio_integrity_payload *bip = bio_integrity(bio);
379 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
380 	unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
381 
382 	bip->bip_iter.bi_sector += bio_integrity_intervals(bi, bytes_done >> 9);
383 	bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
384 }
385 
386 /**
387  * bio_integrity_trim - Trim integrity vector
388  * @bio:	bio whose integrity vector to update
389  *
390  * Description: Used to trim the integrity vector in a cloned bio.
391  */
bio_integrity_trim(struct bio * bio)392 void bio_integrity_trim(struct bio *bio)
393 {
394 	struct bio_integrity_payload *bip = bio_integrity(bio);
395 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
396 
397 	bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
398 }
399 EXPORT_SYMBOL(bio_integrity_trim);
400 
401 /**
402  * bio_integrity_clone - Callback for cloning bios with integrity metadata
403  * @bio:	New bio
404  * @bio_src:	Original bio
405  * @gfp_mask:	Memory allocation mask
406  *
407  * Description:	Called to allocate a bip when cloning a bio
408  */
bio_integrity_clone(struct bio * bio,struct bio * bio_src,gfp_t gfp_mask)409 int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
410 			gfp_t gfp_mask)
411 {
412 	struct bio_integrity_payload *bip_src = bio_integrity(bio_src);
413 	struct bio_integrity_payload *bip;
414 
415 	BUG_ON(bip_src == NULL);
416 
417 	bip = bio_integrity_alloc(bio, gfp_mask, 0);
418 	if (IS_ERR(bip))
419 		return PTR_ERR(bip);
420 
421 	bip->bip_vec = bip_src->bip_vec;
422 	bip->bip_iter = bip_src->bip_iter;
423 	bip->bip_flags = bip_src->bip_flags & BIP_CLONE_FLAGS;
424 	bip->app_tag = bip_src->app_tag;
425 
426 	return 0;
427 }
428