1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
24 * Copyright (c) 2019 by Delphix. All rights reserved.
25 * Copyright (c) 2023, 2024, Klara Inc.
26 */
27
28 /*
29 * See abd.c for a general overview of the arc buffered data (ABD).
30 *
31 * Linear buffers act exactly like normal buffers and are always mapped into the
32 * kernel's virtual memory space, while scattered ABD data chunks are allocated
33 * as physical pages and then mapped in only while they are actually being
34 * accessed through one of the abd_* library functions. Using scattered ABDs
35 * provides several benefits:
36 *
37 * (1) They avoid use of kmem_*, preventing performance problems where running
38 * kmem_reap on very large memory systems never finishes and causes
39 * constant TLB shootdowns.
40 *
41 * (2) Fragmentation is less of an issue since when we are at the limit of
42 * allocatable space, we won't have to search around for a long free
43 * hole in the VA space for large ARC allocations. Each chunk is mapped in
44 * individually, so even if we are using HIGHMEM (see next point) we
45 * wouldn't need to worry about finding a contiguous address range.
46 *
47 * (3) If we are not using HIGHMEM, then all physical memory is always
48 * mapped into the kernel's address space, so we also avoid the map /
49 * unmap costs on each ABD access.
50 *
51 * If we are not using HIGHMEM, scattered buffers which have only one chunk
52 * can be treated as linear buffers, because they are contiguous in the
53 * kernel's virtual address space. See abd_alloc_chunks() for details.
54 */
55
56 #include <sys/abd_impl.h>
57 #include <sys/param.h>
58 #include <sys/zio.h>
59 #include <sys/arc.h>
60 #include <sys/zfs_context.h>
61 #include <sys/zfs_znode.h>
62 #include <linux/kmap_compat.h>
63 #include <linux/mm_compat.h>
64 #include <linux/scatterlist.h>
65 #include <linux/version.h>
66
67 #if defined(MAX_ORDER)
68 #define ABD_MAX_ORDER (MAX_ORDER)
69 #elif defined(MAX_PAGE_ORDER)
70 #define ABD_MAX_ORDER (MAX_PAGE_ORDER)
71 #endif
72
73 typedef struct abd_stats {
74 kstat_named_t abdstat_struct_size;
75 kstat_named_t abdstat_linear_cnt;
76 kstat_named_t abdstat_linear_data_size;
77 kstat_named_t abdstat_scatter_cnt;
78 kstat_named_t abdstat_scatter_data_size;
79 kstat_named_t abdstat_scatter_chunk_waste;
80 kstat_named_t abdstat_scatter_orders[ABD_MAX_ORDER];
81 kstat_named_t abdstat_scatter_page_multi_chunk;
82 kstat_named_t abdstat_scatter_page_multi_zone;
83 kstat_named_t abdstat_scatter_page_alloc_retry;
84 kstat_named_t abdstat_scatter_sg_table_retry;
85 } abd_stats_t;
86
87 static abd_stats_t abd_stats = {
88 /* Amount of memory occupied by all of the abd_t struct allocations */
89 { "struct_size", KSTAT_DATA_UINT64 },
90 /*
91 * The number of linear ABDs which are currently allocated, excluding
92 * ABDs which don't own their data (for instance the ones which were
93 * allocated through abd_get_offset() and abd_get_from_buf()). If an
94 * ABD takes ownership of its buf then it will become tracked.
95 */
96 { "linear_cnt", KSTAT_DATA_UINT64 },
97 /* Amount of data stored in all linear ABDs tracked by linear_cnt */
98 { "linear_data_size", KSTAT_DATA_UINT64 },
99 /*
100 * The number of scatter ABDs which are currently allocated, excluding
101 * ABDs which don't own their data (for instance the ones which were
102 * allocated through abd_get_offset()).
103 */
104 { "scatter_cnt", KSTAT_DATA_UINT64 },
105 /* Amount of data stored in all scatter ABDs tracked by scatter_cnt */
106 { "scatter_data_size", KSTAT_DATA_UINT64 },
107 /*
108 * The amount of space wasted at the end of the last chunk across all
109 * scatter ABDs tracked by scatter_cnt.
110 */
111 { "scatter_chunk_waste", KSTAT_DATA_UINT64 },
112 /*
113 * The number of compound allocations of a given order. These
114 * allocations are spread over all currently allocated ABDs, and
115 * act as a measure of memory fragmentation.
116 */
117 { { "scatter_order_N", KSTAT_DATA_UINT64 } },
118 /*
119 * The number of scatter ABDs which contain multiple chunks.
120 * ABDs are preferentially allocated from the minimum number of
121 * contiguous multi-page chunks, a single chunk is optimal.
122 */
123 { "scatter_page_multi_chunk", KSTAT_DATA_UINT64 },
124 /*
125 * The number of scatter ABDs which are split across memory zones.
126 * ABDs are preferentially allocated using pages from a single zone.
127 */
128 { "scatter_page_multi_zone", KSTAT_DATA_UINT64 },
129 /*
130 * The total number of retries encountered when attempting to
131 * allocate the pages to populate the scatter ABD.
132 */
133 { "scatter_page_alloc_retry", KSTAT_DATA_UINT64 },
134 /*
135 * The total number of retries encountered when attempting to
136 * allocate the sg table for an ABD.
137 */
138 { "scatter_sg_table_retry", KSTAT_DATA_UINT64 },
139 };
140
141 static struct {
142 wmsum_t abdstat_struct_size;
143 wmsum_t abdstat_linear_cnt;
144 wmsum_t abdstat_linear_data_size;
145 wmsum_t abdstat_scatter_cnt;
146 wmsum_t abdstat_scatter_data_size;
147 wmsum_t abdstat_scatter_chunk_waste;
148 wmsum_t abdstat_scatter_orders[ABD_MAX_ORDER];
149 wmsum_t abdstat_scatter_page_multi_chunk;
150 wmsum_t abdstat_scatter_page_multi_zone;
151 wmsum_t abdstat_scatter_page_alloc_retry;
152 wmsum_t abdstat_scatter_sg_table_retry;
153 } abd_sums;
154
155 #define abd_for_each_sg(abd, sg, n, i) \
156 for_each_sg(ABD_SCATTER(abd).abd_sgl, sg, n, i)
157
158 /*
159 * zfs_abd_scatter_min_size is the minimum allocation size to use scatter
160 * ABD's. Smaller allocations will use linear ABD's which uses
161 * zio_[data_]buf_alloc().
162 *
163 * Scatter ABD's use at least one page each, so sub-page allocations waste
164 * some space when allocated as scatter (e.g. 2KB scatter allocation wastes
165 * half of each page). Using linear ABD's for small allocations means that
166 * they will be put on slabs which contain many allocations. This can
167 * improve memory efficiency, but it also makes it much harder for ARC
168 * evictions to actually free pages, because all the buffers on one slab need
169 * to be freed in order for the slab (and underlying pages) to be freed.
170 * Typically, 512B and 1KB kmem caches have 16 buffers per slab, so it's
171 * possible for them to actually waste more memory than scatter (one page per
172 * buf = wasting 3/4 or 7/8th; one buf per slab = wasting 15/16th).
173 *
174 * Spill blocks are typically 512B and are heavily used on systems running
175 * selinux with the default dnode size and the `xattr=sa` property set.
176 *
177 * By default we use linear allocations for 512B and 1KB, and scatter
178 * allocations for larger (1.5KB and up).
179 */
180 static int zfs_abd_scatter_min_size = 512 * 3;
181
182 /*
183 * We use a scattered SPA_MAXBLOCKSIZE sized ABD whose pages are
184 * just a single zero'd page. This allows us to conserve memory by
185 * only using a single zero page for the scatterlist.
186 */
187 abd_t *abd_zero_scatter = NULL;
188
189 struct page;
190
191 /*
192 * abd_zero_page is assigned to each of the pages of abd_zero_scatter. It will
193 * point to ZERO_PAGE if it is available or it will be an allocated zero'd
194 * PAGESIZE buffer.
195 */
196 static struct page *abd_zero_page = NULL;
197
198 static kmem_cache_t *abd_cache = NULL;
199 static kstat_t *abd_ksp;
200
201 static uint_t
abd_chunkcnt_for_bytes(size_t size)202 abd_chunkcnt_for_bytes(size_t size)
203 {
204 return (P2ROUNDUP(size, PAGESIZE) / PAGESIZE);
205 }
206
207 abd_t *
abd_alloc_struct_impl(size_t size)208 abd_alloc_struct_impl(size_t size)
209 {
210 /*
211 * In Linux we do not use the size passed in during ABD
212 * allocation, so we just ignore it.
213 */
214 (void) size;
215 abd_t *abd = kmem_cache_alloc(abd_cache, KM_PUSHPAGE);
216 ASSERT3P(abd, !=, NULL);
217 ABDSTAT_INCR(abdstat_struct_size, sizeof (abd_t));
218
219 return (abd);
220 }
221
222 void
abd_free_struct_impl(abd_t * abd)223 abd_free_struct_impl(abd_t *abd)
224 {
225 kmem_cache_free(abd_cache, abd);
226 ABDSTAT_INCR(abdstat_struct_size, -(int)sizeof (abd_t));
227 }
228
229 static unsigned zfs_abd_scatter_max_order = ABD_MAX_ORDER - 1;
230
231 /*
232 * Mark zfs data pages so they can be excluded from kernel crash dumps
233 */
234 #ifdef _LP64
235 #define ABD_FILE_CACHE_PAGE 0x2F5ABDF11ECAC4E
236
237 static inline void
abd_mark_zfs_page(struct page * page)238 abd_mark_zfs_page(struct page *page)
239 {
240 get_page(page);
241 SetPagePrivate(page);
242 set_page_private(page, ABD_FILE_CACHE_PAGE);
243 }
244
245 static inline void
abd_unmark_zfs_page(struct page * page)246 abd_unmark_zfs_page(struct page *page)
247 {
248 set_page_private(page, 0UL);
249 ClearPagePrivate(page);
250 put_page(page);
251 }
252 #else
253 #define abd_mark_zfs_page(page)
254 #define abd_unmark_zfs_page(page)
255 #endif /* _LP64 */
256
257 #ifndef CONFIG_HIGHMEM
258
259 /*
260 * The goal is to minimize fragmentation by preferentially populating ABDs
261 * with higher order compound pages from a single zone. Allocation size is
262 * progressively decreased until it can be satisfied without performing
263 * reclaim or compaction. When necessary this function will degenerate to
264 * allocating individual pages and allowing reclaim to satisfy allocations.
265 */
266 void
abd_alloc_chunks(abd_t * abd,size_t size)267 abd_alloc_chunks(abd_t *abd, size_t size)
268 {
269 struct list_head pages;
270 struct sg_table table;
271 struct scatterlist *sg;
272 struct page *page, *tmp_page = NULL;
273 gfp_t gfp = __GFP_RECLAIMABLE | __GFP_NOWARN | GFP_NOIO;
274 gfp_t gfp_comp = (gfp | __GFP_NORETRY | __GFP_COMP) & ~__GFP_RECLAIM;
275 unsigned int max_order = MIN(zfs_abd_scatter_max_order,
276 ABD_MAX_ORDER - 1);
277 unsigned int nr_pages = abd_chunkcnt_for_bytes(size);
278 unsigned int chunks = 0, zones = 0;
279 size_t remaining_size;
280 int nid = NUMA_NO_NODE;
281 unsigned int alloc_pages = 0;
282
283 INIT_LIST_HEAD(&pages);
284
285 ASSERT3U(alloc_pages, <, nr_pages);
286
287 while (alloc_pages < nr_pages) {
288 unsigned int chunk_pages;
289 unsigned int order;
290
291 order = MIN(highbit64(nr_pages - alloc_pages) - 1, max_order);
292 chunk_pages = (1U << order);
293
294 page = alloc_pages_node(nid, order ? gfp_comp : gfp, order);
295 if (page == NULL) {
296 if (order == 0) {
297 ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
298 schedule_timeout_interruptible(1);
299 } else {
300 max_order = MAX(0, order - 1);
301 }
302 continue;
303 }
304
305 list_add_tail(&page->lru, &pages);
306
307 if ((nid != NUMA_NO_NODE) && (page_to_nid(page) != nid))
308 zones++;
309
310 nid = page_to_nid(page);
311 ABDSTAT_BUMP(abdstat_scatter_orders[order]);
312 chunks++;
313 alloc_pages += chunk_pages;
314 }
315
316 ASSERT3S(alloc_pages, ==, nr_pages);
317
318 while (sg_alloc_table(&table, chunks, gfp)) {
319 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
320 schedule_timeout_interruptible(1);
321 }
322
323 sg = table.sgl;
324 remaining_size = size;
325 list_for_each_entry_safe(page, tmp_page, &pages, lru) {
326 size_t sg_size = MIN(PAGESIZE << compound_order(page),
327 remaining_size);
328 sg_set_page(sg, page, sg_size, 0);
329 abd_mark_zfs_page(page);
330 remaining_size -= sg_size;
331
332 sg = sg_next(sg);
333 list_del(&page->lru);
334 }
335
336 /*
337 * These conditions ensure that a possible transformation to a linear
338 * ABD would be valid.
339 */
340 ASSERT(!PageHighMem(sg_page(table.sgl)));
341 ASSERT0(ABD_SCATTER(abd).abd_offset);
342
343 if (table.nents == 1) {
344 /*
345 * Since there is only one entry, this ABD can be represented
346 * as a linear buffer. All single-page (4K) ABD's can be
347 * represented this way. Some multi-page ABD's can also be
348 * represented this way, if we were able to allocate a single
349 * "chunk" (higher-order "page" which represents a power-of-2
350 * series of physically-contiguous pages). This is often the
351 * case for 2-page (8K) ABD's.
352 *
353 * Representing a single-entry scatter ABD as a linear ABD
354 * has the performance advantage of avoiding the copy (and
355 * allocation) in abd_borrow_buf_copy / abd_return_buf_copy.
356 * A performance increase of around 5% has been observed for
357 * ARC-cached reads (of small blocks which can take advantage
358 * of this).
359 *
360 * Note that this optimization is only possible because the
361 * pages are always mapped into the kernel's address space.
362 * This is not the case for highmem pages, so the
363 * optimization can not be made there.
364 */
365 abd->abd_flags |= ABD_FLAG_LINEAR;
366 abd->abd_flags |= ABD_FLAG_LINEAR_PAGE;
367 abd->abd_u.abd_linear.abd_sgl = table.sgl;
368 ABD_LINEAR_BUF(abd) = page_address(sg_page(table.sgl));
369 } else if (table.nents > 1) {
370 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
371 abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
372
373 if (zones) {
374 ABDSTAT_BUMP(abdstat_scatter_page_multi_zone);
375 abd->abd_flags |= ABD_FLAG_MULTI_ZONE;
376 }
377
378 ABD_SCATTER(abd).abd_sgl = table.sgl;
379 ABD_SCATTER(abd).abd_nents = table.nents;
380 }
381 }
382 #else
383
384 /*
385 * Allocate N individual pages to construct a scatter ABD. This function
386 * makes no attempt to request contiguous pages and requires the minimal
387 * number of kernel interfaces. It's designed for maximum compatibility.
388 */
389 void
abd_alloc_chunks(abd_t * abd,size_t size)390 abd_alloc_chunks(abd_t *abd, size_t size)
391 {
392 struct scatterlist *sg = NULL;
393 struct sg_table table;
394 struct page *page;
395 gfp_t gfp = __GFP_RECLAIMABLE | __GFP_NOWARN | GFP_NOIO;
396 int nr_pages = abd_chunkcnt_for_bytes(size);
397 int i = 0;
398
399 while (sg_alloc_table(&table, nr_pages, gfp)) {
400 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
401 schedule_timeout_interruptible(1);
402 }
403
404 ASSERT3U(table.nents, ==, nr_pages);
405 ABD_SCATTER(abd).abd_sgl = table.sgl;
406 ABD_SCATTER(abd).abd_nents = nr_pages;
407
408 abd_for_each_sg(abd, sg, nr_pages, i) {
409 while ((page = __page_cache_alloc(gfp)) == NULL) {
410 ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
411 schedule_timeout_interruptible(1);
412 }
413
414 ABDSTAT_BUMP(abdstat_scatter_orders[0]);
415 sg_set_page(sg, page, PAGESIZE, 0);
416 abd_mark_zfs_page(page);
417 }
418
419 if (nr_pages > 1) {
420 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
421 abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
422 }
423 }
424 #endif /* !CONFIG_HIGHMEM */
425
426 /*
427 * This must be called if any of the sg_table allocation functions
428 * are called.
429 */
430 static void
abd_free_sg_table(abd_t * abd)431 abd_free_sg_table(abd_t *abd)
432 {
433 struct sg_table table;
434
435 table.sgl = ABD_SCATTER(abd).abd_sgl;
436 table.nents = table.orig_nents = ABD_SCATTER(abd).abd_nents;
437 sg_free_table(&table);
438 }
439
440 void
abd_free_chunks(abd_t * abd)441 abd_free_chunks(abd_t *abd)
442 {
443 struct scatterlist *sg = NULL;
444 struct page *page;
445 int nr_pages = ABD_SCATTER(abd).abd_nents;
446 int order, i = 0;
447
448 if (abd->abd_flags & ABD_FLAG_MULTI_ZONE)
449 ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_zone);
450
451 if (abd->abd_flags & ABD_FLAG_MULTI_CHUNK)
452 ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_chunk);
453
454 /*
455 * Scatter ABDs may be constructed by abd_alloc_from_pages() from
456 * an array of pages. In which case they should not be freed.
457 */
458 if (!abd_is_from_pages(abd)) {
459 abd_for_each_sg(abd, sg, nr_pages, i) {
460 page = sg_page(sg);
461 abd_unmark_zfs_page(page);
462 order = compound_order(page);
463 __free_pages(page, order);
464 ASSERT3U(sg->length, <=, PAGE_SIZE << order);
465 ABDSTAT_BUMPDOWN(abdstat_scatter_orders[order]);
466 }
467 }
468
469 abd_free_sg_table(abd);
470 }
471
472 /*
473 * Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where each page in
474 * the scatterlist will be set to the zero'd out buffer abd_zero_page.
475 */
476 static void
abd_alloc_zero_scatter(void)477 abd_alloc_zero_scatter(void)
478 {
479 struct scatterlist *sg = NULL;
480 struct sg_table table;
481 gfp_t gfp = __GFP_NOWARN | GFP_NOIO;
482 int nr_pages = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
483 int i = 0;
484
485 #if defined(HAVE_ZERO_PAGE_GPL_ONLY)
486 gfp_t gfp_zero_page = gfp | __GFP_ZERO;
487 while ((abd_zero_page = __page_cache_alloc(gfp_zero_page)) == NULL) {
488 ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
489 schedule_timeout_interruptible(1);
490 }
491 abd_mark_zfs_page(abd_zero_page);
492 #else
493 abd_zero_page = ZERO_PAGE(0);
494 #endif /* HAVE_ZERO_PAGE_GPL_ONLY */
495
496 while (sg_alloc_table(&table, nr_pages, gfp)) {
497 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
498 schedule_timeout_interruptible(1);
499 }
500 ASSERT3U(table.nents, ==, nr_pages);
501
502 abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
503 abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER;
504 ABD_SCATTER(abd_zero_scatter).abd_offset = 0;
505 ABD_SCATTER(abd_zero_scatter).abd_sgl = table.sgl;
506 ABD_SCATTER(abd_zero_scatter).abd_nents = nr_pages;
507 abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;
508 abd_zero_scatter->abd_flags |= ABD_FLAG_MULTI_CHUNK;
509
510 abd_for_each_sg(abd_zero_scatter, sg, nr_pages, i) {
511 sg_set_page(sg, abd_zero_page, PAGESIZE, 0);
512 }
513
514 ABDSTAT_BUMP(abdstat_scatter_cnt);
515 ABDSTAT_INCR(abdstat_scatter_data_size, PAGESIZE);
516 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
517 }
518
519 boolean_t
abd_size_alloc_linear(size_t size)520 abd_size_alloc_linear(size_t size)
521 {
522 return (!zfs_abd_scatter_enabled || size < zfs_abd_scatter_min_size);
523 }
524
525 void
abd_update_scatter_stats(abd_t * abd,abd_stats_op_t op)526 abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op)
527 {
528 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
529 int waste = P2ROUNDUP(abd->abd_size, PAGESIZE) - abd->abd_size;
530 if (op == ABDSTAT_INCR) {
531 ABDSTAT_BUMP(abdstat_scatter_cnt);
532 ABDSTAT_INCR(abdstat_scatter_data_size, abd->abd_size);
533 ABDSTAT_INCR(abdstat_scatter_chunk_waste, waste);
534 arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE);
535 } else {
536 ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
537 ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size);
538 ABDSTAT_INCR(abdstat_scatter_chunk_waste, -waste);
539 arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE);
540 }
541 }
542
543 void
abd_update_linear_stats(abd_t * abd,abd_stats_op_t op)544 abd_update_linear_stats(abd_t *abd, abd_stats_op_t op)
545 {
546 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
547 if (op == ABDSTAT_INCR) {
548 ABDSTAT_BUMP(abdstat_linear_cnt);
549 ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size);
550 } else {
551 ABDSTAT_BUMPDOWN(abdstat_linear_cnt);
552 ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
553 }
554 }
555
556 void
abd_verify_scatter(abd_t * abd)557 abd_verify_scatter(abd_t *abd)
558 {
559 ASSERT3U(ABD_SCATTER(abd).abd_nents, >, 0);
560 ASSERT3U(ABD_SCATTER(abd).abd_offset, <,
561 ABD_SCATTER(abd).abd_sgl->length);
562
563 #ifdef ZFS_DEBUG
564 struct scatterlist *sg = NULL;
565 size_t n = ABD_SCATTER(abd).abd_nents;
566 int i = 0;
567
568 abd_for_each_sg(abd, sg, n, i) {
569 ASSERT3P(sg_page(sg), !=, NULL);
570 }
571 #endif
572 }
573
574 static void
abd_free_zero_scatter(void)575 abd_free_zero_scatter(void)
576 {
577 ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
578 ABDSTAT_INCR(abdstat_scatter_data_size, -(int)PAGESIZE);
579 ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_chunk);
580
581 abd_free_sg_table(abd_zero_scatter);
582 abd_free_struct(abd_zero_scatter);
583 abd_zero_scatter = NULL;
584 ASSERT3P(abd_zero_page, !=, NULL);
585 #if defined(HAVE_ZERO_PAGE_GPL_ONLY)
586 abd_unmark_zfs_page(abd_zero_page);
587 __free_page(abd_zero_page);
588 #endif /* HAVE_ZERO_PAGE_GPL_ONLY */
589 }
590
591 static int
abd_kstats_update(kstat_t * ksp,int rw)592 abd_kstats_update(kstat_t *ksp, int rw)
593 {
594 abd_stats_t *as = ksp->ks_data;
595
596 if (rw == KSTAT_WRITE)
597 return (EACCES);
598 as->abdstat_struct_size.value.ui64 =
599 wmsum_value(&abd_sums.abdstat_struct_size);
600 as->abdstat_linear_cnt.value.ui64 =
601 wmsum_value(&abd_sums.abdstat_linear_cnt);
602 as->abdstat_linear_data_size.value.ui64 =
603 wmsum_value(&abd_sums.abdstat_linear_data_size);
604 as->abdstat_scatter_cnt.value.ui64 =
605 wmsum_value(&abd_sums.abdstat_scatter_cnt);
606 as->abdstat_scatter_data_size.value.ui64 =
607 wmsum_value(&abd_sums.abdstat_scatter_data_size);
608 as->abdstat_scatter_chunk_waste.value.ui64 =
609 wmsum_value(&abd_sums.abdstat_scatter_chunk_waste);
610 for (int i = 0; i < ABD_MAX_ORDER; i++) {
611 as->abdstat_scatter_orders[i].value.ui64 =
612 wmsum_value(&abd_sums.abdstat_scatter_orders[i]);
613 }
614 as->abdstat_scatter_page_multi_chunk.value.ui64 =
615 wmsum_value(&abd_sums.abdstat_scatter_page_multi_chunk);
616 as->abdstat_scatter_page_multi_zone.value.ui64 =
617 wmsum_value(&abd_sums.abdstat_scatter_page_multi_zone);
618 as->abdstat_scatter_page_alloc_retry.value.ui64 =
619 wmsum_value(&abd_sums.abdstat_scatter_page_alloc_retry);
620 as->abdstat_scatter_sg_table_retry.value.ui64 =
621 wmsum_value(&abd_sums.abdstat_scatter_sg_table_retry);
622 return (0);
623 }
624
625 void
abd_init(void)626 abd_init(void)
627 {
628 int i;
629
630 abd_cache = kmem_cache_create("abd_t", sizeof (abd_t),
631 0, NULL, NULL, NULL, NULL, NULL, KMC_RECLAIMABLE);
632
633 wmsum_init(&abd_sums.abdstat_struct_size, 0);
634 wmsum_init(&abd_sums.abdstat_linear_cnt, 0);
635 wmsum_init(&abd_sums.abdstat_linear_data_size, 0);
636 wmsum_init(&abd_sums.abdstat_scatter_cnt, 0);
637 wmsum_init(&abd_sums.abdstat_scatter_data_size, 0);
638 wmsum_init(&abd_sums.abdstat_scatter_chunk_waste, 0);
639 for (i = 0; i < ABD_MAX_ORDER; i++)
640 wmsum_init(&abd_sums.abdstat_scatter_orders[i], 0);
641 wmsum_init(&abd_sums.abdstat_scatter_page_multi_chunk, 0);
642 wmsum_init(&abd_sums.abdstat_scatter_page_multi_zone, 0);
643 wmsum_init(&abd_sums.abdstat_scatter_page_alloc_retry, 0);
644 wmsum_init(&abd_sums.abdstat_scatter_sg_table_retry, 0);
645
646 abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
647 sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
648 if (abd_ksp != NULL) {
649 for (i = 0; i < ABD_MAX_ORDER; i++) {
650 snprintf(abd_stats.abdstat_scatter_orders[i].name,
651 KSTAT_STRLEN, "scatter_order_%d", i);
652 abd_stats.abdstat_scatter_orders[i].data_type =
653 KSTAT_DATA_UINT64;
654 }
655 abd_ksp->ks_data = &abd_stats;
656 abd_ksp->ks_update = abd_kstats_update;
657 kstat_install(abd_ksp);
658 }
659
660 abd_alloc_zero_scatter();
661 }
662
663 void
abd_fini(void)664 abd_fini(void)
665 {
666 abd_free_zero_scatter();
667
668 if (abd_ksp != NULL) {
669 kstat_delete(abd_ksp);
670 abd_ksp = NULL;
671 }
672
673 wmsum_fini(&abd_sums.abdstat_struct_size);
674 wmsum_fini(&abd_sums.abdstat_linear_cnt);
675 wmsum_fini(&abd_sums.abdstat_linear_data_size);
676 wmsum_fini(&abd_sums.abdstat_scatter_cnt);
677 wmsum_fini(&abd_sums.abdstat_scatter_data_size);
678 wmsum_fini(&abd_sums.abdstat_scatter_chunk_waste);
679 for (int i = 0; i < ABD_MAX_ORDER; i++)
680 wmsum_fini(&abd_sums.abdstat_scatter_orders[i]);
681 wmsum_fini(&abd_sums.abdstat_scatter_page_multi_chunk);
682 wmsum_fini(&abd_sums.abdstat_scatter_page_multi_zone);
683 wmsum_fini(&abd_sums.abdstat_scatter_page_alloc_retry);
684 wmsum_fini(&abd_sums.abdstat_scatter_sg_table_retry);
685
686 if (abd_cache) {
687 kmem_cache_destroy(abd_cache);
688 abd_cache = NULL;
689 }
690 }
691
692 void
abd_free_linear_page(abd_t * abd)693 abd_free_linear_page(abd_t *abd)
694 {
695 /* Transform it back into a scatter ABD for freeing */
696 struct scatterlist *sg = abd->abd_u.abd_linear.abd_sgl;
697
698 /* When backed by user page unmap it */
699 if (abd_is_from_pages(abd))
700 zfs_kunmap(sg_page(sg));
701 else
702 abd_update_scatter_stats(abd, ABDSTAT_DECR);
703
704 abd->abd_flags &= ~ABD_FLAG_LINEAR;
705 abd->abd_flags &= ~ABD_FLAG_LINEAR_PAGE;
706 ABD_SCATTER(abd).abd_nents = 1;
707 ABD_SCATTER(abd).abd_offset = 0;
708 ABD_SCATTER(abd).abd_sgl = sg;
709 abd_free_chunks(abd);
710 }
711
712 /*
713 * Allocate a scatter ABD structure from user pages. The pages must be
714 * pinned with get_user_pages, or similiar, but need not be mapped via
715 * the kmap interfaces.
716 */
717 abd_t *
abd_alloc_from_pages(struct page ** pages,unsigned long offset,uint64_t size)718 abd_alloc_from_pages(struct page **pages, unsigned long offset, uint64_t size)
719 {
720 uint_t npages = DIV_ROUND_UP(size, PAGE_SIZE);
721 struct sg_table table;
722
723 VERIFY3U(size, <=, DMU_MAX_ACCESS);
724 ASSERT3U(offset, <, PAGE_SIZE);
725 ASSERT3P(pages, !=, NULL);
726
727 /*
728 * Even if this buf is filesystem metadata, we only track that we
729 * own the underlying data buffer, which is not true in this case.
730 * Therefore, we don't ever use ABD_FLAG_META here.
731 */
732 abd_t *abd = abd_alloc_struct(0);
733 abd->abd_flags |= ABD_FLAG_FROM_PAGES | ABD_FLAG_OWNER;
734 abd->abd_size = size;
735
736 while (sg_alloc_table_from_pages(&table, pages, npages, offset,
737 size, __GFP_NOWARN | GFP_NOIO) != 0) {
738 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
739 schedule_timeout_interruptible(1);
740 }
741
742 if ((offset + size) <= PAGE_SIZE) {
743 /*
744 * Since there is only one entry, this ABD can be represented
745 * as a linear buffer. All single-page (4K) ABD's constructed
746 * from a user page can be represented this way as long as the
747 * page is mapped to a virtual address. This allows us to
748 * apply an offset in to the mapped page.
749 *
750 * Note that kmap() must be used, not kmap_atomic(), because
751 * the mapping needs to bet set up on all CPUs. Using kmap()
752 * also enables the user of highmem pages when required.
753 */
754 abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_LINEAR_PAGE;
755 abd->abd_u.abd_linear.abd_sgl = table.sgl;
756 zfs_kmap(sg_page(table.sgl));
757 ABD_LINEAR_BUF(abd) = sg_virt(table.sgl);
758 } else {
759 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
760 abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
761
762 ABD_SCATTER(abd).abd_offset = offset;
763 ABD_SCATTER(abd).abd_sgl = table.sgl;
764 ABD_SCATTER(abd).abd_nents = table.nents;
765
766 ASSERT0(ABD_SCATTER(abd).abd_offset);
767 }
768
769 return (abd);
770 }
771
772 /*
773 * If we're going to use this ABD for doing I/O using the block layer, the
774 * consumer of the ABD data doesn't care if it's scattered or not, and we don't
775 * plan to store this ABD in memory for a long period of time, we should
776 * allocate the ABD type that requires the least data copying to do the I/O.
777 *
778 * On Linux the optimal thing to do would be to use abd_get_offset() and
779 * construct a new ABD which shares the original pages thereby eliminating
780 * the copy. But for the moment a new linear ABD is allocated until this
781 * performance optimization can be implemented.
782 */
783 abd_t *
abd_alloc_for_io(size_t size,boolean_t is_metadata)784 abd_alloc_for_io(size_t size, boolean_t is_metadata)
785 {
786 return (abd_alloc(size, is_metadata));
787 }
788
789 abd_t *
abd_get_offset_scatter(abd_t * abd,abd_t * sabd,size_t off,size_t size)790 abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off,
791 size_t size)
792 {
793 (void) size;
794 int i = 0;
795 struct scatterlist *sg = NULL;
796
797 abd_verify(sabd);
798 ASSERT3U(off, <=, sabd->abd_size);
799
800 size_t new_offset = ABD_SCATTER(sabd).abd_offset + off;
801
802 if (abd == NULL)
803 abd = abd_alloc_struct(0);
804
805 /*
806 * Even if this buf is filesystem metadata, we only track that
807 * if we own the underlying data buffer, which is not true in
808 * this case. Therefore, we don't ever use ABD_FLAG_META here.
809 */
810
811 abd_for_each_sg(sabd, sg, ABD_SCATTER(sabd).abd_nents, i) {
812 if (new_offset < sg->length)
813 break;
814 new_offset -= sg->length;
815 }
816
817 ABD_SCATTER(abd).abd_sgl = sg;
818 ABD_SCATTER(abd).abd_offset = new_offset;
819 ABD_SCATTER(abd).abd_nents = ABD_SCATTER(sabd).abd_nents - i;
820
821 if (abd_is_from_pages(sabd))
822 abd->abd_flags |= ABD_FLAG_FROM_PAGES;
823
824 return (abd);
825 }
826
827 /*
828 * Initialize the abd_iter.
829 */
830 void
abd_iter_init(struct abd_iter * aiter,abd_t * abd)831 abd_iter_init(struct abd_iter *aiter, abd_t *abd)
832 {
833 ASSERT(!abd_is_gang(abd));
834 abd_verify(abd);
835 memset(aiter, 0, sizeof (struct abd_iter));
836 aiter->iter_abd = abd;
837 if (!abd_is_linear(abd)) {
838 aiter->iter_offset = ABD_SCATTER(abd).abd_offset;
839 aiter->iter_sg = ABD_SCATTER(abd).abd_sgl;
840 }
841 }
842
843 /*
844 * This is just a helper function to see if we have exhausted the
845 * abd_iter and reached the end.
846 */
847 boolean_t
abd_iter_at_end(struct abd_iter * aiter)848 abd_iter_at_end(struct abd_iter *aiter)
849 {
850 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size);
851 return (aiter->iter_pos == aiter->iter_abd->abd_size);
852 }
853
854 /*
855 * Advance the iterator by a certain amount. Cannot be called when a chunk is
856 * in use. This can be safely called when the aiter has already exhausted, in
857 * which case this does nothing.
858 */
859 void
abd_iter_advance(struct abd_iter * aiter,size_t amount)860 abd_iter_advance(struct abd_iter *aiter, size_t amount)
861 {
862 /*
863 * Ensure that last chunk is not in use. abd_iterate_*() must clear
864 * this state (directly or abd_iter_unmap()) before advancing.
865 */
866 ASSERT0P(aiter->iter_mapaddr);
867 ASSERT0(aiter->iter_mapsize);
868 ASSERT0P(aiter->iter_page);
869 ASSERT0(aiter->iter_page_doff);
870 ASSERT0(aiter->iter_page_dsize);
871
872 /* There's nothing left to advance to, so do nothing */
873 if (abd_iter_at_end(aiter))
874 return;
875
876 aiter->iter_pos += amount;
877 aiter->iter_offset += amount;
878 if (!abd_is_linear(aiter->iter_abd)) {
879 while (aiter->iter_offset >= aiter->iter_sg->length) {
880 aiter->iter_offset -= aiter->iter_sg->length;
881 aiter->iter_sg = sg_next(aiter->iter_sg);
882 if (aiter->iter_sg == NULL) {
883 ASSERT0(aiter->iter_offset);
884 break;
885 }
886 }
887 }
888 }
889
890 /*
891 * Map the current chunk into aiter. This can be safely called when the aiter
892 * has already exhausted, in which case this does nothing.
893 */
894 void
abd_iter_map(struct abd_iter * aiter)895 abd_iter_map(struct abd_iter *aiter)
896 {
897 void *paddr;
898 size_t offset = 0;
899
900 ASSERT0P(aiter->iter_mapaddr);
901 ASSERT0(aiter->iter_mapsize);
902
903 /* There's nothing left to iterate over, so do nothing */
904 if (abd_iter_at_end(aiter))
905 return;
906
907 if (abd_is_linear(aiter->iter_abd)) {
908 ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
909 offset = aiter->iter_offset;
910 aiter->iter_mapsize = aiter->iter_abd->abd_size - offset;
911 paddr = ABD_LINEAR_BUF(aiter->iter_abd);
912 } else {
913 offset = aiter->iter_offset;
914 aiter->iter_mapsize = MIN(aiter->iter_sg->length - offset,
915 aiter->iter_abd->abd_size - aiter->iter_pos);
916
917 paddr = zfs_kmap_local(sg_page(aiter->iter_sg));
918 }
919
920 aiter->iter_mapaddr = (char *)paddr + offset;
921 }
922
923 /*
924 * Unmap the current chunk from aiter. This can be safely called when the aiter
925 * has already exhausted, in which case this does nothing.
926 */
927 void
abd_iter_unmap(struct abd_iter * aiter)928 abd_iter_unmap(struct abd_iter *aiter)
929 {
930 /* There's nothing left to unmap, so do nothing */
931 if (abd_iter_at_end(aiter))
932 return;
933
934 if (!abd_is_linear(aiter->iter_abd)) {
935 /* LINTED E_FUNC_SET_NOT_USED */
936 zfs_kunmap_local(aiter->iter_mapaddr - aiter->iter_offset);
937 }
938
939 ASSERT3P(aiter->iter_mapaddr, !=, NULL);
940 ASSERT3U(aiter->iter_mapsize, >, 0);
941
942 aiter->iter_mapaddr = NULL;
943 aiter->iter_mapsize = 0;
944 }
945
946 void
abd_cache_reap_now(void)947 abd_cache_reap_now(void)
948 {
949 }
950
951 /*
952 * Borrow a raw buffer from an ABD without copying the contents of the ABD
953 * into the buffer. If the ABD is scattered, this will allocate a raw buffer
954 * whose contents are undefined. To copy over the existing data in the ABD, use
955 * abd_borrow_buf_copy() instead.
956 */
957 void *
abd_borrow_buf(abd_t * abd,size_t n)958 abd_borrow_buf(abd_t *abd, size_t n)
959 {
960 void *buf;
961 abd_verify(abd);
962 ASSERT3U(abd->abd_size, >=, 0);
963 /*
964 * In the event the ABD is composed of a single user page from Direct
965 * I/O we can not direclty return the raw buffer. This is a consequence
966 * of not being able to write protect the page and the contents of the
967 * page can be changed at any time by the user.
968 */
969 if (abd_is_from_pages(abd)) {
970 buf = zio_buf_alloc(n);
971 } else if (abd_is_linear(abd)) {
972 buf = abd_to_buf(abd);
973 } else {
974 buf = zio_buf_alloc(n);
975 }
976
977 #ifdef ZFS_DEBUG
978 (void) zfs_refcount_add_many(&abd->abd_children, n, buf);
979 #endif
980 return (buf);
981 }
982
983 void *
abd_borrow_buf_copy(abd_t * abd,size_t n)984 abd_borrow_buf_copy(abd_t *abd, size_t n)
985 {
986 void *buf = abd_borrow_buf(abd, n);
987
988 /*
989 * In the event the ABD is composed of a single user page from Direct
990 * I/O we must make sure copy the data over into the newly allocated
991 * buffer. This is a consequence of the fact that we can not write
992 * protect the user page and there is a risk the contents of the page
993 * could be changed by the user at any moment.
994 */
995 if (!abd_is_linear(abd) || abd_is_from_pages(abd)) {
996 abd_copy_to_buf(buf, abd, n);
997 }
998 return (buf);
999 }
1000
1001 /*
1002 * Return a borrowed raw buffer to an ABD. If the ABD is scatterd, this will
1003 * not change the contents of the ABD. If you want any changes you made to
1004 * buf to be copied back to abd, use abd_return_buf_copy() instead. If the
1005 * ABD is not constructed from user pages for Direct I/O then an ASSERT
1006 * checks to make sure the contents of buffer have not changed since it was
1007 * borrowed. We can not ASSERT that the contents of the buffer have not changed
1008 * if it is composed of user pages because the pages can not be placed under
1009 * write protection and the user could have possibly changed the contents in
1010 * the pages at any time. This is also an issue for Direct I/O reads. Checksum
1011 * verifications in the ZIO pipeline check for this issue and handle it by
1012 * returning an error on checksum verification failure.
1013 */
1014 void
abd_return_buf(abd_t * abd,void * buf,size_t n)1015 abd_return_buf(abd_t *abd, void *buf, size_t n)
1016 {
1017 abd_verify(abd);
1018 ASSERT3U(abd->abd_size, >=, n);
1019 #ifdef ZFS_DEBUG
1020 (void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
1021 #endif
1022 if (abd_is_from_pages(abd)) {
1023 zio_buf_free(buf, n);
1024 } else if (abd_is_linear(abd)) {
1025 ASSERT3P(buf, ==, abd_to_buf(abd));
1026 } else if (abd_is_gang(abd)) {
1027 #ifdef ZFS_DEBUG
1028 /*
1029 * We have to be careful with gang ABD's that we do not ASSERT0
1030 * for any ABD's that contain user pages from Direct I/O. In
1031 * order to handle this, we just iterate through the gang ABD
1032 * and only verify ABDs that are not from user pages.
1033 */
1034 void *cmp_buf = buf;
1035
1036 for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
1037 cabd != NULL;
1038 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
1039 if (!abd_is_from_pages(cabd)) {
1040 ASSERT0(abd_cmp_buf(cabd, cmp_buf,
1041 cabd->abd_size));
1042 }
1043 cmp_buf = (char *)cmp_buf + cabd->abd_size;
1044 }
1045 #endif
1046 zio_buf_free(buf, n);
1047 } else {
1048 ASSERT0(abd_cmp_buf(abd, buf, n));
1049 zio_buf_free(buf, n);
1050 }
1051 }
1052
1053 void
abd_return_buf_copy(abd_t * abd,void * buf,size_t n)1054 abd_return_buf_copy(abd_t *abd, void *buf, size_t n)
1055 {
1056 if (!abd_is_linear(abd) || abd_is_from_pages(abd)) {
1057 abd_copy_from_buf(abd, buf, n);
1058 }
1059 abd_return_buf(abd, buf, n);
1060 }
1061
1062 /*
1063 * This is abd_iter_page(), the function underneath abd_iterate_page_func().
1064 * It yields the next page struct and data offset and size within it, without
1065 * mapping it into the address space.
1066 */
1067
1068 /*
1069 * "Compound pages" are a group of pages that can be referenced from a single
1070 * struct page *. Its organised as a "head" page, followed by a series of
1071 * "tail" pages.
1072 *
1073 * In OpenZFS, compound pages are allocated using the __GFP_COMP flag, which we
1074 * get from scatter ABDs and SPL vmalloc slabs (ie >16K allocations). So a
1075 * great many of the IO buffers we get are going to be of this type.
1076 *
1077 * The tail pages are just regular PAGESIZE pages, and can be safely used
1078 * as-is. However, the head page has length covering itself and all the tail
1079 * pages. If the ABD chunk spans multiple pages, then we can use the head page
1080 * and a >PAGESIZE length, which is far more efficient.
1081 *
1082 * Before kernel 4.5 however, compound page heads were refcounted separately
1083 * from tail pages, such that moving back to the head page would require us to
1084 * take a reference to it and releasing it once we're completely finished with
1085 * it. In practice, that meant when our caller is done with the ABD, which we
1086 * have no insight into from here. Rather than contort this API to track head
1087 * page references on such ancient kernels, we disabled this special compound
1088 * page handling on kernels before 4.5, instead just using treating each page
1089 * within it as a regular PAGESIZE page (which it is). This is slightly less
1090 * efficient, but makes everything far simpler.
1091 *
1092 * We no longer support kernels before 4.5, so in theory none of this is
1093 * necessary. However, this code is still relatively new in the grand scheme of
1094 * things, so I'm leaving the ability to compile this out for the moment.
1095 *
1096 * Setting/clearing ABD_ITER_COMPOUND_PAGES below enables/disables the special
1097 * handling, by defining the ABD_ITER_PAGE_SIZE(page) macro to understand
1098 * compound pages, or not, and compiling in/out the support to detect compound
1099 * tail pages and move back to the start.
1100 */
1101
1102 /* On by default */
1103 #define ABD_ITER_COMPOUND_PAGES
1104
1105 #ifdef ABD_ITER_COMPOUND_PAGES
1106 #define ABD_ITER_PAGE_SIZE(page) \
1107 (PageCompound(page) ? page_size(page) : PAGESIZE)
1108 #else
1109 #define ABD_ITER_PAGE_SIZE(page) (PAGESIZE)
1110 #endif
1111
1112 void
abd_iter_page(struct abd_iter * aiter)1113 abd_iter_page(struct abd_iter *aiter)
1114 {
1115 if (abd_iter_at_end(aiter)) {
1116 aiter->iter_page = NULL;
1117 aiter->iter_page_doff = 0;
1118 aiter->iter_page_dsize = 0;
1119 return;
1120 }
1121
1122 struct page *page;
1123 size_t doff, dsize;
1124
1125 /*
1126 * Find the page, and the start of the data within it. This is computed
1127 * differently for linear and scatter ABDs; linear is referenced by
1128 * virtual memory location, while scatter is referenced by page
1129 * pointer.
1130 */
1131 if (abd_is_linear(aiter->iter_abd)) {
1132 ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
1133
1134 /* memory address at iter_pos */
1135 void *paddr = ABD_LINEAR_BUF(aiter->iter_abd) + aiter->iter_pos;
1136
1137 /* struct page for address */
1138 page = is_vmalloc_addr(paddr) ?
1139 vmalloc_to_page(paddr) : virt_to_page(paddr);
1140
1141 /* offset of address within the page */
1142 doff = offset_in_page(paddr);
1143 } else {
1144 ASSERT(!abd_is_gang(aiter->iter_abd));
1145
1146 /* current scatter page */
1147 page = nth_page(sg_page(aiter->iter_sg),
1148 aiter->iter_offset >> PAGE_SHIFT);
1149
1150 /* position within page */
1151 doff = aiter->iter_offset & (PAGESIZE - 1);
1152 }
1153
1154 #ifdef ABD_ITER_COMPOUND_PAGES
1155 if (PageTail(page)) {
1156 /*
1157 * If this is a compound tail page, move back to the head, and
1158 * adjust the offset to match. This may let us yield a much
1159 * larger amount of data from a single logical page, and so
1160 * leave our caller with fewer pages to process.
1161 */
1162 struct page *head = compound_head(page);
1163 doff += ((page - head) * PAGESIZE);
1164 page = head;
1165 }
1166 #endif
1167
1168 ASSERT(page);
1169
1170 /*
1171 * Compute the maximum amount of data we can take from this page. This
1172 * is the smaller of:
1173 * - the remaining space in the page
1174 * - the remaining space in this scatterlist entry (which may not cover
1175 * the entire page)
1176 * - the remaining space in the abd (which may not cover the entire
1177 * scatterlist entry)
1178 */
1179 dsize = MIN(ABD_ITER_PAGE_SIZE(page) - doff,
1180 aiter->iter_abd->abd_size - aiter->iter_pos);
1181 if (!abd_is_linear(aiter->iter_abd))
1182 dsize = MIN(dsize, aiter->iter_sg->length - aiter->iter_offset);
1183 ASSERT3U(dsize, >, 0);
1184
1185 /* final iterator outputs */
1186 aiter->iter_page = page;
1187 aiter->iter_page_doff = doff;
1188 aiter->iter_page_dsize = dsize;
1189 }
1190
1191 /*
1192 * Note: ABD BIO functions only needed to support vdev_classic. See comments in
1193 * vdev_disk.c.
1194 */
1195
1196 /*
1197 * bio_nr_pages for ABD.
1198 * @off is the offset in @abd
1199 */
1200 unsigned long
abd_nr_pages_off(abd_t * abd,unsigned int size,size_t off)1201 abd_nr_pages_off(abd_t *abd, unsigned int size, size_t off)
1202 {
1203 unsigned long pos;
1204
1205 if (abd_is_gang(abd)) {
1206 unsigned long count = 0;
1207
1208 for (abd_t *cabd = abd_gang_get_offset(abd, &off);
1209 cabd != NULL && size != 0;
1210 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
1211 ASSERT3U(off, <, cabd->abd_size);
1212 int mysize = MIN(size, cabd->abd_size - off);
1213 count += abd_nr_pages_off(cabd, mysize, off);
1214 size -= mysize;
1215 off = 0;
1216 }
1217 return (count);
1218 }
1219
1220 if (abd_is_linear(abd))
1221 pos = (unsigned long)abd_to_buf(abd) + off;
1222 else
1223 pos = ABD_SCATTER(abd).abd_offset + off;
1224
1225 return (((pos + size + PAGESIZE - 1) >> PAGE_SHIFT) -
1226 (pos >> PAGE_SHIFT));
1227 }
1228
1229 static unsigned int
bio_map(struct bio * bio,void * buf_ptr,unsigned int bio_size)1230 bio_map(struct bio *bio, void *buf_ptr, unsigned int bio_size)
1231 {
1232 unsigned int offset, size, i;
1233 struct page *page;
1234
1235 offset = offset_in_page(buf_ptr);
1236 for (i = 0; i < bio->bi_max_vecs; i++) {
1237 size = PAGE_SIZE - offset;
1238
1239 if (bio_size <= 0)
1240 break;
1241
1242 if (size > bio_size)
1243 size = bio_size;
1244
1245 if (is_vmalloc_addr(buf_ptr))
1246 page = vmalloc_to_page(buf_ptr);
1247 else
1248 page = virt_to_page(buf_ptr);
1249
1250 /*
1251 * Some network related block device uses tcp_sendpage, which
1252 * doesn't behave well when using 0-count page, this is a
1253 * safety net to catch them.
1254 */
1255 ASSERT3S(page_count(page), >, 0);
1256
1257 if (bio_add_page(bio, page, size, offset) != size)
1258 break;
1259
1260 buf_ptr += size;
1261 bio_size -= size;
1262 offset = 0;
1263 }
1264
1265 return (bio_size);
1266 }
1267
1268 /*
1269 * bio_map for gang ABD.
1270 */
1271 static unsigned int
abd_gang_bio_map_off(struct bio * bio,abd_t * abd,unsigned int io_size,size_t off)1272 abd_gang_bio_map_off(struct bio *bio, abd_t *abd,
1273 unsigned int io_size, size_t off)
1274 {
1275 ASSERT(abd_is_gang(abd));
1276
1277 for (abd_t *cabd = abd_gang_get_offset(abd, &off);
1278 cabd != NULL;
1279 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
1280 ASSERT3U(off, <, cabd->abd_size);
1281 int size = MIN(io_size, cabd->abd_size - off);
1282 int remainder = abd_bio_map_off(bio, cabd, size, off);
1283 io_size -= (size - remainder);
1284 if (io_size == 0 || remainder > 0)
1285 return (io_size);
1286 off = 0;
1287 }
1288 ASSERT0(io_size);
1289 return (io_size);
1290 }
1291
1292 /*
1293 * bio_map for ABD.
1294 * @off is the offset in @abd
1295 * Remaining IO size is returned
1296 */
1297 unsigned int
abd_bio_map_off(struct bio * bio,abd_t * abd,unsigned int io_size,size_t off)1298 abd_bio_map_off(struct bio *bio, abd_t *abd,
1299 unsigned int io_size, size_t off)
1300 {
1301 struct abd_iter aiter;
1302
1303 ASSERT3U(io_size, <=, abd->abd_size - off);
1304 if (abd_is_linear(abd))
1305 return (bio_map(bio, ((char *)abd_to_buf(abd)) + off, io_size));
1306
1307 ASSERT(!abd_is_linear(abd));
1308 if (abd_is_gang(abd))
1309 return (abd_gang_bio_map_off(bio, abd, io_size, off));
1310
1311 abd_iter_init(&aiter, abd);
1312 abd_iter_advance(&aiter, off);
1313
1314 for (int i = 0; i < bio->bi_max_vecs; i++) {
1315 struct page *pg;
1316 size_t len, sgoff, pgoff;
1317 struct scatterlist *sg;
1318
1319 if (io_size <= 0)
1320 break;
1321
1322 sg = aiter.iter_sg;
1323 sgoff = aiter.iter_offset;
1324 pgoff = sgoff & (PAGESIZE - 1);
1325 len = MIN(io_size, PAGESIZE - pgoff);
1326 ASSERT(len > 0);
1327
1328 pg = nth_page(sg_page(sg), sgoff >> PAGE_SHIFT);
1329 if (bio_add_page(bio, pg, len, pgoff) != len)
1330 break;
1331
1332 io_size -= len;
1333 abd_iter_advance(&aiter, len);
1334 }
1335
1336 return (io_size);
1337 }
1338
1339 EXPORT_SYMBOL(abd_alloc_from_pages);
1340
1341 /* Tunable Parameters */
1342 module_param(zfs_abd_scatter_enabled, int, 0644);
1343 MODULE_PARM_DESC(zfs_abd_scatter_enabled,
1344 "Toggle whether ABD allocations must be linear.");
1345 module_param(zfs_abd_scatter_min_size, int, 0644);
1346 MODULE_PARM_DESC(zfs_abd_scatter_min_size,
1347 "Minimum size of scatter allocations.");
1348 module_param(zfs_abd_scatter_max_order, uint, 0644);
1349 MODULE_PARM_DESC(zfs_abd_scatter_max_order,
1350 "Maximum order allocation used for a scatter ABD.");
1351