1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 24 * Copyright (c) 2016, 2019 by Delphix. All rights reserved. 25 * Copyright (c) 2023, 2024, Klara Inc. 26 */ 27 28 #ifndef _ABD_IMPL_H 29 #define _ABD_IMPL_H 30 31 #include <sys/abd.h> 32 #include <sys/abd_impl_os.h> 33 #include <sys/wmsum.h> 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 typedef enum abd_stats_op { 40 ABDSTAT_INCR, /* Increase abdstat values */ 41 ABDSTAT_DECR /* Decrease abdstat values */ 42 } abd_stats_op_t; 43 44 /* forward declarations */ 45 struct scatterlist; 46 struct page; 47 #if defined(__FreeBSD__) && defined(_KERNEL) 48 struct sf_buf; 49 #endif 50 51 struct abd_iter { 52 /* public interface */ 53 union { 54 /* for abd_iter_map()/abd_iter_unmap() */ 55 struct { 56 /* addr corresponding to iter_pos */ 57 void *iter_mapaddr; 58 /* length of data valid at mapaddr */ 59 size_t iter_mapsize; 60 }; 61 /* for abd_iter_page() */ 62 struct { 63 /* current page */ 64 struct page *iter_page; 65 /* offset of data in page */ 66 size_t iter_page_doff; 67 /* size of data in page */ 68 size_t iter_page_dsize; 69 }; 70 }; 71 72 /* private */ 73 abd_t *iter_abd; /* ABD being iterated through */ 74 size_t iter_pos; 75 size_t iter_offset; /* offset in current sg/abd_buf, */ 76 /* abd_offset included */ 77 #if defined(__FreeBSD__) && defined(_KERNEL) 78 struct sf_buf *sf; /* used to map in vm_page_t FreeBSD */ 79 #else 80 struct scatterlist *iter_sg; /* current sg */ 81 #endif 82 }; 83 84 extern abd_t *abd_zero_scatter; 85 86 abd_t *abd_gang_get_offset(abd_t *, size_t *); 87 abd_t *abd_alloc_struct(size_t); 88 void abd_free_struct(abd_t *); 89 void abd_init_struct(abd_t *); 90 91 /* 92 * OS specific functions 93 */ 94 95 abd_t *abd_alloc_struct_impl(size_t); 96 abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t, size_t); 97 void abd_free_struct_impl(abd_t *); 98 void abd_alloc_chunks(abd_t *, size_t); 99 void abd_free_chunks(abd_t *); 100 void abd_update_scatter_stats(abd_t *, abd_stats_op_t); 101 void abd_update_linear_stats(abd_t *, abd_stats_op_t); 102 void abd_verify_scatter(abd_t *); 103 void abd_free_linear_page(abd_t *); 104 /* OS specific abd_iter functions */ 105 void abd_iter_init(struct abd_iter *, abd_t *); 106 boolean_t abd_iter_at_end(struct abd_iter *); 107 void abd_iter_advance(struct abd_iter *, size_t); 108 void abd_iter_map(struct abd_iter *); 109 void abd_iter_unmap(struct abd_iter *); 110 void abd_iter_page(struct abd_iter *); 111 112 /* 113 * Helper macros 114 */ 115 #define ABDSTAT_INCR(stat, val) \ 116 wmsum_add(&abd_sums.stat, (val)) 117 #define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1) 118 #define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1) 119 120 #define ABD_SCATTER(abd) ((abd)->abd_u.abd_scatter) 121 #define ABD_LINEAR_BUF(abd) ((abd)->abd_u.abd_linear.abd_buf) 122 #define ABD_GANG(abd) ((abd)->abd_u.abd_gang) 123 124 #ifdef __cplusplus 125 } 126 #endif 127 128 #endif /* _ABD_IMPL_H */ 129