1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 14 * Copyright (c) 2016, 2019 by Delphix. All rights reserved. 15 * Copyright (c) 2023, 2024, Klara Inc. 16 */ 17 18 #ifndef _ABD_IMPL_H 19 #define _ABD_IMPL_H 20 21 #include <sys/abd.h> 22 #include <sys/abd_impl_os.h> 23 #include <sys/wmsum.h> 24 25 #ifdef __cplusplus 26 extern "C" { 27 #endif 28 29 typedef enum abd_stats_op { 30 ABDSTAT_INCR, /* Increase abdstat values */ 31 ABDSTAT_DECR /* Decrease abdstat values */ 32 } abd_stats_op_t; 33 34 /* forward declarations */ 35 struct scatterlist; 36 struct page; 37 #if defined(__FreeBSD__) && defined(_KERNEL) 38 struct sf_buf; 39 #endif 40 41 struct abd_iter { 42 /* public interface */ 43 union { 44 /* for abd_iter_map()/abd_iter_unmap() */ 45 struct { 46 /* addr corresponding to iter_pos */ 47 void *iter_mapaddr; 48 /* length of data valid at mapaddr */ 49 size_t iter_mapsize; 50 }; 51 /* for abd_iter_page() */ 52 struct { 53 /* current page */ 54 struct page *iter_page; 55 /* offset of data in page */ 56 size_t iter_page_doff; 57 /* size of data in page */ 58 size_t iter_page_dsize; 59 }; 60 }; 61 62 /* private */ 63 abd_t *iter_abd; /* ABD being iterated through */ 64 size_t iter_pos; 65 size_t iter_offset; /* offset in current sg/abd_buf, */ 66 /* abd_offset included */ 67 #if defined(__FreeBSD__) && defined(_KERNEL) 68 struct sf_buf *sf; /* used to map in vm_page_t FreeBSD */ 69 #else 70 struct scatterlist *iter_sg; /* current sg */ 71 #endif 72 }; 73 74 extern abd_t *abd_zero_scatter; 75 76 abd_t *abd_gang_get_offset(abd_t *, size_t *); 77 abd_t *abd_alloc_struct(size_t); 78 void abd_free_struct(abd_t *); 79 void abd_init_struct(abd_t *); 80 81 /* 82 * OS specific functions 83 */ 84 85 abd_t *abd_alloc_struct_impl(size_t); 86 abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t, size_t); 87 void abd_free_struct_impl(abd_t *); 88 void abd_alloc_chunks(abd_t *, size_t); 89 void abd_free_chunks(abd_t *); 90 void abd_update_scatter_stats(abd_t *, abd_stats_op_t); 91 void abd_update_linear_stats(abd_t *, abd_stats_op_t); 92 void abd_verify_scatter(abd_t *); 93 void abd_free_linear_page(abd_t *); 94 /* OS specific abd_iter functions */ 95 void abd_iter_init(struct abd_iter *, abd_t *); 96 boolean_t abd_iter_at_end(struct abd_iter *); 97 void abd_iter_advance(struct abd_iter *, size_t); 98 void abd_iter_map(struct abd_iter *); 99 void abd_iter_unmap(struct abd_iter *); 100 void abd_iter_page(struct abd_iter *); 101 102 /* 103 * Helper macros 104 */ 105 #define ABDSTAT_INCR(stat, val) \ 106 wmsum_add(&abd_sums.stat, (val)) 107 #define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1) 108 #define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1) 109 110 #define ABD_SCATTER(abd) ((abd)->abd_u.abd_scatter) 111 #define ABD_LINEAR_BUF(abd) ((abd)->abd_u.abd_linear.abd_buf) 112 #define ABD_GANG(abd) ((abd)->abd_u.abd_gang) 113 114 #ifdef __cplusplus 115 } 116 #endif 117 118 #endif /* _ABD_IMPL_H */ 119