1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright 2013 Saso Kiselkov. All rights reserved. 27 */ 28 29 #ifndef _ZFS_FLETCHER_H 30 #define _ZFS_FLETCHER_H extern __attribute__((visibility("default"))) 31 32 #include <sys/types.h> 33 #include <sys/spa_checksum.h> 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 /* 40 * fletcher checksum functions 41 * 42 * Note: Fletcher checksum methods expect buffer size to be 4B aligned. This 43 * limitation stems from the algorithm design. Performing incremental checksum 44 * without said alignment would yield different results. Therefore, the code 45 * includes assertions for the size alignment. 46 * For compatibility, it is required that some code paths calculate checksum of 47 * non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()` 48 * checksum method is added. This method will ignore last (size % 4) bytes of 49 * the data buffer. 50 */ 51 _ZFS_FLETCHER_H void fletcher_init(zio_cksum_t *); 52 _ZFS_FLETCHER_H void fletcher_2_native(const void *, uint64_t, const void *, 53 zio_cksum_t *); 54 _ZFS_FLETCHER_H void fletcher_2_byteswap(const void *, uint64_t, const void *, 55 zio_cksum_t *); 56 _ZFS_FLETCHER_H void fletcher_4_native(const void *, uint64_t, const void *, 57 zio_cksum_t *); 58 _ZFS_FLETCHER_H int fletcher_2_incremental_native(void *, size_t, void *); 59 _ZFS_FLETCHER_H int fletcher_2_incremental_byteswap(void *, size_t, void *); 60 _ZFS_FLETCHER_H void fletcher_4_native_varsize(const void *, uint64_t, 61 zio_cksum_t *); 62 _ZFS_FLETCHER_H void fletcher_4_byteswap(const void *, uint64_t, const void *, 63 zio_cksum_t *); 64 _ZFS_FLETCHER_H int fletcher_4_incremental_native(void *, size_t, void *); 65 _ZFS_FLETCHER_H int fletcher_4_incremental_byteswap(void *, size_t, void *); 66 _ZFS_FLETCHER_H int fletcher_4_impl_set(const char *selector); 67 _ZFS_FLETCHER_H void fletcher_4_init(void); 68 _ZFS_FLETCHER_H void fletcher_4_fini(void); 69 70 71 72 /* Internal fletcher ctx */ 73 74 typedef struct zfs_fletcher_superscalar { 75 uint64_t v[4]; 76 } zfs_fletcher_superscalar_t; 77 78 typedef struct zfs_fletcher_sse { 79 uint64_t v[2] __attribute__((aligned(16))); 80 } zfs_fletcher_sse_t; 81 82 typedef struct zfs_fletcher_avx { 83 uint64_t v[4] __attribute__((aligned(32))); 84 } zfs_fletcher_avx_t; 85 86 typedef struct zfs_fletcher_avx512 { 87 uint64_t v[8] __attribute__((aligned(64))); 88 } zfs_fletcher_avx512_t; 89 90 typedef struct zfs_fletcher_aarch64_neon { 91 uint64_t v[2] __attribute__((aligned(16))); 92 } zfs_fletcher_aarch64_neon_t; 93 94 95 typedef union fletcher_4_ctx { 96 zio_cksum_t scalar; 97 zfs_fletcher_superscalar_t superscalar[4]; 98 99 #if defined(HAVE_SSE2) || (defined(HAVE_SSE2) && defined(HAVE_SSSE3)) 100 zfs_fletcher_sse_t sse[4]; 101 #endif 102 #if defined(HAVE_AVX) && defined(HAVE_AVX2) 103 zfs_fletcher_avx_t avx[4]; 104 #endif 105 #if defined(__x86_64) && defined(HAVE_AVX512F) 106 zfs_fletcher_avx512_t avx512[4]; 107 #endif 108 #if defined(__aarch64__) 109 zfs_fletcher_aarch64_neon_t aarch64_neon[4]; 110 #endif 111 } fletcher_4_ctx_t; 112 113 /* 114 * fletcher checksum struct 115 */ 116 typedef void (*fletcher_4_init_f)(fletcher_4_ctx_t *); 117 typedef void (*fletcher_4_fini_f)(fletcher_4_ctx_t *, zio_cksum_t *); 118 typedef void (*fletcher_4_compute_f)(fletcher_4_ctx_t *, 119 const void *, uint64_t); 120 121 typedef struct fletcher_4_func { 122 fletcher_4_init_f init_native; 123 fletcher_4_fini_f fini_native; 124 fletcher_4_compute_f compute_native; 125 fletcher_4_init_f init_byteswap; 126 fletcher_4_fini_f fini_byteswap; 127 fletcher_4_compute_f compute_byteswap; 128 boolean_t (*valid)(void); 129 const char *name; 130 } fletcher_4_ops_t; 131 132 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar_ops; 133 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar4_ops; 134 135 #if defined(HAVE_SSE2) 136 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_sse2_ops; 137 #endif 138 139 #if defined(HAVE_SSE2) && defined(HAVE_SSSE3) 140 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_ssse3_ops; 141 #endif 142 143 #if defined(HAVE_AVX) && defined(HAVE_AVX2) 144 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx2_ops; 145 #endif 146 147 #if defined(__x86_64) && defined(HAVE_AVX512F) 148 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512f_ops; 149 #endif 150 151 #if defined(__x86_64) && defined(HAVE_AVX512BW) 152 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512bw_ops; 153 #endif 154 155 #if defined(__aarch64__) 156 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_aarch64_neon_ops; 157 #endif 158 159 #ifdef __cplusplus 160 } 161 #endif 162 163 #if defined(ZFS_UBSAN_ENABLED) 164 #if defined(__has_attribute) 165 #if __has_attribute(no_sanitize_undefined) 166 #define ZFS_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) 167 #elif __has_attribute(no_sanitize) 168 #define ZFS_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) 169 #else 170 #error "Compiler has to support attribute " 171 "`no_sanitize_undefined` or `no_sanitize(\"undefined\")`" 172 "when compiling with UBSan enabled" 173 #endif /* __has_attribute(no_sanitize_undefined) */ 174 #endif /* defined(__has_attribute) */ 175 #else 176 #define ZFS_NO_SANITIZE_UNDEFINED 177 #endif /* defined(ZFS_UBSAN_ENABLED) */ 178 179 #endif /* _ZFS_FLETCHER_H */ 180