1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 */ 29 30 #ifndef _ZFS_FLETCHER_H 31 #define _ZFS_FLETCHER_H extern __attribute__((visibility("default"))) 32 33 #include <sys/types.h> 34 #include <sys/spa_checksum.h> 35 36 #ifdef __cplusplus 37 extern "C" { 38 #endif 39 40 /* 41 * fletcher checksum functions 42 * 43 * Note: Fletcher checksum methods expect buffer size to be 4B aligned. This 44 * limitation stems from the algorithm design. Performing incremental checksum 45 * without said alignment would yield different results. Therefore, the code 46 * includes assertions for the size alignment. 47 * For compatibility, it is required that some code paths calculate checksum of 48 * non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()` 49 * checksum method is added. This method will ignore last (size % 4) bytes of 50 * the data buffer. 51 */ 52 _ZFS_FLETCHER_H void fletcher_init(zio_cksum_t *); 53 _ZFS_FLETCHER_H void fletcher_2_native(const void *, uint64_t, const void *, 54 zio_cksum_t *); 55 _ZFS_FLETCHER_H void fletcher_2_byteswap(const void *, uint64_t, const void *, 56 zio_cksum_t *); 57 _ZFS_FLETCHER_H void fletcher_4_native(const void *, uint64_t, const void *, 58 zio_cksum_t *); 59 _ZFS_FLETCHER_H int fletcher_2_incremental_native(void *, size_t, void *); 60 _ZFS_FLETCHER_H int fletcher_2_incremental_byteswap(void *, size_t, void *); 61 _ZFS_FLETCHER_H void fletcher_4_native_varsize(const void *, uint64_t, 62 zio_cksum_t *); 63 _ZFS_FLETCHER_H void fletcher_4_byteswap_varsize(const void *, uint64_t, 64 zio_cksum_t *); 65 _ZFS_FLETCHER_H void fletcher_4_byteswap(const void *, uint64_t, const void *, 66 zio_cksum_t *); 67 _ZFS_FLETCHER_H int fletcher_4_incremental_native(void *, size_t, void *); 68 _ZFS_FLETCHER_H int fletcher_4_incremental_byteswap(void *, size_t, void *); 69 _ZFS_FLETCHER_H int fletcher_4_impl_set(const char *selector); 70 _ZFS_FLETCHER_H void fletcher_4_init(void); 71 _ZFS_FLETCHER_H void fletcher_4_fini(void); 72 73 74 75 /* Internal fletcher ctx */ 76 77 typedef struct zfs_fletcher_superscalar { 78 uint64_t v[4]; 79 } zfs_fletcher_superscalar_t; 80 81 typedef struct zfs_fletcher_sse { 82 uint64_t v[2]; 83 } zfs_fletcher_sse_t; 84 85 typedef struct zfs_fletcher_avx { 86 uint64_t v[4]; 87 } zfs_fletcher_avx_t; 88 89 typedef struct zfs_fletcher_avx512 { 90 uint64_t v[8]; 91 } zfs_fletcher_avx512_t; 92 93 typedef struct zfs_fletcher_aarch64_neon { 94 uint64_t v[2]; 95 } zfs_fletcher_aarch64_neon_t; 96 97 98 typedef union fletcher_4_ctx { 99 zio_cksum_t scalar; 100 zfs_fletcher_superscalar_t superscalar[4]; 101 102 #if HAVE_SIMD(SSE2) || (HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3)) 103 zfs_fletcher_sse_t sse[4]; 104 #endif 105 #if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) 106 zfs_fletcher_avx_t avx[4]; 107 #endif 108 #if defined(__x86_64) && HAVE_SIMD(AVX512F) 109 zfs_fletcher_avx512_t avx512[4]; 110 #endif 111 #if defined(__aarch64__) 112 zfs_fletcher_aarch64_neon_t aarch64_neon[4]; 113 #endif 114 } fletcher_4_ctx_t; 115 116 /* 117 * fletcher checksum struct 118 */ 119 typedef void (*fletcher_4_init_f)(fletcher_4_ctx_t *); 120 typedef void (*fletcher_4_fini_f)(fletcher_4_ctx_t *, zio_cksum_t *); 121 typedef void (*fletcher_4_compute_f)(fletcher_4_ctx_t *, 122 const void *, uint64_t); 123 124 typedef struct fletcher_4_func { 125 fletcher_4_init_f init_native; 126 fletcher_4_fini_f fini_native; 127 fletcher_4_compute_f compute_native; 128 fletcher_4_init_f init_byteswap; 129 fletcher_4_fini_f fini_byteswap; 130 fletcher_4_compute_f compute_byteswap; 131 boolean_t (*valid)(void); 132 boolean_t uses_fpu; 133 const char *name; 134 } __attribute__((aligned(64))) fletcher_4_ops_t; 135 136 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar_ops; 137 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar4_ops; 138 139 #if HAVE_SIMD(SSE2) 140 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_sse2_ops; 141 #endif 142 143 #if HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3) 144 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_ssse3_ops; 145 #endif 146 147 #if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) 148 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx2_ops; 149 #endif 150 151 #if defined(__x86_64) && HAVE_SIMD(AVX512F) 152 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512f_ops; 153 #endif 154 155 #if defined(__x86_64) && HAVE_SIMD(AVX512BW) 156 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512bw_ops; 157 #endif 158 159 #if defined(__aarch64__) 160 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_aarch64_neon_ops; 161 #endif 162 163 #ifdef __cplusplus 164 } 165 #endif 166 167 #endif /* _ZFS_FLETCHER_H */ 168