1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 14 * Use is subject to license terms. 15 */ 16 /* 17 * Copyright 2013 Saso Kiselkov. All rights reserved. 18 */ 19 20 #ifndef _ZFS_FLETCHER_H 21 #define _ZFS_FLETCHER_H extern __attribute__((visibility("default"))) 22 23 #include <sys/types.h> 24 #include <sys/spa_checksum.h> 25 26 #ifdef __cplusplus 27 extern "C" { 28 #endif 29 30 /* 31 * fletcher checksum functions 32 * 33 * Note: Fletcher checksum methods expect buffer size to be 4B aligned. This 34 * limitation stems from the algorithm design. Performing incremental checksum 35 * without said alignment would yield different results. Therefore, the code 36 * includes assertions for the size alignment. 37 * For compatibility, it is required that some code paths calculate checksum of 38 * non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()` 39 * checksum method is added. This method will ignore last (size % 4) bytes of 40 * the data buffer. 41 */ 42 _ZFS_FLETCHER_H void fletcher_init(zio_cksum_t *); 43 _ZFS_FLETCHER_H void fletcher_2_native(const void *, uint64_t, const void *, 44 zio_cksum_t *); 45 _ZFS_FLETCHER_H void fletcher_2_byteswap(const void *, uint64_t, const void *, 46 zio_cksum_t *); 47 _ZFS_FLETCHER_H void fletcher_4_native(const void *, uint64_t, const void *, 48 zio_cksum_t *); 49 _ZFS_FLETCHER_H int fletcher_2_incremental_native(void *, size_t, void *); 50 _ZFS_FLETCHER_H int fletcher_2_incremental_byteswap(void *, size_t, void *); 51 _ZFS_FLETCHER_H void fletcher_4_native_varsize(const void *, uint64_t, 52 zio_cksum_t *); 53 _ZFS_FLETCHER_H void fletcher_4_byteswap_varsize(const void *, uint64_t, 54 zio_cksum_t *); 55 _ZFS_FLETCHER_H void fletcher_4_byteswap(const void *, uint64_t, const void *, 56 zio_cksum_t *); 57 _ZFS_FLETCHER_H int fletcher_4_incremental_native(void *, size_t, void *); 58 _ZFS_FLETCHER_H int fletcher_4_incremental_byteswap(void *, size_t, void *); 59 _ZFS_FLETCHER_H int fletcher_4_impl_set(const char *selector); 60 _ZFS_FLETCHER_H void fletcher_4_init(void); 61 _ZFS_FLETCHER_H void fletcher_4_fini(void); 62 63 64 65 /* Internal fletcher ctx */ 66 67 typedef struct zfs_fletcher_superscalar { 68 uint64_t v[4]; 69 } zfs_fletcher_superscalar_t; 70 71 typedef struct zfs_fletcher_sse { 72 uint64_t v[2]; 73 } zfs_fletcher_sse_t; 74 75 typedef struct zfs_fletcher_avx { 76 uint64_t v[4]; 77 } zfs_fletcher_avx_t; 78 79 typedef struct zfs_fletcher_avx512 { 80 uint64_t v[8]; 81 } zfs_fletcher_avx512_t; 82 83 typedef struct zfs_fletcher_aarch64_neon { 84 uint64_t v[2]; 85 } zfs_fletcher_aarch64_neon_t; 86 87 88 typedef union fletcher_4_ctx { 89 zio_cksum_t scalar; 90 zfs_fletcher_superscalar_t superscalar[4]; 91 92 #if HAVE_SIMD(SSE2) || (HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3)) 93 zfs_fletcher_sse_t sse[4]; 94 #endif 95 #if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) 96 zfs_fletcher_avx_t avx[4]; 97 #endif 98 #if defined(__x86_64) && HAVE_SIMD(AVX512F) 99 zfs_fletcher_avx512_t avx512[4]; 100 #endif 101 #if defined(__aarch64__) 102 zfs_fletcher_aarch64_neon_t aarch64_neon[4]; 103 #endif 104 } fletcher_4_ctx_t; 105 106 /* 107 * fletcher checksum struct 108 */ 109 typedef void (*fletcher_4_init_f)(fletcher_4_ctx_t *); 110 typedef void (*fletcher_4_fini_f)(fletcher_4_ctx_t *, zio_cksum_t *); 111 typedef void (*fletcher_4_compute_f)(fletcher_4_ctx_t *, 112 const void *, uint64_t); 113 114 typedef struct fletcher_4_func { 115 fletcher_4_init_f init_native; 116 fletcher_4_fini_f fini_native; 117 fletcher_4_compute_f compute_native; 118 fletcher_4_init_f init_byteswap; 119 fletcher_4_fini_f fini_byteswap; 120 fletcher_4_compute_f compute_byteswap; 121 boolean_t (*valid)(void); 122 boolean_t uses_fpu; 123 const char *name; 124 } __attribute__((aligned(64))) fletcher_4_ops_t; 125 126 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar_ops; 127 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar4_ops; 128 129 #if HAVE_SIMD(SSE2) 130 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_sse2_ops; 131 #endif 132 133 #if HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3) 134 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_ssse3_ops; 135 #endif 136 137 #if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) 138 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx2_ops; 139 #endif 140 141 #if defined(__x86_64) && HAVE_SIMD(AVX512F) 142 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512f_ops; 143 #endif 144 145 #if defined(__x86_64) && HAVE_SIMD(AVX512BW) 146 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512bw_ops; 147 #endif 148 149 #if defined(__aarch64__) 150 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_aarch64_neon_ops; 151 #endif 152 153 #ifdef __cplusplus 154 } 155 #endif 156 157 #endif /* _ZFS_FLETCHER_H */ 158