1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 */ 29 30 #ifndef _ZFS_FLETCHER_H 31 #define _ZFS_FLETCHER_H extern __attribute__((visibility("default"))) 32 33 #include <sys/types.h> 34 #include <sys/spa_checksum.h> 35 36 #ifdef __cplusplus 37 extern "C" { 38 #endif 39 40 /* 41 * fletcher checksum functions 42 * 43 * Note: Fletcher checksum methods expect buffer size to be 4B aligned. This 44 * limitation stems from the algorithm design. Performing incremental checksum 45 * without said alignment would yield different results. Therefore, the code 46 * includes assertions for the size alignment. 47 * For compatibility, it is required that some code paths calculate checksum of 48 * non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()` 49 * checksum method is added. This method will ignore last (size % 4) bytes of 50 * the data buffer. 51 */ 52 _ZFS_FLETCHER_H void fletcher_init(zio_cksum_t *); 53 _ZFS_FLETCHER_H void fletcher_2_native(const void *, uint64_t, const void *, 54 zio_cksum_t *); 55 _ZFS_FLETCHER_H void fletcher_2_byteswap(const void *, uint64_t, const void *, 56 zio_cksum_t *); 57 _ZFS_FLETCHER_H void fletcher_4_native(const void *, uint64_t, const void *, 58 zio_cksum_t *); 59 _ZFS_FLETCHER_H int fletcher_2_incremental_native(void *, size_t, void *); 60 _ZFS_FLETCHER_H int fletcher_2_incremental_byteswap(void *, size_t, void *); 61 _ZFS_FLETCHER_H void fletcher_4_native_varsize(const void *, uint64_t, 62 zio_cksum_t *); 63 _ZFS_FLETCHER_H void fletcher_4_byteswap(const void *, uint64_t, const void *, 64 zio_cksum_t *); 65 _ZFS_FLETCHER_H int fletcher_4_incremental_native(void *, size_t, void *); 66 _ZFS_FLETCHER_H int fletcher_4_incremental_byteswap(void *, size_t, void *); 67 _ZFS_FLETCHER_H int fletcher_4_impl_set(const char *selector); 68 _ZFS_FLETCHER_H void fletcher_4_init(void); 69 _ZFS_FLETCHER_H void fletcher_4_fini(void); 70 71 72 73 /* Internal fletcher ctx */ 74 75 typedef struct zfs_fletcher_superscalar { 76 uint64_t v[4]; 77 } zfs_fletcher_superscalar_t; 78 79 typedef struct zfs_fletcher_sse { 80 uint64_t v[2]; 81 } zfs_fletcher_sse_t; 82 83 typedef struct zfs_fletcher_avx { 84 uint64_t v[4]; 85 } zfs_fletcher_avx_t; 86 87 typedef struct zfs_fletcher_avx512 { 88 uint64_t v[8]; 89 } zfs_fletcher_avx512_t; 90 91 typedef struct zfs_fletcher_aarch64_neon { 92 uint64_t v[2]; 93 } zfs_fletcher_aarch64_neon_t; 94 95 96 typedef union fletcher_4_ctx { 97 zio_cksum_t scalar; 98 zfs_fletcher_superscalar_t superscalar[4]; 99 100 #if defined(HAVE_SSE2) || (defined(HAVE_SSE2) && defined(HAVE_SSSE3)) 101 zfs_fletcher_sse_t sse[4]; 102 #endif 103 #if defined(HAVE_AVX) && defined(HAVE_AVX2) 104 zfs_fletcher_avx_t avx[4]; 105 #endif 106 #if defined(__x86_64) && defined(HAVE_AVX512F) 107 zfs_fletcher_avx512_t avx512[4]; 108 #endif 109 #if defined(__aarch64__) 110 zfs_fletcher_aarch64_neon_t aarch64_neon[4]; 111 #endif 112 } fletcher_4_ctx_t; 113 114 /* 115 * fletcher checksum struct 116 */ 117 typedef void (*fletcher_4_init_f)(fletcher_4_ctx_t *); 118 typedef void (*fletcher_4_fini_f)(fletcher_4_ctx_t *, zio_cksum_t *); 119 typedef void (*fletcher_4_compute_f)(fletcher_4_ctx_t *, 120 const void *, uint64_t); 121 122 typedef struct fletcher_4_func { 123 fletcher_4_init_f init_native; 124 fletcher_4_fini_f fini_native; 125 fletcher_4_compute_f compute_native; 126 fletcher_4_init_f init_byteswap; 127 fletcher_4_fini_f fini_byteswap; 128 fletcher_4_compute_f compute_byteswap; 129 boolean_t (*valid)(void); 130 boolean_t uses_fpu; 131 const char *name; 132 } __attribute__((aligned(64))) fletcher_4_ops_t; 133 134 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar_ops; 135 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar4_ops; 136 137 #if defined(HAVE_SSE2) 138 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_sse2_ops; 139 #endif 140 141 #if defined(HAVE_SSE2) && defined(HAVE_SSSE3) 142 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_ssse3_ops; 143 #endif 144 145 #if defined(HAVE_AVX) && defined(HAVE_AVX2) 146 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx2_ops; 147 #endif 148 149 #if defined(__x86_64) && defined(HAVE_AVX512F) 150 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512f_ops; 151 #endif 152 153 #if defined(__x86_64) && defined(HAVE_AVX512BW) 154 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512bw_ops; 155 #endif 156 157 #if defined(__aarch64__) 158 _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_aarch64_neon_ops; 159 #endif 160 161 #ifdef __cplusplus 162 } 163 #endif 164 165 #endif /* _ZFS_FLETCHER_H */ 166