1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3
25 * Copyright (c) 2019-2020 Samuel Neves and Jack O'Connor
26 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de>
27 */
28
29 #ifndef BLAKE3_IMPL_H
30 #define BLAKE3_IMPL_H
31
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35
36 #include <sys/blake3.h>
37 #include <sys/simd.h>
38 #include <sys/asm_linkage.h>
39
40 /*
41 * Methods used to define BLAKE3 assembler implementations
42 */
43 typedef void (*blake3_compress_in_place_f)(uint32_t cv[8],
44 const uint8_t block[BLAKE3_BLOCK_LEN],
45 uint8_t block_len, uint64_t counter,
46 uint8_t flags);
47
48 typedef void (*blake3_compress_xof_f)(const uint32_t cv[8],
49 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
50 uint64_t counter, uint8_t flags, uint8_t out[64]);
51
52 typedef void (*blake3_hash_many_f)(const uint8_t * const *inputs,
53 size_t num_inputs, size_t blocks, const uint32_t key[8],
54 uint64_t counter, boolean_t increment_counter, uint8_t flags,
55 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
56
57 typedef boolean_t (*blake3_is_supported_f)(void);
58
59 typedef struct {
60 blake3_compress_in_place_f compress_in_place;
61 blake3_compress_xof_f compress_xof;
62 blake3_hash_many_f hash_many;
63 blake3_is_supported_f is_supported;
64 int degree;
65 const char *name;
66 } blake3_ops_t;
67
68 /* return selected BLAKE3 implementation ops */
69 extern const blake3_ops_t *blake3_get_ops(void);
70
71 #if defined(__x86_64)
72 #define MAX_SIMD_DEGREE 16
73 #else
74 #define MAX_SIMD_DEGREE 4
75 #endif
76
77 #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
78
79 static const uint32_t BLAKE3_IV[8] = {
80 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL,
81 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL};
82
83 static const uint8_t BLAKE3_MSG_SCHEDULE[7][16] = {
84 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
85 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
86 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
87 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
88 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
89 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
90 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
91 };
92
93 /* Find index of the highest set bit */
highest_one(uint64_t x)94 static inline unsigned int highest_one(uint64_t x) {
95 #if defined(__GNUC__) || defined(__clang__)
96 return (63 ^ __builtin_clzll(x));
97 #elif defined(_MSC_VER) && defined(IS_X86_64)
98 unsigned long index;
99 _BitScanReverse64(&index, x);
100 return (index);
101 #elif defined(_MSC_VER) && defined(IS_X86_32)
102 if (x >> 32) {
103 unsigned long index;
104 _BitScanReverse(&index, x >> 32);
105 return (32 + index);
106 } else {
107 unsigned long index;
108 _BitScanReverse(&index, x);
109 return (index);
110 }
111 #else
112 unsigned int c = 0;
113 if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
114 if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
115 if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
116 if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
117 if (x & 0x000000000000000cULL) { x >>= 2; c += 2; }
118 if (x & 0x0000000000000002ULL) { c += 1; }
119 return (c);
120 #endif
121 }
122
123 /* Count the number of 1 bits. */
popcnt(uint64_t x)124 static inline unsigned int popcnt(uint64_t x) {
125 unsigned int count = 0;
126
127 while (x != 0) {
128 count += 1;
129 x &= x - 1;
130 }
131
132 return (count);
133 }
134
135 /*
136 * Largest power of two less than or equal to x.
137 * As a special case, returns 1 when x is 0.
138 */
round_down_to_power_of_2(uint64_t x)139 static inline uint64_t round_down_to_power_of_2(uint64_t x) {
140 return (1ULL << highest_one(x | 1));
141 }
142
counter_low(uint64_t counter)143 static inline uint32_t counter_low(uint64_t counter) {
144 return ((uint32_t)counter);
145 }
146
counter_high(uint64_t counter)147 static inline uint32_t counter_high(uint64_t counter) {
148 return ((uint32_t)(counter >> 32));
149 }
150
load32(const void * src)151 static inline uint32_t load32(const void *src) {
152 const uint8_t *p = (const uint8_t *)src;
153 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
154 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
155 }
156
load_key_words(const uint8_t key[BLAKE3_KEY_LEN],uint32_t key_words[8])157 static inline void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
158 uint32_t key_words[8]) {
159 key_words[0] = load32(&key[0 * 4]);
160 key_words[1] = load32(&key[1 * 4]);
161 key_words[2] = load32(&key[2 * 4]);
162 key_words[3] = load32(&key[3 * 4]);
163 key_words[4] = load32(&key[4 * 4]);
164 key_words[5] = load32(&key[5 * 4]);
165 key_words[6] = load32(&key[6 * 4]);
166 key_words[7] = load32(&key[7 * 4]);
167 }
168
store32(void * dst,uint32_t w)169 static inline void store32(void *dst, uint32_t w) {
170 uint8_t *p = (uint8_t *)dst;
171 p[0] = (uint8_t)(w >> 0);
172 p[1] = (uint8_t)(w >> 8);
173 p[2] = (uint8_t)(w >> 16);
174 p[3] = (uint8_t)(w >> 24);
175 }
176
store_cv_words(uint8_t bytes_out[32],uint32_t cv_words[8])177 static inline void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
178 store32(&bytes_out[0 * 4], cv_words[0]);
179 store32(&bytes_out[1 * 4], cv_words[1]);
180 store32(&bytes_out[2 * 4], cv_words[2]);
181 store32(&bytes_out[3 * 4], cv_words[3]);
182 store32(&bytes_out[4 * 4], cv_words[4]);
183 store32(&bytes_out[5 * 4], cv_words[5]);
184 store32(&bytes_out[6 * 4], cv_words[6]);
185 store32(&bytes_out[7 * 4], cv_words[7]);
186 }
187
188 #ifdef __cplusplus
189 }
190 #endif
191
192 #endif /* BLAKE3_IMPL_H */
193