1
2 #ifndef blake2b_compress_avx2_H
3 #define blake2b_compress_avx2_H
4
5 #define LOADU128(p) _mm_loadu_si128((__m128i *) (p))
6 #define STOREU128(p, r) _mm_storeu_si128((__m128i *) (p), r)
7
8 #define LOAD(p) _mm256_load_si256((__m256i *) (p))
9 #define STORE(p, r) _mm256_store_si256((__m256i *) (p), r)
10
11 #define LOADU(p) _mm256_loadu_si256((__m256i *) (p))
12 #define STOREU(p, r) _mm256_storeu_si256((__m256i *) (p), r)
13
14 static inline uint64_t
LOADU64(const void * p)15 LOADU64(const void *p)
16 {
17 uint64_t v;
18 memcpy(&v, p, sizeof v);
19 return v;
20 }
21
22 #define ROTATE16 \
23 _mm256_setr_epi8(2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, 2, \
24 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9)
25
26 #define ROTATE24 \
27 _mm256_setr_epi8(3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, 3, \
28 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10)
29
30 #define ADD(a, b) _mm256_add_epi64(a, b)
31 #define SUB(a, b) _mm256_sub_epi64(a, b)
32
33 #define XOR(a, b) _mm256_xor_si256(a, b)
34 #define AND(a, b) _mm256_and_si256(a, b)
35 #define OR(a, b) _mm256_or_si256(a, b)
36
37 #define ROT32(x) _mm256_shuffle_epi32((x), _MM_SHUFFLE(2, 3, 0, 1))
38 #define ROT24(x) _mm256_shuffle_epi8((x), ROTATE24)
39 #define ROT16(x) _mm256_shuffle_epi8((x), ROTATE16)
40 #define ROT63(x) _mm256_or_si256(_mm256_srli_epi64((x), 63), ADD((x), (x)))
41
42 #define BLAKE2B_G1_V1(a, b, c, d, m) \
43 do { \
44 a = ADD(a, m); \
45 a = ADD(a, b); \
46 d = XOR(d, a); \
47 d = ROT32(d); \
48 c = ADD(c, d); \
49 b = XOR(b, c); \
50 b = ROT24(b); \
51 } while (0)
52
53 #define BLAKE2B_G2_V1(a, b, c, d, m) \
54 do { \
55 a = ADD(a, m); \
56 a = ADD(a, b); \
57 d = XOR(d, a); \
58 d = ROT16(d); \
59 c = ADD(c, d); \
60 b = XOR(b, c); \
61 b = ROT63(b); \
62 } while (0)
63
64 #define BLAKE2B_DIAG_V1(a, b, c, d) \
65 do { \
66 d = _mm256_permute4x64_epi64(d, _MM_SHUFFLE(2, 1, 0, 3)); \
67 c = _mm256_permute4x64_epi64(c, _MM_SHUFFLE(1, 0, 3, 2)); \
68 b = _mm256_permute4x64_epi64(b, _MM_SHUFFLE(0, 3, 2, 1)); \
69 } while (0)
70
71 #define BLAKE2B_UNDIAG_V1(a, b, c, d) \
72 do { \
73 d = _mm256_permute4x64_epi64(d, _MM_SHUFFLE(0, 3, 2, 1)); \
74 c = _mm256_permute4x64_epi64(c, _MM_SHUFFLE(1, 0, 3, 2)); \
75 b = _mm256_permute4x64_epi64(b, _MM_SHUFFLE(2, 1, 0, 3)); \
76 } while (0)
77
78 #include "blake2b-load-avx2.h"
79
80 #define BLAKE2B_ROUND_V1(a, b, c, d, r, m) \
81 do { \
82 __m256i b0; \
83 BLAKE2B_LOAD_MSG_##r##_1(b0); \
84 BLAKE2B_G1_V1(a, b, c, d, b0); \
85 BLAKE2B_LOAD_MSG_##r##_2(b0); \
86 BLAKE2B_G2_V1(a, b, c, d, b0); \
87 BLAKE2B_DIAG_V1(a, b, c, d); \
88 BLAKE2B_LOAD_MSG_##r##_3(b0); \
89 BLAKE2B_G1_V1(a, b, c, d, b0); \
90 BLAKE2B_LOAD_MSG_##r##_4(b0); \
91 BLAKE2B_G2_V1(a, b, c, d, b0); \
92 BLAKE2B_UNDIAG_V1(a, b, c, d); \
93 } while (0)
94
95 #define BLAKE2B_ROUNDS_V1(a, b, c, d, m) \
96 do { \
97 BLAKE2B_ROUND_V1(a, b, c, d, 0, (m)); \
98 BLAKE2B_ROUND_V1(a, b, c, d, 1, (m)); \
99 BLAKE2B_ROUND_V1(a, b, c, d, 2, (m)); \
100 BLAKE2B_ROUND_V1(a, b, c, d, 3, (m)); \
101 BLAKE2B_ROUND_V1(a, b, c, d, 4, (m)); \
102 BLAKE2B_ROUND_V1(a, b, c, d, 5, (m)); \
103 BLAKE2B_ROUND_V1(a, b, c, d, 6, (m)); \
104 BLAKE2B_ROUND_V1(a, b, c, d, 7, (m)); \
105 BLAKE2B_ROUND_V1(a, b, c, d, 8, (m)); \
106 BLAKE2B_ROUND_V1(a, b, c, d, 9, (m)); \
107 BLAKE2B_ROUND_V1(a, b, c, d, 10, (m)); \
108 BLAKE2B_ROUND_V1(a, b, c, d, 11, (m)); \
109 } while (0)
110
111 #define DECLARE_MESSAGE_WORDS(m) \
112 const __m256i m0 = _mm256_broadcastsi128_si256(LOADU128((m) + 0)); \
113 const __m256i m1 = _mm256_broadcastsi128_si256(LOADU128((m) + 16)); \
114 const __m256i m2 = _mm256_broadcastsi128_si256(LOADU128((m) + 32)); \
115 const __m256i m3 = _mm256_broadcastsi128_si256(LOADU128((m) + 48)); \
116 const __m256i m4 = _mm256_broadcastsi128_si256(LOADU128((m) + 64)); \
117 const __m256i m5 = _mm256_broadcastsi128_si256(LOADU128((m) + 80)); \
118 const __m256i m6 = _mm256_broadcastsi128_si256(LOADU128((m) + 96)); \
119 const __m256i m7 = _mm256_broadcastsi128_si256(LOADU128((m) + 112)); \
120 __m256i t0, t1;
121
122 #define BLAKE2B_COMPRESS_V1(a, b, m, t0, t1, f0, f1) \
123 do { \
124 DECLARE_MESSAGE_WORDS(m) \
125 const __m256i iv0 = a; \
126 const __m256i iv1 = b; \
127 __m256i c = LOAD(&blake2b_IV[0]); \
128 __m256i d = \
129 XOR(LOAD(&blake2b_IV[4]), _mm256_set_epi64x(f1, f0, t1, t0)); \
130 BLAKE2B_ROUNDS_V1(a, b, c, d, m); \
131 a = XOR(a, c); \
132 b = XOR(b, d); \
133 a = XOR(a, iv0); \
134 b = XOR(b, iv1); \
135 } while (0)
136
137 #endif
138