1
2 #include <stdint.h>
3 #include <stdlib.h>
4 #include <string.h>
5
6 #include "core.h"
7 #include "crypto_stream_chacha20.h"
8 #include "private/common.h"
9 #include "private/sse2_64_32.h"
10 #include "utils.h"
11
12 #if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_EMMINTRIN_H) && \
13 defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H)
14
15 # ifdef __GNUC__
16 # pragma GCC target("sse2")
17 # pragma GCC target("ssse3")
18 # pragma GCC target("sse4.1")
19 # pragma GCC target("avx2")
20 # endif
21
22 # include <emmintrin.h>
23 # include <immintrin.h>
24 # include <smmintrin.h>
25 # include <tmmintrin.h>
26
27 # include "../stream_chacha20.h"
28 # include "chacha20_dolbeau-avx2.h"
29
30 # define ROUNDS 20
31
32 typedef struct chacha_ctx {
33 uint32_t input[16];
34 } chacha_ctx;
35
36 static void
chacha_keysetup(chacha_ctx * ctx,const uint8_t * k)37 chacha_keysetup(chacha_ctx *ctx, const uint8_t *k)
38 {
39 ctx->input[0] = 0x61707865;
40 ctx->input[1] = 0x3320646e;
41 ctx->input[2] = 0x79622d32;
42 ctx->input[3] = 0x6b206574;
43 ctx->input[4] = LOAD32_LE(k + 0);
44 ctx->input[5] = LOAD32_LE(k + 4);
45 ctx->input[6] = LOAD32_LE(k + 8);
46 ctx->input[7] = LOAD32_LE(k + 12);
47 ctx->input[8] = LOAD32_LE(k + 16);
48 ctx->input[9] = LOAD32_LE(k + 20);
49 ctx->input[10] = LOAD32_LE(k + 24);
50 ctx->input[11] = LOAD32_LE(k + 28);
51 }
52
53 static void
chacha_ivsetup(chacha_ctx * ctx,const uint8_t * iv,const uint8_t * counter)54 chacha_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter)
55 {
56 ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter + 0);
57 ctx->input[13] = counter == NULL ? 0 : LOAD32_LE(counter + 4);
58 ctx->input[14] = LOAD32_LE(iv + 0);
59 ctx->input[15] = LOAD32_LE(iv + 4);
60 }
61
62 static void
chacha_ietf_ivsetup(chacha_ctx * ctx,const uint8_t * iv,const uint8_t * counter)63 chacha_ietf_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter)
64 {
65 ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter);
66 ctx->input[13] = LOAD32_LE(iv + 0);
67 ctx->input[14] = LOAD32_LE(iv + 4);
68 ctx->input[15] = LOAD32_LE(iv + 8);
69 }
70
71 static void
chacha20_encrypt_bytes(chacha_ctx * ctx,const uint8_t * m,uint8_t * c,unsigned long long bytes)72 chacha20_encrypt_bytes(chacha_ctx *ctx, const uint8_t *m, uint8_t *c,
73 unsigned long long bytes)
74 {
75 uint32_t * const x = &ctx->input[0];
76
77 if (!bytes) {
78 return; /* LCOV_EXCL_LINE */
79 }
80 if (bytes > crypto_stream_chacha20_MESSAGEBYTES_MAX) {
81 sodium_misuse();
82 }
83 # include "u8.h"
84 # include "u4.h"
85 # include "u1.h"
86 # include "u0.h"
87 }
88
89 static int
stream_ref(unsigned char * c,unsigned long long clen,const unsigned char * n,const unsigned char * k)90 stream_ref(unsigned char *c, unsigned long long clen, const unsigned char *n,
91 const unsigned char *k)
92 {
93 struct chacha_ctx ctx;
94
95 if (!clen) {
96 return 0;
97 }
98 COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8);
99 chacha_keysetup(&ctx, k);
100 chacha_ivsetup(&ctx, n, NULL);
101 memset(c, 0, clen);
102 chacha20_encrypt_bytes(&ctx, c, c, clen);
103 sodium_memzero(&ctx, sizeof ctx);
104
105 return 0;
106 }
107
108 static int
stream_ietf_ref(unsigned char * c,unsigned long long clen,const unsigned char * n,const unsigned char * k)109 stream_ietf_ref(unsigned char *c, unsigned long long clen,
110 const unsigned char *n, const unsigned char *k)
111 {
112 struct chacha_ctx ctx;
113
114 if (!clen) {
115 return 0;
116 }
117 COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8);
118 chacha_keysetup(&ctx, k);
119 chacha_ietf_ivsetup(&ctx, n, NULL);
120 memset(c, 0, clen);
121 chacha20_encrypt_bytes(&ctx, c, c, clen);
122 sodium_memzero(&ctx, sizeof ctx);
123
124 return 0;
125 }
126
127 static int
stream_ref_xor_ic(unsigned char * c,const unsigned char * m,unsigned long long mlen,const unsigned char * n,uint64_t ic,const unsigned char * k)128 stream_ref_xor_ic(unsigned char *c, const unsigned char *m,
129 unsigned long long mlen, const unsigned char *n, uint64_t ic,
130 const unsigned char *k)
131 {
132 struct chacha_ctx ctx;
133 uint8_t ic_bytes[8];
134 uint32_t ic_high;
135 uint32_t ic_low;
136
137 if (!mlen) {
138 return 0;
139 }
140 ic_high = (uint32_t) (ic >> 32);
141 ic_low = (uint32_t) ic;
142 STORE32_LE(&ic_bytes[0], ic_low);
143 STORE32_LE(&ic_bytes[4], ic_high);
144 chacha_keysetup(&ctx, k);
145 chacha_ivsetup(&ctx, n, ic_bytes);
146 chacha20_encrypt_bytes(&ctx, m, c, mlen);
147 sodium_memzero(&ctx, sizeof ctx);
148
149 return 0;
150 }
151
152 static int
stream_ietf_ref_xor_ic(unsigned char * c,const unsigned char * m,unsigned long long mlen,const unsigned char * n,uint32_t ic,const unsigned char * k)153 stream_ietf_ref_xor_ic(unsigned char *c, const unsigned char *m,
154 unsigned long long mlen, const unsigned char *n,
155 uint32_t ic, const unsigned char *k)
156 {
157 struct chacha_ctx ctx;
158 uint8_t ic_bytes[4];
159
160 if (!mlen) {
161 return 0;
162 }
163 STORE32_LE(ic_bytes, ic);
164 chacha_keysetup(&ctx, k);
165 chacha_ietf_ivsetup(&ctx, n, ic_bytes);
166 chacha20_encrypt_bytes(&ctx, m, c, mlen);
167 sodium_memzero(&ctx, sizeof ctx);
168
169 return 0;
170 }
171
172 struct crypto_stream_chacha20_implementation
173 crypto_stream_chacha20_dolbeau_avx2_implementation = {
174 SODIUM_C99(.stream =) stream_ref,
175 SODIUM_C99(.stream_ietf =) stream_ietf_ref,
176 SODIUM_C99(.stream_xor_ic =) stream_ref_xor_ic,
177 SODIUM_C99(.stream_ietf_xor_ic =) stream_ietf_ref_xor_ic
178 };
179
180 #endif
181