1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2018 Google, Inc. 4 */ 5 6#include <linux/linkage.h> 7#include <asm/assembler.h> 8 9/* 10 * Design notes: 11 * 12 * 16 registers would be needed to hold the state matrix, but only 14 are 13 * available because 'sp' and 'pc' cannot be used. So we spill the elements 14 * (x8, x9) to the stack and swap them out with (x10, x11). This adds one 15 * 'ldrd' and one 'strd' instruction per round. 16 * 17 * All rotates are performed using the implicit rotate operand accepted by the 18 * 'add' and 'eor' instructions. This is faster than using explicit rotate 19 * instructions. To make this work, we allow the values in the second and last 20 * rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the 21 * wrong rotation amount. The rotation amount is then fixed up just in time 22 * when the values are used. 'brot' is the number of bits the values in row 'b' 23 * need to be rotated right to arrive at the correct values, and 'drot' 24 * similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such 25 * that they end up as (25, 24) after every round. 26 */ 27 28 // ChaCha state registers 29 X0 .req r0 30 X1 .req r1 31 X2 .req r2 32 X3 .req r3 33 X4 .req r4 34 X5 .req r5 35 X6 .req r6 36 X7 .req r7 37 X8_X10 .req r8 // shared by x8 and x10 38 X9_X11 .req r9 // shared by x9 and x11 39 X12 .req r10 40 X13 .req r11 41 X14 .req r12 42 X15 .req r14 43 44.macro _le32_bswap_4x a, b, c, d, tmp 45#ifdef __ARMEB__ 46 rev_l \a, \tmp 47 rev_l \b, \tmp 48 rev_l \c, \tmp 49 rev_l \d, \tmp 50#endif 51.endm 52 53.macro __ldrd a, b, src, offset 54#if __LINUX_ARM_ARCH__ >= 6 55 ldrd \a, \b, [\src, #\offset] 56#else 57 ldr \a, [\src, #\offset] 58 ldr \b, [\src, #\offset + 4] 59#endif 60.endm 61 62.macro __strd a, b, dst, offset 63#if __LINUX_ARM_ARCH__ >= 6 64 strd \a, \b, [\dst, #\offset] 65#else 66 str \a, [\dst, #\offset] 67 str \b, [\dst, #\offset + 4] 68#endif 69.endm 70 71.macro _halfround a1, b1, c1, d1, a2, b2, c2, d2 72 73 // a += b; d ^= a; d = rol(d, 16); 74 add \a1, \a1, \b1, ror #brot 75 add \a2, \a2, \b2, ror #brot 76 eor \d1, \a1, \d1, ror #drot 77 eor \d2, \a2, \d2, ror #drot 78 // drot == 32 - 16 == 16 79 80 // c += d; b ^= c; b = rol(b, 12); 81 add \c1, \c1, \d1, ror #16 82 add \c2, \c2, \d2, ror #16 83 eor \b1, \c1, \b1, ror #brot 84 eor \b2, \c2, \b2, ror #brot 85 // brot == 32 - 12 == 20 86 87 // a += b; d ^= a; d = rol(d, 8); 88 add \a1, \a1, \b1, ror #20 89 add \a2, \a2, \b2, ror #20 90 eor \d1, \a1, \d1, ror #16 91 eor \d2, \a2, \d2, ror #16 92 // drot == 32 - 8 == 24 93 94 // c += d; b ^= c; b = rol(b, 7); 95 add \c1, \c1, \d1, ror #24 96 add \c2, \c2, \d2, ror #24 97 eor \b1, \c1, \b1, ror #20 98 eor \b2, \c2, \b2, ror #20 99 // brot == 32 - 7 == 25 100.endm 101 102.macro _doubleround 103 104 // column round 105 106 // quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13) 107 _halfround X0, X4, X8_X10, X12, X1, X5, X9_X11, X13 108 109 // save (x8, x9); restore (x10, x11) 110 __strd X8_X10, X9_X11, sp, 0 111 __ldrd X8_X10, X9_X11, sp, 8 112 113 // quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15) 114 _halfround X2, X6, X8_X10, X14, X3, X7, X9_X11, X15 115 116 .set brot, 25 117 .set drot, 24 118 119 // diagonal round 120 121 // quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12) 122 _halfround X0, X5, X8_X10, X15, X1, X6, X9_X11, X12 123 124 // save (x10, x11); restore (x8, x9) 125 __strd X8_X10, X9_X11, sp, 8 126 __ldrd X8_X10, X9_X11, sp, 0 127 128 // quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14) 129 _halfround X2, X7, X8_X10, X13, X3, X4, X9_X11, X14 130.endm 131 132.macro _chacha_permute nrounds 133 .set brot, 0 134 .set drot, 0 135 .rept \nrounds / 2 136 _doubleround 137 .endr 138.endm 139 140.macro _chacha nrounds 141 142.Lnext_block\@: 143 // Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN 144 // Registers contain x0-x9,x12-x15. 145 146 // Do the core ChaCha permutation to update x0-x15. 147 _chacha_permute \nrounds 148 149 add sp, #8 150 // Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN 151 // Registers contain x0-x9,x12-x15. 152 // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. 153 154 // Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15). 155 push {X8_X10, X9_X11, X12, X13, X14, X15} 156 157 // Load (OUT, IN, LEN). 158 ldr r14, [sp, #96] 159 ldr r12, [sp, #100] 160 ldr r11, [sp, #104] 161 162 orr r10, r14, r12 163 164 // Use slow path if fewer than 64 bytes remain. 165 cmp r11, #64 166 blt .Lxor_slowpath\@ 167 168 // Use slow path if IN and/or OUT isn't 4-byte aligned. Needed even on 169 // ARMv6+, since ldmia and stmia (used below) still require alignment. 170 tst r10, #3 171 bne .Lxor_slowpath\@ 172 173 // Fast path: XOR 64 bytes of aligned data. 174 175 // Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN 176 // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT. 177 // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. 178 179 // x0-x3 180 __ldrd r8, r9, sp, 32 181 __ldrd r10, r11, sp, 40 182 add X0, X0, r8 183 add X1, X1, r9 184 add X2, X2, r10 185 add X3, X3, r11 186 _le32_bswap_4x X0, X1, X2, X3, r8 187 ldmia r12!, {r8-r11} 188 eor X0, X0, r8 189 eor X1, X1, r9 190 eor X2, X2, r10 191 eor X3, X3, r11 192 stmia r14!, {X0-X3} 193 194 // x4-x7 195 __ldrd r8, r9, sp, 48 196 __ldrd r10, r11, sp, 56 197 add X4, r8, X4, ror #brot 198 add X5, r9, X5, ror #brot 199 ldmia r12!, {X0-X3} 200 add X6, r10, X6, ror #brot 201 add X7, r11, X7, ror #brot 202 _le32_bswap_4x X4, X5, X6, X7, r8 203 eor X4, X4, X0 204 eor X5, X5, X1 205 eor X6, X6, X2 206 eor X7, X7, X3 207 stmia r14!, {X4-X7} 208 209 // x8-x15 210 pop {r0-r7} // (x8-x9,x12-x15,x10-x11) 211 __ldrd r8, r9, sp, 32 212 __ldrd r10, r11, sp, 40 213 add r0, r0, r8 // x8 214 add r1, r1, r9 // x9 215 add r6, r6, r10 // x10 216 add r7, r7, r11 // x11 217 _le32_bswap_4x r0, r1, r6, r7, r8 218 ldmia r12!, {r8-r11} 219 eor r0, r0, r8 // x8 220 eor r1, r1, r9 // x9 221 eor r6, r6, r10 // x10 222 eor r7, r7, r11 // x11 223 stmia r14!, {r0,r1,r6,r7} 224 ldmia r12!, {r0,r1,r6,r7} 225 __ldrd r8, r9, sp, 48 226 __ldrd r10, r11, sp, 56 227 add r2, r8, r2, ror #drot // x12 228 add r3, r9, r3, ror #drot // x13 229 add r4, r10, r4, ror #drot // x14 230 add r5, r11, r5, ror #drot // x15 231 _le32_bswap_4x r2, r3, r4, r5, r9 232 ldr r9, [sp, #72] // load LEN 233 eor r2, r2, r0 // x12 234 eor r3, r3, r1 // x13 235 eor r4, r4, r6 // x14 236 eor r5, r5, r7 // x15 237 subs r9, #64 // decrement and check LEN 238 stmia r14!, {r2-r5} 239 240 beq .Ldone\@ 241 242.Lprepare_for_next_block\@: 243 244 // Stack: x0-x15 OUT IN LEN 245 246 // Increment block counter (x12) 247 add r8, #1 248 249 // Store updated (OUT, IN, LEN) 250 str r14, [sp, #64] 251 str r12, [sp, #68] 252 str r9, [sp, #72] 253 254 mov r14, sp 255 256 // Store updated block counter (x12) 257 str r8, [sp, #48] 258 259 sub sp, #16 260 261 // Reload state and do next block 262 ldmia r14!, {r0-r11} // load x0-x11 263 __strd r10, r11, sp, 8 // store x10-x11 before state 264 ldmia r14, {r10-r12,r14} // load x12-x15 265 b .Lnext_block\@ 266 267.Lxor_slowpath\@: 268 // Slow path: < 64 bytes remaining, or unaligned input or output buffer. 269 // We handle it by storing the 64 bytes of keystream to the stack, then 270 // XOR-ing the needed portion with the data. 271 272 // Allocate keystream buffer 273 sub sp, #64 274 mov r14, sp 275 276 // Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN 277 // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0. 278 // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. 279 280 // Save keystream for x0-x3 281 __ldrd r8, r9, sp, 96 282 __ldrd r10, r11, sp, 104 283 add X0, X0, r8 284 add X1, X1, r9 285 add X2, X2, r10 286 add X3, X3, r11 287 _le32_bswap_4x X0, X1, X2, X3, r8 288 stmia r14!, {X0-X3} 289 290 // Save keystream for x4-x7 291 __ldrd r8, r9, sp, 112 292 __ldrd r10, r11, sp, 120 293 add X4, r8, X4, ror #brot 294 add X5, r9, X5, ror #brot 295 add X6, r10, X6, ror #brot 296 add X7, r11, X7, ror #brot 297 _le32_bswap_4x X4, X5, X6, X7, r8 298 add r8, sp, #64 299 stmia r14!, {X4-X7} 300 301 // Save keystream for x8-x15 302 ldm r8, {r0-r7} // (x8-x9,x12-x15,x10-x11) 303 __ldrd r8, r9, sp, 128 304 __ldrd r10, r11, sp, 136 305 add r0, r0, r8 // x8 306 add r1, r1, r9 // x9 307 add r6, r6, r10 // x10 308 add r7, r7, r11 // x11 309 _le32_bswap_4x r0, r1, r6, r7, r8 310 stmia r14!, {r0,r1,r6,r7} 311 __ldrd r8, r9, sp, 144 312 __ldrd r10, r11, sp, 152 313 add r2, r8, r2, ror #drot // x12 314 add r3, r9, r3, ror #drot // x13 315 add r4, r10, r4, ror #drot // x14 316 add r5, r11, r5, ror #drot // x15 317 _le32_bswap_4x r2, r3, r4, r5, r9 318 stmia r14, {r2-r5} 319 320 // Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN 321 // Registers: r8 is block counter, r12 is IN. 322 323 ldr r9, [sp, #168] // LEN 324 ldr r14, [sp, #160] // OUT 325 cmp r9, #64 326 mov r0, sp 327 movle r1, r9 328 movgt r1, #64 329 // r1 is number of bytes to XOR, in range [1, 64] 330 331.if __LINUX_ARM_ARCH__ < 6 332 orr r2, r12, r14 333 tst r2, #3 // IN or OUT misaligned? 334 bne .Lxor_next_byte\@ 335.endif 336 337 // XOR a word at a time 338.rept 16 339 subs r1, #4 340 blt .Lxor_words_done\@ 341 ldr r2, [r12], #4 342 ldr r3, [r0], #4 343 eor r2, r2, r3 344 str r2, [r14], #4 345.endr 346 b .Lxor_slowpath_done\@ 347.Lxor_words_done\@: 348 ands r1, r1, #3 349 beq .Lxor_slowpath_done\@ 350 351 // XOR a byte at a time 352.Lxor_next_byte\@: 353 ldrb r2, [r12], #1 354 ldrb r3, [r0], #1 355 eor r2, r2, r3 356 strb r2, [r14], #1 357 subs r1, #1 358 bne .Lxor_next_byte\@ 359 360.Lxor_slowpath_done\@: 361 subs r9, #64 362 add sp, #96 363 bgt .Lprepare_for_next_block\@ 364 365.Ldone\@: 366.endm // _chacha 367 368/* 369 * void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes, 370 * const u32 *state, int nrounds); 371 */ 372ENTRY(chacha_doarm) 373 cmp r2, #0 // len == 0? 374 reteq lr 375 376 ldr ip, [sp] 377 cmp ip, #12 378 379 push {r0-r2,r4-r11,lr} 380 381 // Push state x0-x15 onto stack. 382 // Also store an extra copy of x10-x11 just before the state. 383 384 add X12, r3, #48 385 ldm X12, {X12,X13,X14,X15} 386 push {X12,X13,X14,X15} 387 sub sp, sp, #64 388 389 __ldrd X8_X10, X9_X11, r3, 40 390 __strd X8_X10, X9_X11, sp, 8 391 __strd X8_X10, X9_X11, sp, 56 392 ldm r3, {X0-X9_X11} 393 __strd X0, X1, sp, 16 394 __strd X2, X3, sp, 24 395 __strd X4, X5, sp, 32 396 __strd X6, X7, sp, 40 397 __strd X8_X10, X9_X11, sp, 48 398 399 beq 1f 400 _chacha 20 401 4020: add sp, #76 403 pop {r4-r11, pc} 404 4051: _chacha 12 406 b 0b 407ENDPROC(chacha_doarm) 408 409/* 410 * void hchacha_block_arm(const u32 state[16], u32 out[8], int nrounds); 411 */ 412ENTRY(hchacha_block_arm) 413 push {r1,r4-r11,lr} 414 415 cmp r2, #12 // ChaCha12 ? 416 417 mov r14, r0 418 ldmia r14!, {r0-r11} // load x0-x11 419 push {r10-r11} // store x10-x11 to stack 420 ldm r14, {r10-r12,r14} // load x12-x15 421 sub sp, #8 422 423 beq 1f 424 _chacha_permute 20 425 426 // Skip over (unused0-unused1, x10-x11) 4270: add sp, #16 428 429 // Fix up rotations of x12-x15 430 ror X12, X12, #drot 431 ror X13, X13, #drot 432 pop {r4} // load 'out' 433 ror X14, X14, #drot 434 ror X15, X15, #drot 435 436 // Store (x0-x3,x12-x15) to 'out' 437 stm r4, {X0,X1,X2,X3,X12,X13,X14,X15} 438 439 pop {r4-r11,pc} 440 4411: _chacha_permute 12 442 b 0b 443ENDPROC(hchacha_block_arm) 444