1/* 2 * ChaCha/XChaCha NEON helper functions 3 * 4 * Copyright (C) 2016-2018 Linaro, Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * Originally based on: 11 * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions 12 * 13 * Copyright (C) 2015 Martin Willi 14 * 15 * This program is free software; you can redistribute it and/or modify 16 * it under the terms of the GNU General Public License as published by 17 * the Free Software Foundation; either version 2 of the License, or 18 * (at your option) any later version. 19 */ 20 21#include <linux/linkage.h> 22#include <asm/assembler.h> 23#include <asm/cache.h> 24 25 .text 26 .align 6 27 28/* 29 * chacha_permute - permute one block 30 * 31 * Permute one 64-byte block where the state matrix is stored in the four NEON 32 * registers v0-v3. It performs matrix operations on four words in parallel, 33 * but requires shuffling to rearrange the words after each round. 34 * 35 * The round count is given in w3. 36 * 37 * Clobbers: w3, x10, v4, v12 38 */ 39SYM_FUNC_START_LOCAL(chacha_permute) 40 41 adr_l x10, ROT8 42 ld1 {v12.4s}, [x10] 43 44.Ldoubleround: 45 // x0 += x1, x3 = rotl32(x3 ^ x0, 16) 46 add v0.4s, v0.4s, v1.4s 47 eor v3.16b, v3.16b, v0.16b 48 rev32 v3.8h, v3.8h 49 50 // x2 += x3, x1 = rotl32(x1 ^ x2, 12) 51 add v2.4s, v2.4s, v3.4s 52 eor v4.16b, v1.16b, v2.16b 53 shl v1.4s, v4.4s, #12 54 sri v1.4s, v4.4s, #20 55 56 // x0 += x1, x3 = rotl32(x3 ^ x0, 8) 57 add v0.4s, v0.4s, v1.4s 58 eor v3.16b, v3.16b, v0.16b 59 tbl v3.16b, {v3.16b}, v12.16b 60 61 // x2 += x3, x1 = rotl32(x1 ^ x2, 7) 62 add v2.4s, v2.4s, v3.4s 63 eor v4.16b, v1.16b, v2.16b 64 shl v1.4s, v4.4s, #7 65 sri v1.4s, v4.4s, #25 66 67 // x1 = shuffle32(x1, MASK(0, 3, 2, 1)) 68 ext v1.16b, v1.16b, v1.16b, #4 69 // x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 70 ext v2.16b, v2.16b, v2.16b, #8 71 // x3 = shuffle32(x3, MASK(2, 1, 0, 3)) 72 ext v3.16b, v3.16b, v3.16b, #12 73 74 // x0 += x1, x3 = rotl32(x3 ^ x0, 16) 75 add v0.4s, v0.4s, v1.4s 76 eor v3.16b, v3.16b, v0.16b 77 rev32 v3.8h, v3.8h 78 79 // x2 += x3, x1 = rotl32(x1 ^ x2, 12) 80 add v2.4s, v2.4s, v3.4s 81 eor v4.16b, v1.16b, v2.16b 82 shl v1.4s, v4.4s, #12 83 sri v1.4s, v4.4s, #20 84 85 // x0 += x1, x3 = rotl32(x3 ^ x0, 8) 86 add v0.4s, v0.4s, v1.4s 87 eor v3.16b, v3.16b, v0.16b 88 tbl v3.16b, {v3.16b}, v12.16b 89 90 // x2 += x3, x1 = rotl32(x1 ^ x2, 7) 91 add v2.4s, v2.4s, v3.4s 92 eor v4.16b, v1.16b, v2.16b 93 shl v1.4s, v4.4s, #7 94 sri v1.4s, v4.4s, #25 95 96 // x1 = shuffle32(x1, MASK(2, 1, 0, 3)) 97 ext v1.16b, v1.16b, v1.16b, #12 98 // x2 = shuffle32(x2, MASK(1, 0, 3, 2)) 99 ext v2.16b, v2.16b, v2.16b, #8 100 // x3 = shuffle32(x3, MASK(0, 3, 2, 1)) 101 ext v3.16b, v3.16b, v3.16b, #4 102 103 subs w3, w3, #2 104 b.ne .Ldoubleround 105 106 ret 107SYM_FUNC_END(chacha_permute) 108 109SYM_FUNC_START(chacha_block_xor_neon) 110 // x0: Input state matrix, s 111 // x1: 1 data block output, o 112 // x2: 1 data block input, i 113 // w3: nrounds 114 115 stp x29, x30, [sp, #-16]! 116 mov x29, sp 117 118 // x0..3 = s0..3 119 ld1 {v0.4s-v3.4s}, [x0] 120 ld1 {v8.4s-v11.4s}, [x0] 121 122 bl chacha_permute 123 124 ld1 {v4.16b-v7.16b}, [x2] 125 126 // o0 = i0 ^ (x0 + s0) 127 add v0.4s, v0.4s, v8.4s 128 eor v0.16b, v0.16b, v4.16b 129 130 // o1 = i1 ^ (x1 + s1) 131 add v1.4s, v1.4s, v9.4s 132 eor v1.16b, v1.16b, v5.16b 133 134 // o2 = i2 ^ (x2 + s2) 135 add v2.4s, v2.4s, v10.4s 136 eor v2.16b, v2.16b, v6.16b 137 138 // o3 = i3 ^ (x3 + s3) 139 add v3.4s, v3.4s, v11.4s 140 eor v3.16b, v3.16b, v7.16b 141 142 st1 {v0.16b-v3.16b}, [x1] 143 144 ldp x29, x30, [sp], #16 145 ret 146SYM_FUNC_END(chacha_block_xor_neon) 147 148SYM_FUNC_START(hchacha_block_neon) 149 // x0: Input state matrix, s 150 // x1: output (8 32-bit words) 151 // w2: nrounds 152 153 stp x29, x30, [sp, #-16]! 154 mov x29, sp 155 156 ld1 {v0.4s-v3.4s}, [x0] 157 158 mov w3, w2 159 bl chacha_permute 160 161 st1 {v0.4s}, [x1], #16 162 st1 {v3.4s}, [x1] 163 164 ldp x29, x30, [sp], #16 165 ret 166SYM_FUNC_END(hchacha_block_neon) 167 168 a0 .req w12 169 a1 .req w13 170 a2 .req w14 171 a3 .req w15 172 a4 .req w16 173 a5 .req w17 174 a6 .req w19 175 a7 .req w20 176 a8 .req w21 177 a9 .req w22 178 a10 .req w23 179 a11 .req w24 180 a12 .req w25 181 a13 .req w26 182 a14 .req w27 183 a15 .req w28 184 185 .align 6 186SYM_FUNC_START(chacha_4block_xor_neon) 187 frame_push 10 188 189 // x0: Input state matrix, s 190 // x1: 4 data blocks output, o 191 // x2: 4 data blocks input, i 192 // w3: nrounds 193 // x4: byte count 194 195 adr_l x10, .Lpermute 196 and x5, x4, #63 197 add x10, x10, x5 198 199 // 200 // This function encrypts four consecutive ChaCha blocks by loading 201 // the state matrix in NEON registers four times. The algorithm performs 202 // each operation on the corresponding word of each state matrix, hence 203 // requires no word shuffling. For final XORing step we transpose the 204 // matrix by interleaving 32- and then 64-bit words, which allows us to 205 // do XOR in NEON registers. 206 // 207 // At the same time, a fifth block is encrypted in parallel using 208 // scalar registers 209 // 210 adr_l x9, CTRINC // ... and ROT8 211 ld1 {v30.4s-v31.4s}, [x9] 212 213 // x0..15[0-3] = s0..3[0..3] 214 add x8, x0, #16 215 ld4r { v0.4s- v3.4s}, [x0] 216 ld4r { v4.4s- v7.4s}, [x8], #16 217 ld4r { v8.4s-v11.4s}, [x8], #16 218 ld4r {v12.4s-v15.4s}, [x8] 219 220 mov a0, v0.s[0] 221 mov a1, v1.s[0] 222 mov a2, v2.s[0] 223 mov a3, v3.s[0] 224 mov a4, v4.s[0] 225 mov a5, v5.s[0] 226 mov a6, v6.s[0] 227 mov a7, v7.s[0] 228 mov a8, v8.s[0] 229 mov a9, v9.s[0] 230 mov a10, v10.s[0] 231 mov a11, v11.s[0] 232 mov a12, v12.s[0] 233 mov a13, v13.s[0] 234 mov a14, v14.s[0] 235 mov a15, v15.s[0] 236 237 // x12 += counter values 1-4 238 add v12.4s, v12.4s, v30.4s 239 240.Ldoubleround4: 241 // x0 += x4, x12 = rotl32(x12 ^ x0, 16) 242 // x1 += x5, x13 = rotl32(x13 ^ x1, 16) 243 // x2 += x6, x14 = rotl32(x14 ^ x2, 16) 244 // x3 += x7, x15 = rotl32(x15 ^ x3, 16) 245 add v0.4s, v0.4s, v4.4s 246 add a0, a0, a4 247 add v1.4s, v1.4s, v5.4s 248 add a1, a1, a5 249 add v2.4s, v2.4s, v6.4s 250 add a2, a2, a6 251 add v3.4s, v3.4s, v7.4s 252 add a3, a3, a7 253 254 eor v12.16b, v12.16b, v0.16b 255 eor a12, a12, a0 256 eor v13.16b, v13.16b, v1.16b 257 eor a13, a13, a1 258 eor v14.16b, v14.16b, v2.16b 259 eor a14, a14, a2 260 eor v15.16b, v15.16b, v3.16b 261 eor a15, a15, a3 262 263 rev32 v12.8h, v12.8h 264 ror a12, a12, #16 265 rev32 v13.8h, v13.8h 266 ror a13, a13, #16 267 rev32 v14.8h, v14.8h 268 ror a14, a14, #16 269 rev32 v15.8h, v15.8h 270 ror a15, a15, #16 271 272 // x8 += x12, x4 = rotl32(x4 ^ x8, 12) 273 // x9 += x13, x5 = rotl32(x5 ^ x9, 12) 274 // x10 += x14, x6 = rotl32(x6 ^ x10, 12) 275 // x11 += x15, x7 = rotl32(x7 ^ x11, 12) 276 add v8.4s, v8.4s, v12.4s 277 add a8, a8, a12 278 add v9.4s, v9.4s, v13.4s 279 add a9, a9, a13 280 add v10.4s, v10.4s, v14.4s 281 add a10, a10, a14 282 add v11.4s, v11.4s, v15.4s 283 add a11, a11, a15 284 285 eor v16.16b, v4.16b, v8.16b 286 eor a4, a4, a8 287 eor v17.16b, v5.16b, v9.16b 288 eor a5, a5, a9 289 eor v18.16b, v6.16b, v10.16b 290 eor a6, a6, a10 291 eor v19.16b, v7.16b, v11.16b 292 eor a7, a7, a11 293 294 shl v4.4s, v16.4s, #12 295 shl v5.4s, v17.4s, #12 296 shl v6.4s, v18.4s, #12 297 shl v7.4s, v19.4s, #12 298 299 sri v4.4s, v16.4s, #20 300 ror a4, a4, #20 301 sri v5.4s, v17.4s, #20 302 ror a5, a5, #20 303 sri v6.4s, v18.4s, #20 304 ror a6, a6, #20 305 sri v7.4s, v19.4s, #20 306 ror a7, a7, #20 307 308 // x0 += x4, x12 = rotl32(x12 ^ x0, 8) 309 // x1 += x5, x13 = rotl32(x13 ^ x1, 8) 310 // x2 += x6, x14 = rotl32(x14 ^ x2, 8) 311 // x3 += x7, x15 = rotl32(x15 ^ x3, 8) 312 add v0.4s, v0.4s, v4.4s 313 add a0, a0, a4 314 add v1.4s, v1.4s, v5.4s 315 add a1, a1, a5 316 add v2.4s, v2.4s, v6.4s 317 add a2, a2, a6 318 add v3.4s, v3.4s, v7.4s 319 add a3, a3, a7 320 321 eor v12.16b, v12.16b, v0.16b 322 eor a12, a12, a0 323 eor v13.16b, v13.16b, v1.16b 324 eor a13, a13, a1 325 eor v14.16b, v14.16b, v2.16b 326 eor a14, a14, a2 327 eor v15.16b, v15.16b, v3.16b 328 eor a15, a15, a3 329 330 tbl v12.16b, {v12.16b}, v31.16b 331 ror a12, a12, #24 332 tbl v13.16b, {v13.16b}, v31.16b 333 ror a13, a13, #24 334 tbl v14.16b, {v14.16b}, v31.16b 335 ror a14, a14, #24 336 tbl v15.16b, {v15.16b}, v31.16b 337 ror a15, a15, #24 338 339 // x8 += x12, x4 = rotl32(x4 ^ x8, 7) 340 // x9 += x13, x5 = rotl32(x5 ^ x9, 7) 341 // x10 += x14, x6 = rotl32(x6 ^ x10, 7) 342 // x11 += x15, x7 = rotl32(x7 ^ x11, 7) 343 add v8.4s, v8.4s, v12.4s 344 add a8, a8, a12 345 add v9.4s, v9.4s, v13.4s 346 add a9, a9, a13 347 add v10.4s, v10.4s, v14.4s 348 add a10, a10, a14 349 add v11.4s, v11.4s, v15.4s 350 add a11, a11, a15 351 352 eor v16.16b, v4.16b, v8.16b 353 eor a4, a4, a8 354 eor v17.16b, v5.16b, v9.16b 355 eor a5, a5, a9 356 eor v18.16b, v6.16b, v10.16b 357 eor a6, a6, a10 358 eor v19.16b, v7.16b, v11.16b 359 eor a7, a7, a11 360 361 shl v4.4s, v16.4s, #7 362 shl v5.4s, v17.4s, #7 363 shl v6.4s, v18.4s, #7 364 shl v7.4s, v19.4s, #7 365 366 sri v4.4s, v16.4s, #25 367 ror a4, a4, #25 368 sri v5.4s, v17.4s, #25 369 ror a5, a5, #25 370 sri v6.4s, v18.4s, #25 371 ror a6, a6, #25 372 sri v7.4s, v19.4s, #25 373 ror a7, a7, #25 374 375 // x0 += x5, x15 = rotl32(x15 ^ x0, 16) 376 // x1 += x6, x12 = rotl32(x12 ^ x1, 16) 377 // x2 += x7, x13 = rotl32(x13 ^ x2, 16) 378 // x3 += x4, x14 = rotl32(x14 ^ x3, 16) 379 add v0.4s, v0.4s, v5.4s 380 add a0, a0, a5 381 add v1.4s, v1.4s, v6.4s 382 add a1, a1, a6 383 add v2.4s, v2.4s, v7.4s 384 add a2, a2, a7 385 add v3.4s, v3.4s, v4.4s 386 add a3, a3, a4 387 388 eor v15.16b, v15.16b, v0.16b 389 eor a15, a15, a0 390 eor v12.16b, v12.16b, v1.16b 391 eor a12, a12, a1 392 eor v13.16b, v13.16b, v2.16b 393 eor a13, a13, a2 394 eor v14.16b, v14.16b, v3.16b 395 eor a14, a14, a3 396 397 rev32 v15.8h, v15.8h 398 ror a15, a15, #16 399 rev32 v12.8h, v12.8h 400 ror a12, a12, #16 401 rev32 v13.8h, v13.8h 402 ror a13, a13, #16 403 rev32 v14.8h, v14.8h 404 ror a14, a14, #16 405 406 // x10 += x15, x5 = rotl32(x5 ^ x10, 12) 407 // x11 += x12, x6 = rotl32(x6 ^ x11, 12) 408 // x8 += x13, x7 = rotl32(x7 ^ x8, 12) 409 // x9 += x14, x4 = rotl32(x4 ^ x9, 12) 410 add v10.4s, v10.4s, v15.4s 411 add a10, a10, a15 412 add v11.4s, v11.4s, v12.4s 413 add a11, a11, a12 414 add v8.4s, v8.4s, v13.4s 415 add a8, a8, a13 416 add v9.4s, v9.4s, v14.4s 417 add a9, a9, a14 418 419 eor v16.16b, v5.16b, v10.16b 420 eor a5, a5, a10 421 eor v17.16b, v6.16b, v11.16b 422 eor a6, a6, a11 423 eor v18.16b, v7.16b, v8.16b 424 eor a7, a7, a8 425 eor v19.16b, v4.16b, v9.16b 426 eor a4, a4, a9 427 428 shl v5.4s, v16.4s, #12 429 shl v6.4s, v17.4s, #12 430 shl v7.4s, v18.4s, #12 431 shl v4.4s, v19.4s, #12 432 433 sri v5.4s, v16.4s, #20 434 ror a5, a5, #20 435 sri v6.4s, v17.4s, #20 436 ror a6, a6, #20 437 sri v7.4s, v18.4s, #20 438 ror a7, a7, #20 439 sri v4.4s, v19.4s, #20 440 ror a4, a4, #20 441 442 // x0 += x5, x15 = rotl32(x15 ^ x0, 8) 443 // x1 += x6, x12 = rotl32(x12 ^ x1, 8) 444 // x2 += x7, x13 = rotl32(x13 ^ x2, 8) 445 // x3 += x4, x14 = rotl32(x14 ^ x3, 8) 446 add v0.4s, v0.4s, v5.4s 447 add a0, a0, a5 448 add v1.4s, v1.4s, v6.4s 449 add a1, a1, a6 450 add v2.4s, v2.4s, v7.4s 451 add a2, a2, a7 452 add v3.4s, v3.4s, v4.4s 453 add a3, a3, a4 454 455 eor v15.16b, v15.16b, v0.16b 456 eor a15, a15, a0 457 eor v12.16b, v12.16b, v1.16b 458 eor a12, a12, a1 459 eor v13.16b, v13.16b, v2.16b 460 eor a13, a13, a2 461 eor v14.16b, v14.16b, v3.16b 462 eor a14, a14, a3 463 464 tbl v15.16b, {v15.16b}, v31.16b 465 ror a15, a15, #24 466 tbl v12.16b, {v12.16b}, v31.16b 467 ror a12, a12, #24 468 tbl v13.16b, {v13.16b}, v31.16b 469 ror a13, a13, #24 470 tbl v14.16b, {v14.16b}, v31.16b 471 ror a14, a14, #24 472 473 // x10 += x15, x5 = rotl32(x5 ^ x10, 7) 474 // x11 += x12, x6 = rotl32(x6 ^ x11, 7) 475 // x8 += x13, x7 = rotl32(x7 ^ x8, 7) 476 // x9 += x14, x4 = rotl32(x4 ^ x9, 7) 477 add v10.4s, v10.4s, v15.4s 478 add a10, a10, a15 479 add v11.4s, v11.4s, v12.4s 480 add a11, a11, a12 481 add v8.4s, v8.4s, v13.4s 482 add a8, a8, a13 483 add v9.4s, v9.4s, v14.4s 484 add a9, a9, a14 485 486 eor v16.16b, v5.16b, v10.16b 487 eor a5, a5, a10 488 eor v17.16b, v6.16b, v11.16b 489 eor a6, a6, a11 490 eor v18.16b, v7.16b, v8.16b 491 eor a7, a7, a8 492 eor v19.16b, v4.16b, v9.16b 493 eor a4, a4, a9 494 495 shl v5.4s, v16.4s, #7 496 shl v6.4s, v17.4s, #7 497 shl v7.4s, v18.4s, #7 498 shl v4.4s, v19.4s, #7 499 500 sri v5.4s, v16.4s, #25 501 ror a5, a5, #25 502 sri v6.4s, v17.4s, #25 503 ror a6, a6, #25 504 sri v7.4s, v18.4s, #25 505 ror a7, a7, #25 506 sri v4.4s, v19.4s, #25 507 ror a4, a4, #25 508 509 subs w3, w3, #2 510 b.ne .Ldoubleround4 511 512 ld4r {v16.4s-v19.4s}, [x0], #16 513 ld4r {v20.4s-v23.4s}, [x0], #16 514 515 // x12 += counter values 0-3 516 add v12.4s, v12.4s, v30.4s 517 518 // x0[0-3] += s0[0] 519 // x1[0-3] += s0[1] 520 // x2[0-3] += s0[2] 521 // x3[0-3] += s0[3] 522 add v0.4s, v0.4s, v16.4s 523 mov w6, v16.s[0] 524 mov w7, v17.s[0] 525 add v1.4s, v1.4s, v17.4s 526 mov w8, v18.s[0] 527 mov w9, v19.s[0] 528 add v2.4s, v2.4s, v18.4s 529 add a0, a0, w6 530 add a1, a1, w7 531 add v3.4s, v3.4s, v19.4s 532 add a2, a2, w8 533 add a3, a3, w9 534CPU_BE( rev a0, a0 ) 535CPU_BE( rev a1, a1 ) 536CPU_BE( rev a2, a2 ) 537CPU_BE( rev a3, a3 ) 538 539 ld4r {v24.4s-v27.4s}, [x0], #16 540 ld4r {v28.4s-v31.4s}, [x0] 541 542 // x4[0-3] += s1[0] 543 // x5[0-3] += s1[1] 544 // x6[0-3] += s1[2] 545 // x7[0-3] += s1[3] 546 add v4.4s, v4.4s, v20.4s 547 mov w6, v20.s[0] 548 mov w7, v21.s[0] 549 add v5.4s, v5.4s, v21.4s 550 mov w8, v22.s[0] 551 mov w9, v23.s[0] 552 add v6.4s, v6.4s, v22.4s 553 add a4, a4, w6 554 add a5, a5, w7 555 add v7.4s, v7.4s, v23.4s 556 add a6, a6, w8 557 add a7, a7, w9 558CPU_BE( rev a4, a4 ) 559CPU_BE( rev a5, a5 ) 560CPU_BE( rev a6, a6 ) 561CPU_BE( rev a7, a7 ) 562 563 // x8[0-3] += s2[0] 564 // x9[0-3] += s2[1] 565 // x10[0-3] += s2[2] 566 // x11[0-3] += s2[3] 567 add v8.4s, v8.4s, v24.4s 568 mov w6, v24.s[0] 569 mov w7, v25.s[0] 570 add v9.4s, v9.4s, v25.4s 571 mov w8, v26.s[0] 572 mov w9, v27.s[0] 573 add v10.4s, v10.4s, v26.4s 574 add a8, a8, w6 575 add a9, a9, w7 576 add v11.4s, v11.4s, v27.4s 577 add a10, a10, w8 578 add a11, a11, w9 579CPU_BE( rev a8, a8 ) 580CPU_BE( rev a9, a9 ) 581CPU_BE( rev a10, a10 ) 582CPU_BE( rev a11, a11 ) 583 584 // x12[0-3] += s3[0] 585 // x13[0-3] += s3[1] 586 // x14[0-3] += s3[2] 587 // x15[0-3] += s3[3] 588 add v12.4s, v12.4s, v28.4s 589 mov w6, v28.s[0] 590 mov w7, v29.s[0] 591 add v13.4s, v13.4s, v29.4s 592 mov w8, v30.s[0] 593 mov w9, v31.s[0] 594 add v14.4s, v14.4s, v30.4s 595 add a12, a12, w6 596 add a13, a13, w7 597 add v15.4s, v15.4s, v31.4s 598 add a14, a14, w8 599 add a15, a15, w9 600CPU_BE( rev a12, a12 ) 601CPU_BE( rev a13, a13 ) 602CPU_BE( rev a14, a14 ) 603CPU_BE( rev a15, a15 ) 604 605 // interleave 32-bit words in state n, n+1 606 ldp w6, w7, [x2], #64 607 zip1 v16.4s, v0.4s, v1.4s 608 ldp w8, w9, [x2, #-56] 609 eor a0, a0, w6 610 zip2 v17.4s, v0.4s, v1.4s 611 eor a1, a1, w7 612 zip1 v18.4s, v2.4s, v3.4s 613 eor a2, a2, w8 614 zip2 v19.4s, v2.4s, v3.4s 615 eor a3, a3, w9 616 ldp w6, w7, [x2, #-48] 617 zip1 v20.4s, v4.4s, v5.4s 618 ldp w8, w9, [x2, #-40] 619 eor a4, a4, w6 620 zip2 v21.4s, v4.4s, v5.4s 621 eor a5, a5, w7 622 zip1 v22.4s, v6.4s, v7.4s 623 eor a6, a6, w8 624 zip2 v23.4s, v6.4s, v7.4s 625 eor a7, a7, w9 626 ldp w6, w7, [x2, #-32] 627 zip1 v24.4s, v8.4s, v9.4s 628 ldp w8, w9, [x2, #-24] 629 eor a8, a8, w6 630 zip2 v25.4s, v8.4s, v9.4s 631 eor a9, a9, w7 632 zip1 v26.4s, v10.4s, v11.4s 633 eor a10, a10, w8 634 zip2 v27.4s, v10.4s, v11.4s 635 eor a11, a11, w9 636 ldp w6, w7, [x2, #-16] 637 zip1 v28.4s, v12.4s, v13.4s 638 ldp w8, w9, [x2, #-8] 639 eor a12, a12, w6 640 zip2 v29.4s, v12.4s, v13.4s 641 eor a13, a13, w7 642 zip1 v30.4s, v14.4s, v15.4s 643 eor a14, a14, w8 644 zip2 v31.4s, v14.4s, v15.4s 645 eor a15, a15, w9 646 647 add x3, x2, x4 648 sub x3, x3, #128 // start of last block 649 650 subs x5, x4, #128 651 csel x2, x2, x3, ge 652 653 // interleave 64-bit words in state n, n+2 654 zip1 v0.2d, v16.2d, v18.2d 655 zip2 v4.2d, v16.2d, v18.2d 656 stp a0, a1, [x1], #64 657 zip1 v8.2d, v17.2d, v19.2d 658 zip2 v12.2d, v17.2d, v19.2d 659 stp a2, a3, [x1, #-56] 660 661 subs x6, x4, #192 662 ld1 {v16.16b-v19.16b}, [x2], #64 663 csel x2, x2, x3, ge 664 665 zip1 v1.2d, v20.2d, v22.2d 666 zip2 v5.2d, v20.2d, v22.2d 667 stp a4, a5, [x1, #-48] 668 zip1 v9.2d, v21.2d, v23.2d 669 zip2 v13.2d, v21.2d, v23.2d 670 stp a6, a7, [x1, #-40] 671 672 subs x7, x4, #256 673 ld1 {v20.16b-v23.16b}, [x2], #64 674 csel x2, x2, x3, ge 675 676 zip1 v2.2d, v24.2d, v26.2d 677 zip2 v6.2d, v24.2d, v26.2d 678 stp a8, a9, [x1, #-32] 679 zip1 v10.2d, v25.2d, v27.2d 680 zip2 v14.2d, v25.2d, v27.2d 681 stp a10, a11, [x1, #-24] 682 683 subs x8, x4, #320 684 ld1 {v24.16b-v27.16b}, [x2], #64 685 csel x2, x2, x3, ge 686 687 zip1 v3.2d, v28.2d, v30.2d 688 zip2 v7.2d, v28.2d, v30.2d 689 stp a12, a13, [x1, #-16] 690 zip1 v11.2d, v29.2d, v31.2d 691 zip2 v15.2d, v29.2d, v31.2d 692 stp a14, a15, [x1, #-8] 693 694 tbnz x5, #63, .Lt128 695 ld1 {v28.16b-v31.16b}, [x2] 696 697 // xor with corresponding input, write to output 698 eor v16.16b, v16.16b, v0.16b 699 eor v17.16b, v17.16b, v1.16b 700 eor v18.16b, v18.16b, v2.16b 701 eor v19.16b, v19.16b, v3.16b 702 703 tbnz x6, #63, .Lt192 704 705 eor v20.16b, v20.16b, v4.16b 706 eor v21.16b, v21.16b, v5.16b 707 eor v22.16b, v22.16b, v6.16b 708 eor v23.16b, v23.16b, v7.16b 709 710 st1 {v16.16b-v19.16b}, [x1], #64 711 tbnz x7, #63, .Lt256 712 713 eor v24.16b, v24.16b, v8.16b 714 eor v25.16b, v25.16b, v9.16b 715 eor v26.16b, v26.16b, v10.16b 716 eor v27.16b, v27.16b, v11.16b 717 718 st1 {v20.16b-v23.16b}, [x1], #64 719 tbnz x8, #63, .Lt320 720 721 eor v28.16b, v28.16b, v12.16b 722 eor v29.16b, v29.16b, v13.16b 723 eor v30.16b, v30.16b, v14.16b 724 eor v31.16b, v31.16b, v15.16b 725 726 st1 {v24.16b-v27.16b}, [x1], #64 727 st1 {v28.16b-v31.16b}, [x1] 728 729.Lout: frame_pop 730 ret 731 732 // fewer than 192 bytes of in/output 733.Lt192: cbz x5, 1f // exactly 128 bytes? 734 ld1 {v28.16b-v31.16b}, [x10] 735 add x5, x5, x1 736 tbl v28.16b, {v4.16b-v7.16b}, v28.16b 737 tbl v29.16b, {v4.16b-v7.16b}, v29.16b 738 tbl v30.16b, {v4.16b-v7.16b}, v30.16b 739 tbl v31.16b, {v4.16b-v7.16b}, v31.16b 740 7410: eor v20.16b, v20.16b, v28.16b 742 eor v21.16b, v21.16b, v29.16b 743 eor v22.16b, v22.16b, v30.16b 744 eor v23.16b, v23.16b, v31.16b 745 st1 {v20.16b-v23.16b}, [x5] // overlapping stores 7461: st1 {v16.16b-v19.16b}, [x1] 747 b .Lout 748 749 // fewer than 128 bytes of in/output 750.Lt128: ld1 {v28.16b-v31.16b}, [x10] 751 add x5, x5, x1 752 sub x1, x1, #64 753 tbl v28.16b, {v0.16b-v3.16b}, v28.16b 754 tbl v29.16b, {v0.16b-v3.16b}, v29.16b 755 tbl v30.16b, {v0.16b-v3.16b}, v30.16b 756 tbl v31.16b, {v0.16b-v3.16b}, v31.16b 757 ld1 {v16.16b-v19.16b}, [x1] // reload first output block 758 b 0b 759 760 // fewer than 256 bytes of in/output 761.Lt256: cbz x6, 2f // exactly 192 bytes? 762 ld1 {v4.16b-v7.16b}, [x10] 763 add x6, x6, x1 764 tbl v0.16b, {v8.16b-v11.16b}, v4.16b 765 tbl v1.16b, {v8.16b-v11.16b}, v5.16b 766 tbl v2.16b, {v8.16b-v11.16b}, v6.16b 767 tbl v3.16b, {v8.16b-v11.16b}, v7.16b 768 769 eor v28.16b, v28.16b, v0.16b 770 eor v29.16b, v29.16b, v1.16b 771 eor v30.16b, v30.16b, v2.16b 772 eor v31.16b, v31.16b, v3.16b 773 st1 {v28.16b-v31.16b}, [x6] // overlapping stores 7742: st1 {v20.16b-v23.16b}, [x1] 775 b .Lout 776 777 // fewer than 320 bytes of in/output 778.Lt320: cbz x7, 3f // exactly 256 bytes? 779 ld1 {v4.16b-v7.16b}, [x10] 780 add x7, x7, x1 781 tbl v0.16b, {v12.16b-v15.16b}, v4.16b 782 tbl v1.16b, {v12.16b-v15.16b}, v5.16b 783 tbl v2.16b, {v12.16b-v15.16b}, v6.16b 784 tbl v3.16b, {v12.16b-v15.16b}, v7.16b 785 786 eor v28.16b, v28.16b, v0.16b 787 eor v29.16b, v29.16b, v1.16b 788 eor v30.16b, v30.16b, v2.16b 789 eor v31.16b, v31.16b, v3.16b 790 st1 {v28.16b-v31.16b}, [x7] // overlapping stores 7913: st1 {v24.16b-v27.16b}, [x1] 792 b .Lout 793SYM_FUNC_END(chacha_4block_xor_neon) 794 795 .section ".rodata", "a", %progbits 796 .align L1_CACHE_SHIFT 797.Lpermute: 798 .set .Li, 0 799 .rept 128 800 .byte (.Li - 64) 801 .set .Li, .Li + 1 802 .endr 803 804CTRINC: .word 1, 2, 3, 4 805ROT8: .word 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f 806