1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * sha2-ce-core.S - core SHA-224/SHA-256 transform using v8 Crypto Extensions 4 * 5 * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8#include <linux/linkage.h> 9#include <asm/assembler.h> 10 11 .text 12 .arch armv8-a+crypto 13 14 dga .req q20 15 dgav .req v20 16 dgb .req q21 17 dgbv .req v21 18 19 t0 .req v22 20 t1 .req v23 21 22 dg0q .req q24 23 dg0v .req v24 24 dg1q .req q25 25 dg1v .req v25 26 dg2q .req q26 27 dg2v .req v26 28 29 .macro add_only, ev, rc, s0 30 mov dg2v.16b, dg0v.16b 31 .ifeq \ev 32 add t1.4s, v\s0\().4s, \rc\().4s 33 sha256h dg0q, dg1q, t0.4s 34 sha256h2 dg1q, dg2q, t0.4s 35 .else 36 .ifnb \s0 37 add t0.4s, v\s0\().4s, \rc\().4s 38 .endif 39 sha256h dg0q, dg1q, t1.4s 40 sha256h2 dg1q, dg2q, t1.4s 41 .endif 42 .endm 43 44 .macro add_update, ev, rc, s0, s1, s2, s3 45 sha256su0 v\s0\().4s, v\s1\().4s 46 add_only \ev, \rc, \s1 47 sha256su1 v\s0\().4s, v\s2\().4s, v\s3\().4s 48 .endm 49 50 /* 51 * The SHA-256 round constants 52 */ 53 .section ".rodata", "a" 54 .align 4 55.Lsha2_rcon: 56 .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 57 .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 58 .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 59 .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 60 .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc 61 .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da 62 .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 63 .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 64 .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 65 .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 66 .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 67 .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 68 .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 69 .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 70 .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 71 .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 72 73 .macro load_round_constants tmp 74 adr_l \tmp, .Lsha2_rcon 75 ld1 { v0.4s- v3.4s}, [\tmp], #64 76 ld1 { v4.4s- v7.4s}, [\tmp], #64 77 ld1 { v8.4s-v11.4s}, [\tmp], #64 78 ld1 {v12.4s-v15.4s}, [\tmp] 79 .endm 80 81 /* 82 * size_t __sha256_ce_transform(struct sha256_block_state *state, 83 * const u8 *data, size_t nblocks); 84 */ 85 .text 86SYM_FUNC_START(__sha256_ce_transform) 87 88 load_round_constants x8 89 90 /* load state */ 91 ld1 {dgav.4s, dgbv.4s}, [x0] 92 93 /* load input */ 940: ld1 {v16.4s-v19.4s}, [x1], #64 95 sub x2, x2, #1 96 97CPU_LE( rev32 v16.16b, v16.16b ) 98CPU_LE( rev32 v17.16b, v17.16b ) 99CPU_LE( rev32 v18.16b, v18.16b ) 100CPU_LE( rev32 v19.16b, v19.16b ) 101 102 add t0.4s, v16.4s, v0.4s 103 mov dg0v.16b, dgav.16b 104 mov dg1v.16b, dgbv.16b 105 106 add_update 0, v1, 16, 17, 18, 19 107 add_update 1, v2, 17, 18, 19, 16 108 add_update 0, v3, 18, 19, 16, 17 109 add_update 1, v4, 19, 16, 17, 18 110 111 add_update 0, v5, 16, 17, 18, 19 112 add_update 1, v6, 17, 18, 19, 16 113 add_update 0, v7, 18, 19, 16, 17 114 add_update 1, v8, 19, 16, 17, 18 115 116 add_update 0, v9, 16, 17, 18, 19 117 add_update 1, v10, 17, 18, 19, 16 118 add_update 0, v11, 18, 19, 16, 17 119 add_update 1, v12, 19, 16, 17, 18 120 121 add_only 0, v13, 17 122 add_only 1, v14, 18 123 add_only 0, v15, 19 124 add_only 1 125 126 /* update state */ 127 add dgav.4s, dgav.4s, dg0v.4s 128 add dgbv.4s, dgbv.4s, dg1v.4s 129 130 /* return early if voluntary preemption is needed */ 131 cond_yield 1f, x5, x6 132 133 /* handled all input blocks? */ 134 cbnz x2, 0b 135 136 /* store new state */ 1371: st1 {dgav.4s, dgbv.4s}, [x0] 138 mov x0, x2 139 ret 140SYM_FUNC_END(__sha256_ce_transform) 141 142 .unreq dga 143 .unreq dgav 144 .unreq dgb 145 .unreq dgbv 146 .unreq t0 147 .unreq t1 148 .unreq dg0q 149 .unreq dg0v 150 .unreq dg1q 151 .unreq dg1v 152 .unreq dg2q 153 .unreq dg2v 154 155 // parameters for sha256_ce_finup2x() 156 ctx .req x0 157 data1 .req x1 158 data2 .req x2 159 len .req w3 160 out1 .req x4 161 out2 .req x5 162 163 // other scalar variables 164 count .req x6 165 final_step .req w7 166 167 // x8-x9 are used as temporaries. 168 169 // v0-v15 are used to cache the SHA-256 round constants. 170 // v16-v19 are used for the message schedule for the first message. 171 // v20-v23 are used for the message schedule for the second message. 172 // v24-v31 are used for the state and temporaries as given below. 173 // *_a are for the first message and *_b for the second. 174 state0_a_q .req q24 175 state0_a .req v24 176 state1_a_q .req q25 177 state1_a .req v25 178 state0_b_q .req q26 179 state0_b .req v26 180 state1_b_q .req q27 181 state1_b .req v27 182 t0_a .req v28 183 t0_b .req v29 184 t1_a_q .req q30 185 t1_a .req v30 186 t1_b_q .req q31 187 t1_b .req v31 188 189#define OFFSETOF_BYTECOUNT 32 // offsetof(struct __sha256_ctx, bytecount) 190#define OFFSETOF_BUF 40 // offsetof(struct __sha256_ctx, buf) 191// offsetof(struct __sha256_ctx, state) is assumed to be 0. 192 193 // Do 4 rounds of SHA-256 for each of two messages (interleaved). m0_a 194 // and m0_b contain the current 4 message schedule words for the first 195 // and second message respectively. 196 // 197 // If not all the message schedule words have been computed yet, then 198 // this also computes 4 more message schedule words for each message. 199 // m1_a-m3_a contain the next 3 groups of 4 message schedule words for 200 // the first message, and likewise m1_b-m3_b for the second. After 201 // consuming the current value of m0_a, this macro computes the group 202 // after m3_a and writes it to m0_a, and likewise for *_b. This means 203 // that the next (m0_a, m1_a, m2_a, m3_a) is the current (m1_a, m2_a, 204 // m3_a, m0_a), and likewise for *_b, so the caller must cycle through 205 // the registers accordingly. 206 .macro do_4rounds_2x i, k, m0_a, m1_a, m2_a, m3_a, \ 207 m0_b, m1_b, m2_b, m3_b 208 add t0_a\().4s, \m0_a\().4s, \k\().4s 209 add t0_b\().4s, \m0_b\().4s, \k\().4s 210 .if \i < 48 211 sha256su0 \m0_a\().4s, \m1_a\().4s 212 sha256su0 \m0_b\().4s, \m1_b\().4s 213 sha256su1 \m0_a\().4s, \m2_a\().4s, \m3_a\().4s 214 sha256su1 \m0_b\().4s, \m2_b\().4s, \m3_b\().4s 215 .endif 216 mov t1_a.16b, state0_a.16b 217 mov t1_b.16b, state0_b.16b 218 sha256h state0_a_q, state1_a_q, t0_a\().4s 219 sha256h state0_b_q, state1_b_q, t0_b\().4s 220 sha256h2 state1_a_q, t1_a_q, t0_a\().4s 221 sha256h2 state1_b_q, t1_b_q, t0_b\().4s 222 .endm 223 224 .macro do_16rounds_2x i, k0, k1, k2, k3 225 do_4rounds_2x \i + 0, \k0, v16, v17, v18, v19, v20, v21, v22, v23 226 do_4rounds_2x \i + 4, \k1, v17, v18, v19, v16, v21, v22, v23, v20 227 do_4rounds_2x \i + 8, \k2, v18, v19, v16, v17, v22, v23, v20, v21 228 do_4rounds_2x \i + 12, \k3, v19, v16, v17, v18, v23, v20, v21, v22 229 .endm 230 231// 232// void sha256_ce_finup2x(const struct __sha256_ctx *ctx, 233// const u8 *data1, const u8 *data2, int len, 234// u8 out1[SHA256_DIGEST_SIZE], 235// u8 out2[SHA256_DIGEST_SIZE]); 236// 237// This function computes the SHA-256 digests of two messages |data1| and 238// |data2| that are both |len| bytes long, starting from the initial context 239// |ctx|. |len| must be at least SHA256_BLOCK_SIZE. 240// 241// The instructions for the two SHA-256 operations are interleaved. On many 242// CPUs, this is almost twice as fast as hashing each message individually due 243// to taking better advantage of the CPU's SHA-256 and SIMD throughput. 244// 245SYM_FUNC_START(sha256_ce_finup2x) 246 sub sp, sp, #128 247 mov final_step, #0 248 load_round_constants x8 249 250 // Load the initial state from ctx->state. 251 ld1 {state0_a.4s-state1_a.4s}, [ctx] 252 253 // Load ctx->bytecount. Take the mod 64 of it to get the number of 254 // bytes that are buffered in ctx->buf. Also save it in a register with 255 // len added to it. 256 ldr x8, [ctx, #OFFSETOF_BYTECOUNT] 257 add count, x8, len, sxtw 258 and x8, x8, #63 259 cbz x8, .Lfinup2x_enter_loop // No bytes buffered? 260 261 // x8 bytes (1 to 63) are currently buffered in ctx->buf. Load them 262 // followed by the first 64 - x8 bytes of data. Since len >= 64, we 263 // just load 64 bytes from each of ctx->buf, data1, and data2 264 // unconditionally and rearrange the data as needed. 265 add x9, ctx, #OFFSETOF_BUF 266 ld1 {v16.16b-v19.16b}, [x9] 267 st1 {v16.16b-v19.16b}, [sp] 268 269 ld1 {v16.16b-v19.16b}, [data1], #64 270 add x9, sp, x8 271 st1 {v16.16b-v19.16b}, [x9] 272 ld1 {v16.4s-v19.4s}, [sp] 273 274 ld1 {v20.16b-v23.16b}, [data2], #64 275 st1 {v20.16b-v23.16b}, [x9] 276 ld1 {v20.4s-v23.4s}, [sp] 277 278 sub len, len, #64 279 sub data1, data1, x8 280 sub data2, data2, x8 281 add len, len, w8 282 mov state0_b.16b, state0_a.16b 283 mov state1_b.16b, state1_a.16b 284 b .Lfinup2x_loop_have_data 285 286.Lfinup2x_enter_loop: 287 sub len, len, #64 288 mov state0_b.16b, state0_a.16b 289 mov state1_b.16b, state1_a.16b 290.Lfinup2x_loop: 291 // Load the next two data blocks. 292 ld1 {v16.4s-v19.4s}, [data1], #64 293 ld1 {v20.4s-v23.4s}, [data2], #64 294.Lfinup2x_loop_have_data: 295 // Convert the words of the data blocks from big endian. 296CPU_LE( rev32 v16.16b, v16.16b ) 297CPU_LE( rev32 v17.16b, v17.16b ) 298CPU_LE( rev32 v18.16b, v18.16b ) 299CPU_LE( rev32 v19.16b, v19.16b ) 300CPU_LE( rev32 v20.16b, v20.16b ) 301CPU_LE( rev32 v21.16b, v21.16b ) 302CPU_LE( rev32 v22.16b, v22.16b ) 303CPU_LE( rev32 v23.16b, v23.16b ) 304.Lfinup2x_loop_have_bswapped_data: 305 306 // Save the original state for each block. 307 st1 {state0_a.4s-state1_b.4s}, [sp] 308 309 // Do the SHA-256 rounds on each block. 310 do_16rounds_2x 0, v0, v1, v2, v3 311 do_16rounds_2x 16, v4, v5, v6, v7 312 do_16rounds_2x 32, v8, v9, v10, v11 313 do_16rounds_2x 48, v12, v13, v14, v15 314 315 // Add the original state for each block. 316 ld1 {v16.4s-v19.4s}, [sp] 317 add state0_a.4s, state0_a.4s, v16.4s 318 add state1_a.4s, state1_a.4s, v17.4s 319 add state0_b.4s, state0_b.4s, v18.4s 320 add state1_b.4s, state1_b.4s, v19.4s 321 322 // Update len and loop back if more blocks remain. 323 sub len, len, #64 324 tbz len, #31, .Lfinup2x_loop // len >= 0? 325 326 // Check if any final blocks need to be handled. 327 // final_step = 2: all done 328 // final_step = 1: need to do count-only padding block 329 // final_step = 0: need to do the block with 0x80 padding byte 330 tbnz final_step, #1, .Lfinup2x_done 331 tbnz final_step, #0, .Lfinup2x_finalize_countonly 332 add len, len, #64 333 cbz len, .Lfinup2x_finalize_blockaligned 334 335 // Not block-aligned; 1 <= len <= 63 data bytes remain. Pad the block. 336 // To do this, write the padding starting with the 0x80 byte to 337 // &sp[64]. Then for each message, copy the last 64 data bytes to sp 338 // and load from &sp[64 - len] to get the needed padding block. This 339 // code relies on the data buffers being >= 64 bytes in length. 340 sub w8, len, #64 // w8 = len - 64 341 add data1, data1, w8, sxtw // data1 += len - 64 342 add data2, data2, w8, sxtw // data2 += len - 64 343CPU_LE( mov x9, #0x80 ) 344CPU_LE( fmov d16, x9 ) 345CPU_BE( movi v16.16b, #0 ) 346CPU_BE( mov x9, #0x8000000000000000 ) 347CPU_BE( mov v16.d[1], x9 ) 348 movi v17.16b, #0 349 stp q16, q17, [sp, #64] 350 stp q17, q17, [sp, #96] 351 sub x9, sp, w8, sxtw // x9 = &sp[64 - len] 352 cmp len, #56 353 b.ge 1f // will count spill into its own block? 354 lsl count, count, #3 355CPU_LE( rev count, count ) 356 str count, [x9, #56] 357 mov final_step, #2 // won't need count-only block 358 b 2f 3591: 360 mov final_step, #1 // will need count-only block 3612: 362 ld1 {v16.16b-v19.16b}, [data1] 363 st1 {v16.16b-v19.16b}, [sp] 364 ld1 {v16.4s-v19.4s}, [x9] 365 ld1 {v20.16b-v23.16b}, [data2] 366 st1 {v20.16b-v23.16b}, [sp] 367 ld1 {v20.4s-v23.4s}, [x9] 368 b .Lfinup2x_loop_have_data 369 370 // Prepare a padding block, either: 371 // 372 // {0x80, 0, 0, 0, ..., count (as __be64)} 373 // This is for a block aligned message. 374 // 375 // { 0, 0, 0, 0, ..., count (as __be64)} 376 // This is for a message whose length mod 64 is >= 56. 377 // 378 // Pre-swap the endianness of the words. 379.Lfinup2x_finalize_countonly: 380 movi v16.2d, #0 381 b 1f 382.Lfinup2x_finalize_blockaligned: 383 mov x8, #0x80000000 384 fmov d16, x8 3851: 386 movi v17.2d, #0 387 movi v18.2d, #0 388 ror count, count, #29 // ror(lsl(count, 3), 32) 389 mov v19.d[0], xzr 390 mov v19.d[1], count 391 mov v20.16b, v16.16b 392 movi v21.2d, #0 393 movi v22.2d, #0 394 mov v23.16b, v19.16b 395 mov final_step, #2 396 b .Lfinup2x_loop_have_bswapped_data 397 398.Lfinup2x_done: 399 // Write the two digests with all bytes in the correct order. 400CPU_LE( rev32 state0_a.16b, state0_a.16b ) 401CPU_LE( rev32 state1_a.16b, state1_a.16b ) 402CPU_LE( rev32 state0_b.16b, state0_b.16b ) 403CPU_LE( rev32 state1_b.16b, state1_b.16b ) 404 st1 {state0_a.4s-state1_a.4s}, [out1] 405 st1 {state0_b.4s-state1_b.4s}, [out2] 406 add sp, sp, #128 407 ret 408SYM_FUNC_END(sha256_ce_finup2x) 409