1######################################################################## 2# Implement fast SHA-256 with SSSE3 instructions. (x86_64) 3# 4# Copyright (C) 2013 Intel Corporation. 5# 6# Authors: 7# James Guilford <james.guilford@intel.com> 8# Kirk Yap <kirk.s.yap@intel.com> 9# Tim Chen <tim.c.chen@linux.intel.com> 10# 11# This software is available to you under a choice of one of two 12# licenses. You may choose to be licensed under the terms of the GNU 13# General Public License (GPL) Version 2, available from the file 14# COPYING in the main directory of this source tree, or the 15# OpenIB.org BSD license below: 16# 17# Redistribution and use in source and binary forms, with or 18# without modification, are permitted provided that the following 19# conditions are met: 20# 21# - Redistributions of source code must retain the above 22# copyright notice, this list of conditions and the following 23# disclaimer. 24# 25# - Redistributions in binary form must reproduce the above 26# copyright notice, this list of conditions and the following 27# disclaimer in the documentation and/or other materials 28# provided with the distribution. 29# 30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 37# SOFTWARE. 38# 39######################################################################## 40# 41# This code is described in an Intel White-Paper: 42# "Fast SHA-256 Implementations on Intel Architecture Processors" 43# 44# To find it, surf to http://www.intel.com/p/en_US/embedded 45# and search for that title. 46# 47######################################################################## 48 49#include <linux/linkage.h> 50#include <linux/cfi_types.h> 51 52## assume buffers not aligned 53#define MOVDQ movdqu 54 55################################ Define Macros 56 57# addm [mem], reg 58# Add reg to mem using reg-mem add and store 59.macro addm p1 p2 60 add \p1, \p2 61 mov \p2, \p1 62.endm 63 64################################ 65 66# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask 67# Load xmm with mem and byte swap each dword 68.macro COPY_XMM_AND_BSWAP p1 p2 p3 69 MOVDQ \p2, \p1 70 pshufb \p3, \p1 71.endm 72 73################################ 74 75X0 = %xmm4 76X1 = %xmm5 77X2 = %xmm6 78X3 = %xmm7 79 80XTMP0 = %xmm0 81XTMP1 = %xmm1 82XTMP2 = %xmm2 83XTMP3 = %xmm3 84XTMP4 = %xmm8 85XFER = %xmm9 86 87SHUF_00BA = %xmm10 # shuffle xBxA -> 00BA 88SHUF_DC00 = %xmm11 # shuffle xDxC -> DC00 89BYTE_FLIP_MASK = %xmm12 90 91NUM_BLKS = %rdx # 3rd arg 92INP = %rsi # 2nd arg 93CTX = %rdi # 1st arg 94 95SRND = %rsi # clobbers INP 96c = %ecx 97d = %r8d 98e = %edx 99TBL = %r12 100a = %eax 101b = %ebx 102 103f = %r9d 104g = %r10d 105h = %r11d 106 107y0 = %r13d 108y1 = %r14d 109y2 = %r15d 110 111 112 113_INP_END_SIZE = 8 114_INP_SIZE = 8 115_XFER_SIZE = 16 116_XMM_SAVE_SIZE = 0 117 118_INP_END = 0 119_INP = _INP_END + _INP_END_SIZE 120_XFER = _INP + _INP_SIZE 121_XMM_SAVE = _XFER + _XFER_SIZE 122STACK_SIZE = _XMM_SAVE + _XMM_SAVE_SIZE 123 124# rotate_Xs 125# Rotate values of symbols X0...X3 126.macro rotate_Xs 127X_ = X0 128X0 = X1 129X1 = X2 130X2 = X3 131X3 = X_ 132.endm 133 134# ROTATE_ARGS 135# Rotate values of symbols a...h 136.macro ROTATE_ARGS 137TMP_ = h 138h = g 139g = f 140f = e 141e = d 142d = c 143c = b 144b = a 145a = TMP_ 146.endm 147 148.macro FOUR_ROUNDS_AND_SCHED 149 ## compute s0 four at a time and s1 two at a time 150 ## compute W[-16] + W[-7] 4 at a time 151 movdqa X3, XTMP0 152 mov e, y0 # y0 = e 153 ror $(25-11), y0 # y0 = e >> (25-11) 154 mov a, y1 # y1 = a 155 palignr $4, X2, XTMP0 # XTMP0 = W[-7] 156 ror $(22-13), y1 # y1 = a >> (22-13) 157 xor e, y0 # y0 = e ^ (e >> (25-11)) 158 mov f, y2 # y2 = f 159 ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 160 movdqa X1, XTMP1 161 xor a, y1 # y1 = a ^ (a >> (22-13) 162 xor g, y2 # y2 = f^g 163 paddd X0, XTMP0 # XTMP0 = W[-7] + W[-16] 164 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 165 and e, y2 # y2 = (f^g)&e 166 ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 167 ## compute s0 168 palignr $4, X0, XTMP1 # XTMP1 = W[-15] 169 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 170 ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 171 xor g, y2 # y2 = CH = ((f^g)&e)^g 172 movdqa XTMP1, XTMP2 # XTMP2 = W[-15] 173 ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 174 add y0, y2 # y2 = S1 + CH 175 add _XFER(%rsp) , y2 # y2 = k + w + S1 + CH 176 movdqa XTMP1, XTMP3 # XTMP3 = W[-15] 177 mov a, y0 # y0 = a 178 add y2, h # h = h + S1 + CH + k + w 179 mov a, y2 # y2 = a 180 pslld $(32-7), XTMP1 # 181 or c, y0 # y0 = a|c 182 add h, d # d = d + h + S1 + CH + k + w 183 and c, y2 # y2 = a&c 184 psrld $7, XTMP2 # 185 and b, y0 # y0 = (a|c)&b 186 add y1, h # h = h + S1 + CH + k + w + S0 187 por XTMP2, XTMP1 # XTMP1 = W[-15] ror 7 188 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 189 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 190 # 191 ROTATE_ARGS # 192 movdqa XTMP3, XTMP2 # XTMP2 = W[-15] 193 mov e, y0 # y0 = e 194 mov a, y1 # y1 = a 195 movdqa XTMP3, XTMP4 # XTMP4 = W[-15] 196 ror $(25-11), y0 # y0 = e >> (25-11) 197 xor e, y0 # y0 = e ^ (e >> (25-11)) 198 mov f, y2 # y2 = f 199 ror $(22-13), y1 # y1 = a >> (22-13) 200 pslld $(32-18), XTMP3 # 201 xor a, y1 # y1 = a ^ (a >> (22-13) 202 ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 203 xor g, y2 # y2 = f^g 204 psrld $18, XTMP2 # 205 ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 206 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 207 and e, y2 # y2 = (f^g)&e 208 ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 209 pxor XTMP3, XTMP1 210 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 211 xor g, y2 # y2 = CH = ((f^g)&e)^g 212 psrld $3, XTMP4 # XTMP4 = W[-15] >> 3 213 add y0, y2 # y2 = S1 + CH 214 add (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH 215 ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 216 pxor XTMP2, XTMP1 # XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 217 mov a, y0 # y0 = a 218 add y2, h # h = h + S1 + CH + k + w 219 mov a, y2 # y2 = a 220 pxor XTMP4, XTMP1 # XTMP1 = s0 221 or c, y0 # y0 = a|c 222 add h, d # d = d + h + S1 + CH + k + w 223 and c, y2 # y2 = a&c 224 ## compute low s1 225 pshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} 226 and b, y0 # y0 = (a|c)&b 227 add y1, h # h = h + S1 + CH + k + w + S0 228 paddd XTMP1, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 229 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 230 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 231 232 ROTATE_ARGS 233 movdqa XTMP2, XTMP3 # XTMP3 = W[-2] {BBAA} 234 mov e, y0 # y0 = e 235 mov a, y1 # y1 = a 236 ror $(25-11), y0 # y0 = e >> (25-11) 237 movdqa XTMP2, XTMP4 # XTMP4 = W[-2] {BBAA} 238 xor e, y0 # y0 = e ^ (e >> (25-11)) 239 ror $(22-13), y1 # y1 = a >> (22-13) 240 mov f, y2 # y2 = f 241 xor a, y1 # y1 = a ^ (a >> (22-13) 242 ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 243 psrlq $17, XTMP2 # XTMP2 = W[-2] ror 17 {xBxA} 244 xor g, y2 # y2 = f^g 245 psrlq $19, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA} 246 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 247 and e, y2 # y2 = (f^g)&e 248 psrld $10, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} 249 ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 250 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 251 xor g, y2 # y2 = CH = ((f^g)&e)^g 252 ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 253 pxor XTMP3, XTMP2 254 add y0, y2 # y2 = S1 + CH 255 ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 256 add (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH 257 pxor XTMP2, XTMP4 # XTMP4 = s1 {xBxA} 258 mov a, y0 # y0 = a 259 add y2, h # h = h + S1 + CH + k + w 260 mov a, y2 # y2 = a 261 pshufb SHUF_00BA, XTMP4 # XTMP4 = s1 {00BA} 262 or c, y0 # y0 = a|c 263 add h, d # d = d + h + S1 + CH + k + w 264 and c, y2 # y2 = a&c 265 paddd XTMP4, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} 266 and b, y0 # y0 = (a|c)&b 267 add y1, h # h = h + S1 + CH + k + w + S0 268 ## compute high s1 269 pshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {BBAA} 270 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 271 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 272 # 273 ROTATE_ARGS # 274 movdqa XTMP2, XTMP3 # XTMP3 = W[-2] {DDCC} 275 mov e, y0 # y0 = e 276 ror $(25-11), y0 # y0 = e >> (25-11) 277 mov a, y1 # y1 = a 278 movdqa XTMP2, X0 # X0 = W[-2] {DDCC} 279 ror $(22-13), y1 # y1 = a >> (22-13) 280 xor e, y0 # y0 = e ^ (e >> (25-11)) 281 mov f, y2 # y2 = f 282 ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 283 psrlq $17, XTMP2 # XTMP2 = W[-2] ror 17 {xDxC} 284 xor a, y1 # y1 = a ^ (a >> (22-13) 285 xor g, y2 # y2 = f^g 286 psrlq $19, XTMP3 # XTMP3 = W[-2] ror 19 {xDxC} 287 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25 288 and e, y2 # y2 = (f^g)&e 289 ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 290 psrld $10, X0 # X0 = W[-2] >> 10 {DDCC} 291 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22 292 ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>2 293 xor g, y2 # y2 = CH = ((f^g)&e)^g 294 pxor XTMP3, XTMP2 # 295 ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>2 296 add y0, y2 # y2 = S1 + CH 297 add (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH 298 pxor XTMP2, X0 # X0 = s1 {xDxC} 299 mov a, y0 # y0 = a 300 add y2, h # h = h + S1 + CH + k + w 301 mov a, y2 # y2 = a 302 pshufb SHUF_DC00, X0 # X0 = s1 {DC00} 303 or c, y0 # y0 = a|c 304 add h, d # d = d + h + S1 + CH + k + w 305 and c, y2 # y2 = a&c 306 paddd XTMP0, X0 # X0 = {W[3], W[2], W[1], W[0]} 307 and b, y0 # y0 = (a|c)&b 308 add y1, h # h = h + S1 + CH + k + w + S0 309 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 310 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 311 312 ROTATE_ARGS 313 rotate_Xs 314.endm 315 316## input is [rsp + _XFER + %1 * 4] 317.macro DO_ROUND round 318 mov e, y0 # y0 = e 319 ror $(25-11), y0 # y0 = e >> (25-11) 320 mov a, y1 # y1 = a 321 xor e, y0 # y0 = e ^ (e >> (25-11)) 322 ror $(22-13), y1 # y1 = a >> (22-13) 323 mov f, y2 # y2 = f 324 xor a, y1 # y1 = a ^ (a >> (22-13) 325 ror $(11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 326 xor g, y2 # y2 = f^g 327 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 328 ror $(13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 329 and e, y2 # y2 = (f^g)&e 330 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 331 ror $6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 332 xor g, y2 # y2 = CH = ((f^g)&e)^g 333 add y0, y2 # y2 = S1 + CH 334 ror $2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 335 offset = \round * 4 + _XFER 336 add offset(%rsp), y2 # y2 = k + w + S1 + CH 337 mov a, y0 # y0 = a 338 add y2, h # h = h + S1 + CH + k + w 339 mov a, y2 # y2 = a 340 or c, y0 # y0 = a|c 341 add h, d # d = d + h + S1 + CH + k + w 342 and c, y2 # y2 = a&c 343 and b, y0 # y0 = (a|c)&b 344 add y1, h # h = h + S1 + CH + k + w + S0 345 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 346 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 347 ROTATE_ARGS 348.endm 349 350######################################################################## 351## void sha256_transform_ssse3(struct sha256_state *state, const u8 *data, 352## int blocks); 353## arg 1 : pointer to state 354## (struct sha256_state is assumed to begin with u32 state[8]) 355## arg 2 : pointer to input data 356## arg 3 : Num blocks 357######################################################################## 358.text 359SYM_TYPED_FUNC_START(sha256_transform_ssse3) 360 pushq %rbx 361 pushq %r12 362 pushq %r13 363 pushq %r14 364 pushq %r15 365 pushq %rbp 366 mov %rsp, %rbp 367 368 subq $STACK_SIZE, %rsp 369 and $~15, %rsp 370 371 shl $6, NUM_BLKS # convert to bytes 372 jz done_hash 373 add INP, NUM_BLKS 374 mov NUM_BLKS, _INP_END(%rsp) # pointer to end of data 375 376 ## load initial digest 377 mov 4*0(CTX), a 378 mov 4*1(CTX), b 379 mov 4*2(CTX), c 380 mov 4*3(CTX), d 381 mov 4*4(CTX), e 382 mov 4*5(CTX), f 383 mov 4*6(CTX), g 384 mov 4*7(CTX), h 385 386 movdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK 387 movdqa _SHUF_00BA(%rip), SHUF_00BA 388 movdqa _SHUF_DC00(%rip), SHUF_DC00 389 390loop0: 391 lea K256(%rip), TBL 392 393 ## byte swap first 16 dwords 394 COPY_XMM_AND_BSWAP X0, 0*16(INP), BYTE_FLIP_MASK 395 COPY_XMM_AND_BSWAP X1, 1*16(INP), BYTE_FLIP_MASK 396 COPY_XMM_AND_BSWAP X2, 2*16(INP), BYTE_FLIP_MASK 397 COPY_XMM_AND_BSWAP X3, 3*16(INP), BYTE_FLIP_MASK 398 399 mov INP, _INP(%rsp) 400 401 ## schedule 48 input dwords, by doing 3 rounds of 16 each 402 mov $3, SRND 403.align 16 404loop1: 405 movdqa (TBL), XFER 406 paddd X0, XFER 407 movdqa XFER, _XFER(%rsp) 408 FOUR_ROUNDS_AND_SCHED 409 410 movdqa 1*16(TBL), XFER 411 paddd X0, XFER 412 movdqa XFER, _XFER(%rsp) 413 FOUR_ROUNDS_AND_SCHED 414 415 movdqa 2*16(TBL), XFER 416 paddd X0, XFER 417 movdqa XFER, _XFER(%rsp) 418 FOUR_ROUNDS_AND_SCHED 419 420 movdqa 3*16(TBL), XFER 421 paddd X0, XFER 422 movdqa XFER, _XFER(%rsp) 423 add $4*16, TBL 424 FOUR_ROUNDS_AND_SCHED 425 426 sub $1, SRND 427 jne loop1 428 429 mov $2, SRND 430loop2: 431 paddd (TBL), X0 432 movdqa X0, _XFER(%rsp) 433 DO_ROUND 0 434 DO_ROUND 1 435 DO_ROUND 2 436 DO_ROUND 3 437 paddd 1*16(TBL), X1 438 movdqa X1, _XFER(%rsp) 439 add $2*16, TBL 440 DO_ROUND 0 441 DO_ROUND 1 442 DO_ROUND 2 443 DO_ROUND 3 444 445 movdqa X2, X0 446 movdqa X3, X1 447 448 sub $1, SRND 449 jne loop2 450 451 addm (4*0)(CTX),a 452 addm (4*1)(CTX),b 453 addm (4*2)(CTX),c 454 addm (4*3)(CTX),d 455 addm (4*4)(CTX),e 456 addm (4*5)(CTX),f 457 addm (4*6)(CTX),g 458 addm (4*7)(CTX),h 459 460 mov _INP(%rsp), INP 461 add $64, INP 462 cmp _INP_END(%rsp), INP 463 jne loop0 464 465done_hash: 466 467 mov %rbp, %rsp 468 popq %rbp 469 popq %r15 470 popq %r14 471 popq %r13 472 popq %r12 473 popq %rbx 474 475 RET 476SYM_FUNC_END(sha256_transform_ssse3) 477 478.section .rodata.cst256.K256, "aM", @progbits, 256 479.align 64 480K256: 481 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 482 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 483 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 484 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 485 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc 486 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da 487 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 488 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 489 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 490 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 491 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 492 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 493 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 494 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 495 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 496 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 497 498.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16 499.align 16 500PSHUFFLE_BYTE_FLIP_MASK: 501 .octa 0x0c0d0e0f08090a0b0405060700010203 502 503.section .rodata.cst16._SHUF_00BA, "aM", @progbits, 16 504.align 16 505# shuffle xBxA -> 00BA 506_SHUF_00BA: 507 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100 508 509.section .rodata.cst16._SHUF_DC00, "aM", @progbits, 16 510.align 16 511# shuffle xDxC -> DC00 512_SHUF_DC00: 513 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF 514