1######################################################################## 2# Implement fast SHA-256 with AVX2 instructions. (x86_64) 3# 4# Copyright (C) 2013 Intel Corporation. 5# 6# Authors: 7# James Guilford <james.guilford@intel.com> 8# Kirk Yap <kirk.s.yap@intel.com> 9# Tim Chen <tim.c.chen@linux.intel.com> 10# 11# This software is available to you under a choice of one of two 12# licenses. You may choose to be licensed under the terms of the GNU 13# General Public License (GPL) Version 2, available from the file 14# COPYING in the main directory of this source tree, or the 15# OpenIB.org BSD license below: 16# 17# Redistribution and use in source and binary forms, with or 18# without modification, are permitted provided that the following 19# conditions are met: 20# 21# - Redistributions of source code must retain the above 22# copyright notice, this list of conditions and the following 23# disclaimer. 24# 25# - Redistributions in binary form must reproduce the above 26# copyright notice, this list of conditions and the following 27# disclaimer in the documentation and/or other materials 28# provided with the distribution. 29# 30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 37# SOFTWARE. 38# 39######################################################################## 40# 41# This code is described in an Intel White-Paper: 42# "Fast SHA-256 Implementations on Intel Architecture Processors" 43# 44# To find it, surf to http://www.intel.com/p/en_US/embedded 45# and search for that title. 46# 47######################################################################## 48# This code schedules 2 blocks at a time, with 4 lanes per block 49######################################################################## 50 51#ifdef CONFIG_AS_AVX2 52#include <linux/linkage.h> 53 54## assume buffers not aligned 55#define VMOVDQ vmovdqu 56 57################################ Define Macros 58 59# addm [mem], reg 60# Add reg to mem using reg-mem add and store 61.macro addm p1 p2 62 add \p1, \p2 63 mov \p2, \p1 64.endm 65 66################################ 67 68X0 = %ymm4 69X1 = %ymm5 70X2 = %ymm6 71X3 = %ymm7 72 73# XMM versions of above 74XWORD0 = %xmm4 75XWORD1 = %xmm5 76XWORD2 = %xmm6 77XWORD3 = %xmm7 78 79XTMP0 = %ymm0 80XTMP1 = %ymm1 81XTMP2 = %ymm2 82XTMP3 = %ymm3 83XTMP4 = %ymm8 84XFER = %ymm9 85XTMP5 = %ymm11 86 87SHUF_00BA = %ymm10 # shuffle xBxA -> 00BA 88SHUF_DC00 = %ymm12 # shuffle xDxC -> DC00 89BYTE_FLIP_MASK = %ymm13 90 91X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK 92 93NUM_BLKS = %rdx # 3rd arg 94INP = %rsi # 2nd arg 95CTX = %rdi # 1st arg 96c = %ecx 97d = %r8d 98e = %edx # clobbers NUM_BLKS 99y3 = %esi # clobbers INP 100 101 102TBL = %rbp 103SRND = CTX # SRND is same register as CTX 104 105a = %eax 106b = %ebx 107f = %r9d 108g = %r10d 109h = %r11d 110old_h = %r11d 111 112T1 = %r12d 113y0 = %r13d 114y1 = %r14d 115y2 = %r15d 116 117 118_XFER_SIZE = 2*64*4 # 2 blocks, 64 rounds, 4 bytes/round 119_XMM_SAVE_SIZE = 0 120_INP_END_SIZE = 8 121_INP_SIZE = 8 122_CTX_SIZE = 8 123_RSP_SIZE = 8 124 125_XFER = 0 126_XMM_SAVE = _XFER + _XFER_SIZE 127_INP_END = _XMM_SAVE + _XMM_SAVE_SIZE 128_INP = _INP_END + _INP_END_SIZE 129_CTX = _INP + _INP_SIZE 130_RSP = _CTX + _CTX_SIZE 131STACK_SIZE = _RSP + _RSP_SIZE 132 133# rotate_Xs 134# Rotate values of symbols X0...X3 135.macro rotate_Xs 136 X_ = X0 137 X0 = X1 138 X1 = X2 139 X2 = X3 140 X3 = X_ 141.endm 142 143# ROTATE_ARGS 144# Rotate values of symbols a...h 145.macro ROTATE_ARGS 146 old_h = h 147 TMP_ = h 148 h = g 149 g = f 150 f = e 151 e = d 152 d = c 153 c = b 154 b = a 155 a = TMP_ 156.endm 157 158.macro FOUR_ROUNDS_AND_SCHED disp 159################################### RND N + 0 ############################ 160 161 mov a, y3 # y3 = a # MAJA 162 rorx $25, e, y0 # y0 = e >> 25 # S1A 163 rorx $11, e, y1 # y1 = e >> 11 # S1B 164 165 addl \disp(%rsp, SRND), h # h = k + w + h # -- 166 or c, y3 # y3 = a|c # MAJA 167 vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7] 168 mov f, y2 # y2 = f # CH 169 rorx $13, a, T1 # T1 = a >> 13 # S0B 170 171 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 172 xor g, y2 # y2 = f^g # CH 173 vpaddd X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1 174 rorx $6, e, y1 # y1 = (e >> 6) # S1 175 176 and e, y2 # y2 = (f^g)&e # CH 177 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 178 rorx $22, a, y1 # y1 = a >> 22 # S0A 179 add h, d # d = k + w + h + d # -- 180 181 and b, y3 # y3 = (a|c)&b # MAJA 182 vpalignr $4, X0, X1, XTMP1 # XTMP1 = W[-15] 183 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 184 rorx $2, a, T1 # T1 = (a >> 2) # S0 185 186 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 187 vpsrld $7, XTMP1, XTMP2 188 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 189 mov a, T1 # T1 = a # MAJB 190 and c, T1 # T1 = a&c # MAJB 191 192 add y0, y2 # y2 = S1 + CH # -- 193 vpslld $(32-7), XTMP1, XTMP3 194 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 195 add y1, h # h = k + w + h + S0 # -- 196 197 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 198 vpor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 199 200 vpsrld $18, XTMP1, XTMP2 201 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 202 add y3, h # h = t1 + S0 + MAJ # -- 203 204 205 ROTATE_ARGS 206 207################################### RND N + 1 ############################ 208 209 mov a, y3 # y3 = a # MAJA 210 rorx $25, e, y0 # y0 = e >> 25 # S1A 211 rorx $11, e, y1 # y1 = e >> 11 # S1B 212 offset = \disp + 1*4 213 addl offset(%rsp, SRND), h # h = k + w + h # -- 214 or c, y3 # y3 = a|c # MAJA 215 216 217 vpsrld $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3 218 mov f, y2 # y2 = f # CH 219 rorx $13, a, T1 # T1 = a >> 13 # S0B 220 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 221 xor g, y2 # y2 = f^g # CH 222 223 224 rorx $6, e, y1 # y1 = (e >> 6) # S1 225 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 226 rorx $22, a, y1 # y1 = a >> 22 # S0A 227 and e, y2 # y2 = (f^g)&e # CH 228 add h, d # d = k + w + h + d # -- 229 230 vpslld $(32-18), XTMP1, XTMP1 231 and b, y3 # y3 = (a|c)&b # MAJA 232 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 233 234 vpxor XTMP1, XTMP3, XTMP3 235 rorx $2, a, T1 # T1 = (a >> 2) # S0 236 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 237 238 vpxor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 ^ W[-15] ror 18 239 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 240 mov a, T1 # T1 = a # MAJB 241 and c, T1 # T1 = a&c # MAJB 242 add y0, y2 # y2 = S1 + CH # -- 243 244 vpxor XTMP4, XTMP3, XTMP1 # XTMP1 = s0 245 vpshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} 246 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 247 add y1, h # h = k + w + h + S0 # -- 248 249 vpaddd XTMP1, XTMP0, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 250 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 251 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 252 add y3, h # h = t1 + S0 + MAJ # -- 253 254 vpsrld $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} 255 256 257 ROTATE_ARGS 258 259################################### RND N + 2 ############################ 260 261 mov a, y3 # y3 = a # MAJA 262 rorx $25, e, y0 # y0 = e >> 25 # S1A 263 offset = \disp + 2*4 264 addl offset(%rsp, SRND), h # h = k + w + h # -- 265 266 vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA} 267 rorx $11, e, y1 # y1 = e >> 11 # S1B 268 or c, y3 # y3 = a|c # MAJA 269 mov f, y2 # y2 = f # CH 270 xor g, y2 # y2 = f^g # CH 271 272 rorx $13, a, T1 # T1 = a >> 13 # S0B 273 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 274 vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xBxA} 275 and e, y2 # y2 = (f^g)&e # CH 276 277 rorx $6, e, y1 # y1 = (e >> 6) # S1 278 vpxor XTMP3, XTMP2, XTMP2 279 add h, d # d = k + w + h + d # -- 280 and b, y3 # y3 = (a|c)&b # MAJA 281 282 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 283 rorx $22, a, y1 # y1 = a >> 22 # S0A 284 vpxor XTMP2, XTMP4, XTMP4 # XTMP4 = s1 {xBxA} 285 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 286 287 vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA} 288 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 289 rorx $2, a ,T1 # T1 = (a >> 2) # S0 290 vpaddd XTMP4, XTMP0, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} 291 292 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 293 mov a, T1 # T1 = a # MAJB 294 and c, T1 # T1 = a&c # MAJB 295 add y0, y2 # y2 = S1 + CH # -- 296 vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC} 297 298 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 299 add y1,h # h = k + w + h + S0 # -- 300 add y2,d # d = k + w + h + d + S1 + CH = d + t1 # -- 301 add y2,h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 302 303 add y3,h # h = t1 + S0 + MAJ # -- 304 305 306 ROTATE_ARGS 307 308################################### RND N + 3 ############################ 309 310 mov a, y3 # y3 = a # MAJA 311 rorx $25, e, y0 # y0 = e >> 25 # S1A 312 rorx $11, e, y1 # y1 = e >> 11 # S1B 313 offset = \disp + 3*4 314 addl offset(%rsp, SRND), h # h = k + w + h # -- 315 or c, y3 # y3 = a|c # MAJA 316 317 318 vpsrld $10, XTMP2, XTMP5 # XTMP5 = W[-2] >> 10 {DDCC} 319 mov f, y2 # y2 = f # CH 320 rorx $13, a, T1 # T1 = a >> 13 # S0B 321 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 322 xor g, y2 # y2 = f^g # CH 323 324 325 vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xDxC} 326 rorx $6, e, y1 # y1 = (e >> 6) # S1 327 and e, y2 # y2 = (f^g)&e # CH 328 add h, d # d = k + w + h + d # -- 329 and b, y3 # y3 = (a|c)&b # MAJA 330 331 vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xDxC} 332 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 333 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 334 335 vpxor XTMP3, XTMP2, XTMP2 336 rorx $22, a, y1 # y1 = a >> 22 # S0A 337 add y0, y2 # y2 = S1 + CH # -- 338 339 vpxor XTMP2, XTMP5, XTMP5 # XTMP5 = s1 {xDxC} 340 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 341 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 342 343 rorx $2, a, T1 # T1 = (a >> 2) # S0 344 vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00} 345 346 vpaddd XTMP0, XTMP5, X0 # X0 = {W[3], W[2], W[1], W[0]} 347 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 348 mov a, T1 # T1 = a # MAJB 349 and c, T1 # T1 = a&c # MAJB 350 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 351 352 add y1, h # h = k + w + h + S0 # -- 353 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 354 add y3, h # h = t1 + S0 + MAJ # -- 355 356 ROTATE_ARGS 357 rotate_Xs 358.endm 359 360.macro DO_4ROUNDS disp 361################################### RND N + 0 ########################### 362 363 mov f, y2 # y2 = f # CH 364 rorx $25, e, y0 # y0 = e >> 25 # S1A 365 rorx $11, e, y1 # y1 = e >> 11 # S1B 366 xor g, y2 # y2 = f^g # CH 367 368 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 369 rorx $6, e, y1 # y1 = (e >> 6) # S1 370 and e, y2 # y2 = (f^g)&e # CH 371 372 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 373 rorx $13, a, T1 # T1 = a >> 13 # S0B 374 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 375 rorx $22, a, y1 # y1 = a >> 22 # S0A 376 mov a, y3 # y3 = a # MAJA 377 378 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 379 rorx $2, a, T1 # T1 = (a >> 2) # S0 380 addl \disp(%rsp, SRND), h # h = k + w + h # -- 381 or c, y3 # y3 = a|c # MAJA 382 383 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 384 mov a, T1 # T1 = a # MAJB 385 and b, y3 # y3 = (a|c)&b # MAJA 386 and c, T1 # T1 = a&c # MAJB 387 add y0, y2 # y2 = S1 + CH # -- 388 389 390 add h, d # d = k + w + h + d # -- 391 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 392 add y1, h # h = k + w + h + S0 # -- 393 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 394 395 ROTATE_ARGS 396 397################################### RND N + 1 ########################### 398 399 add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 400 mov f, y2 # y2 = f # CH 401 rorx $25, e, y0 # y0 = e >> 25 # S1A 402 rorx $11, e, y1 # y1 = e >> 11 # S1B 403 xor g, y2 # y2 = f^g # CH 404 405 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 406 rorx $6, e, y1 # y1 = (e >> 6) # S1 407 and e, y2 # y2 = (f^g)&e # CH 408 add y3, old_h # h = t1 + S0 + MAJ # -- 409 410 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 411 rorx $13, a, T1 # T1 = a >> 13 # S0B 412 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 413 rorx $22, a, y1 # y1 = a >> 22 # S0A 414 mov a, y3 # y3 = a # MAJA 415 416 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 417 rorx $2, a, T1 # T1 = (a >> 2) # S0 418 offset = 4*1 + \disp 419 addl offset(%rsp, SRND), h # h = k + w + h # -- 420 or c, y3 # y3 = a|c # MAJA 421 422 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 423 mov a, T1 # T1 = a # MAJB 424 and b, y3 # y3 = (a|c)&b # MAJA 425 and c, T1 # T1 = a&c # MAJB 426 add y0, y2 # y2 = S1 + CH # -- 427 428 429 add h, d # d = k + w + h + d # -- 430 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 431 add y1, h # h = k + w + h + S0 # -- 432 433 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 434 435 ROTATE_ARGS 436 437################################### RND N + 2 ############################## 438 439 add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 440 mov f, y2 # y2 = f # CH 441 rorx $25, e, y0 # y0 = e >> 25 # S1A 442 rorx $11, e, y1 # y1 = e >> 11 # S1B 443 xor g, y2 # y2 = f^g # CH 444 445 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 446 rorx $6, e, y1 # y1 = (e >> 6) # S1 447 and e, y2 # y2 = (f^g)&e # CH 448 add y3, old_h # h = t1 + S0 + MAJ # -- 449 450 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 451 rorx $13, a, T1 # T1 = a >> 13 # S0B 452 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 453 rorx $22, a, y1 # y1 = a >> 22 # S0A 454 mov a, y3 # y3 = a # MAJA 455 456 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 457 rorx $2, a, T1 # T1 = (a >> 2) # S0 458 offset = 4*2 + \disp 459 addl offset(%rsp, SRND), h # h = k + w + h # -- 460 or c, y3 # y3 = a|c # MAJA 461 462 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 463 mov a, T1 # T1 = a # MAJB 464 and b, y3 # y3 = (a|c)&b # MAJA 465 and c, T1 # T1 = a&c # MAJB 466 add y0, y2 # y2 = S1 + CH # -- 467 468 469 add h, d # d = k + w + h + d # -- 470 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 471 add y1, h # h = k + w + h + S0 # -- 472 473 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 474 475 ROTATE_ARGS 476 477################################### RND N + 3 ########################### 478 479 add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 480 mov f, y2 # y2 = f # CH 481 rorx $25, e, y0 # y0 = e >> 25 # S1A 482 rorx $11, e, y1 # y1 = e >> 11 # S1B 483 xor g, y2 # y2 = f^g # CH 484 485 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 486 rorx $6, e, y1 # y1 = (e >> 6) # S1 487 and e, y2 # y2 = (f^g)&e # CH 488 add y3, old_h # h = t1 + S0 + MAJ # -- 489 490 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 491 rorx $13, a, T1 # T1 = a >> 13 # S0B 492 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 493 rorx $22, a, y1 # y1 = a >> 22 # S0A 494 mov a, y3 # y3 = a # MAJA 495 496 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 497 rorx $2, a, T1 # T1 = (a >> 2) # S0 498 offset = 4*3 + \disp 499 addl offset(%rsp, SRND), h # h = k + w + h # -- 500 or c, y3 # y3 = a|c # MAJA 501 502 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 503 mov a, T1 # T1 = a # MAJB 504 and b, y3 # y3 = (a|c)&b # MAJA 505 and c, T1 # T1 = a&c # MAJB 506 add y0, y2 # y2 = S1 + CH # -- 507 508 509 add h, d # d = k + w + h + d # -- 510 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 511 add y1, h # h = k + w + h + S0 # -- 512 513 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 514 515 516 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 517 518 add y3, h # h = t1 + S0 + MAJ # -- 519 520 ROTATE_ARGS 521 522.endm 523 524######################################################################## 525## void sha256_transform_rorx(void *input_data, UINT32 digest[8], UINT64 num_blks) 526## arg 1 : pointer to digest 527## arg 2 : pointer to input data 528## arg 3 : Num blocks 529######################################################################## 530.text 531ENTRY(sha256_transform_rorx) 532.align 32 533 pushq %rbx 534 pushq %rbp 535 pushq %r12 536 pushq %r13 537 pushq %r14 538 pushq %r15 539 540 mov %rsp, %rax 541 subq $STACK_SIZE, %rsp 542 and $-32, %rsp # align rsp to 32 byte boundary 543 mov %rax, _RSP(%rsp) 544 545 546 shl $6, NUM_BLKS # convert to bytes 547 jz done_hash 548 lea -64(INP, NUM_BLKS), NUM_BLKS # pointer to last block 549 mov NUM_BLKS, _INP_END(%rsp) 550 551 cmp NUM_BLKS, INP 552 je only_one_block 553 554 ## load initial digest 555 mov (CTX), a 556 mov 4*1(CTX), b 557 mov 4*2(CTX), c 558 mov 4*3(CTX), d 559 mov 4*4(CTX), e 560 mov 4*5(CTX), f 561 mov 4*6(CTX), g 562 mov 4*7(CTX), h 563 564 vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK 565 vmovdqa _SHUF_00BA(%rip), SHUF_00BA 566 vmovdqa _SHUF_DC00(%rip), SHUF_DC00 567 568 mov CTX, _CTX(%rsp) 569 570loop0: 571 lea K256(%rip), TBL 572 573 ## Load first 16 dwords from two blocks 574 VMOVDQ 0*32(INP),XTMP0 575 VMOVDQ 1*32(INP),XTMP1 576 VMOVDQ 2*32(INP),XTMP2 577 VMOVDQ 3*32(INP),XTMP3 578 579 ## byte swap data 580 vpshufb BYTE_FLIP_MASK, XTMP0, XTMP0 581 vpshufb BYTE_FLIP_MASK, XTMP1, XTMP1 582 vpshufb BYTE_FLIP_MASK, XTMP2, XTMP2 583 vpshufb BYTE_FLIP_MASK, XTMP3, XTMP3 584 585 ## transpose data into high/low halves 586 vperm2i128 $0x20, XTMP2, XTMP0, X0 587 vperm2i128 $0x31, XTMP2, XTMP0, X1 588 vperm2i128 $0x20, XTMP3, XTMP1, X2 589 vperm2i128 $0x31, XTMP3, XTMP1, X3 590 591last_block_enter: 592 add $64, INP 593 mov INP, _INP(%rsp) 594 595 ## schedule 48 input dwords, by doing 3 rounds of 12 each 596 xor SRND, SRND 597 598.align 16 599loop1: 600 vpaddd 0*32(TBL, SRND), X0, XFER 601 vmovdqa XFER, 0*32+_XFER(%rsp, SRND) 602 FOUR_ROUNDS_AND_SCHED _XFER + 0*32 603 604 vpaddd 1*32(TBL, SRND), X0, XFER 605 vmovdqa XFER, 1*32+_XFER(%rsp, SRND) 606 FOUR_ROUNDS_AND_SCHED _XFER + 1*32 607 608 vpaddd 2*32(TBL, SRND), X0, XFER 609 vmovdqa XFER, 2*32+_XFER(%rsp, SRND) 610 FOUR_ROUNDS_AND_SCHED _XFER + 2*32 611 612 vpaddd 3*32(TBL, SRND), X0, XFER 613 vmovdqa XFER, 3*32+_XFER(%rsp, SRND) 614 FOUR_ROUNDS_AND_SCHED _XFER + 3*32 615 616 add $4*32, SRND 617 cmp $3*4*32, SRND 618 jb loop1 619 620loop2: 621 ## Do last 16 rounds with no scheduling 622 vpaddd 0*32(TBL, SRND), X0, XFER 623 vmovdqa XFER, 0*32+_XFER(%rsp, SRND) 624 DO_4ROUNDS _XFER + 0*32 625 vpaddd 1*32(TBL, SRND), X1, XFER 626 vmovdqa XFER, 1*32+_XFER(%rsp, SRND) 627 DO_4ROUNDS _XFER + 1*32 628 add $2*32, SRND 629 630 vmovdqa X2, X0 631 vmovdqa X3, X1 632 633 cmp $4*4*32, SRND 634 jb loop2 635 636 mov _CTX(%rsp), CTX 637 mov _INP(%rsp), INP 638 639 addm (4*0)(CTX),a 640 addm (4*1)(CTX),b 641 addm (4*2)(CTX),c 642 addm (4*3)(CTX),d 643 addm (4*4)(CTX),e 644 addm (4*5)(CTX),f 645 addm (4*6)(CTX),g 646 addm (4*7)(CTX),h 647 648 cmp _INP_END(%rsp), INP 649 ja done_hash 650 651 #### Do second block using previously scheduled results 652 xor SRND, SRND 653.align 16 654loop3: 655 DO_4ROUNDS _XFER + 0*32 + 16 656 DO_4ROUNDS _XFER + 1*32 + 16 657 add $2*32, SRND 658 cmp $4*4*32, SRND 659 jb loop3 660 661 mov _CTX(%rsp), CTX 662 mov _INP(%rsp), INP 663 add $64, INP 664 665 addm (4*0)(CTX),a 666 addm (4*1)(CTX),b 667 addm (4*2)(CTX),c 668 addm (4*3)(CTX),d 669 addm (4*4)(CTX),e 670 addm (4*5)(CTX),f 671 addm (4*6)(CTX),g 672 addm (4*7)(CTX),h 673 674 cmp _INP_END(%rsp), INP 675 jb loop0 676 ja done_hash 677 678do_last_block: 679 #### do last block 680 lea K256(%rip), TBL 681 682 VMOVDQ 0*16(INP),XWORD0 683 VMOVDQ 1*16(INP),XWORD1 684 VMOVDQ 2*16(INP),XWORD2 685 VMOVDQ 3*16(INP),XWORD3 686 687 vpshufb X_BYTE_FLIP_MASK, XWORD0, XWORD0 688 vpshufb X_BYTE_FLIP_MASK, XWORD1, XWORD1 689 vpshufb X_BYTE_FLIP_MASK, XWORD2, XWORD2 690 vpshufb X_BYTE_FLIP_MASK, XWORD3, XWORD3 691 692 jmp last_block_enter 693 694only_one_block: 695 696 ## load initial digest 697 mov (4*0)(CTX),a 698 mov (4*1)(CTX),b 699 mov (4*2)(CTX),c 700 mov (4*3)(CTX),d 701 mov (4*4)(CTX),e 702 mov (4*5)(CTX),f 703 mov (4*6)(CTX),g 704 mov (4*7)(CTX),h 705 706 vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK 707 vmovdqa _SHUF_00BA(%rip), SHUF_00BA 708 vmovdqa _SHUF_DC00(%rip), SHUF_DC00 709 710 mov CTX, _CTX(%rsp) 711 jmp do_last_block 712 713done_hash: 714 715 mov _RSP(%rsp), %rsp 716 717 popq %r15 718 popq %r14 719 popq %r13 720 popq %r12 721 popq %rbp 722 popq %rbx 723 ret 724ENDPROC(sha256_transform_rorx) 725 726.data 727.align 64 728K256: 729 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 730 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 731 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 732 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 733 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 734 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 735 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 736 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 737 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc 738 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc 739 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da 740 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da 741 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 742 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 743 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 744 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 745 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 746 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 747 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 748 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 749 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 750 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 751 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 752 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 753 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 754 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 755 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 756 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 757 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 758 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 759 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 760 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 761 762PSHUFFLE_BYTE_FLIP_MASK: 763 .octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203 764 765# shuffle xBxA -> 00BA 766_SHUF_00BA: 767 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100 768 769# shuffle xDxC -> DC00 770_SHUF_DC00: 771 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF 772#endif 773