1/* 2 * Intel SHA Extensions optimized implementation of a SHA-1 update function 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * Copyright(c) 2015 Intel Corporation. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * Contact Information: 21 * Sean Gulley <sean.m.gulley@intel.com> 22 * Tim Chen <tim.c.chen@linux.intel.com> 23 * 24 * BSD LICENSE 25 * 26 * Copyright(c) 2015 Intel Corporation. 27 * 28 * Redistribution and use in source and binary forms, with or without 29 * modification, are permitted provided that the following conditions 30 * are met: 31 * 32 * * Redistributions of source code must retain the above copyright 33 * notice, this list of conditions and the following disclaimer. 34 * * Redistributions in binary form must reproduce the above copyright 35 * notice, this list of conditions and the following disclaimer in 36 * the documentation and/or other materials provided with the 37 * distribution. 38 * * Neither the name of Intel Corporation nor the names of its 39 * contributors may be used to endorse or promote products derived 40 * from this software without specific prior written permission. 41 * 42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 53 * 54 */ 55 56/* 57 * Copyright (c) 2018, Joyent, Inc. 58 */ 59 60/* 61 * illumos uses this file under the terms of the BSD license. 62 * 63 * The following are a series of changes that we have made to this code: 64 * 65 * o Changed the include to be sys/asm_linkage.h 66 * o Use the sys/asm_linkage.h prototypes for assembly functions 67 * o Renamed the function from sha1_ni_transform to sha1_block_data_order to 68 * match the illumos name for the function 69 */ 70 71#include <sys/asm_linkage.h> 72 73#define DIGEST_PTR %rdi /* 1st arg */ 74#define DATA_PTR %rsi /* 2nd arg */ 75#define NUM_BLKS %rdx /* 3rd arg */ 76 77#define RSPSAVE %rax 78 79/* gcc conversion */ 80#define FRAME_SIZE 32 /* space for 2x16 bytes */ 81 82#define ABCD %xmm0 83#define E0 %xmm1 /* Need two E's b/c they ping pong */ 84#define E1 %xmm2 85#define MSG0 %xmm3 86#define MSG1 %xmm4 87#define MSG2 %xmm5 88#define MSG3 %xmm6 89#define SHUF_MASK %xmm7 90 91 92/* 93 * Intel SHA Extensions optimized implementation of a SHA-1 update function 94 * 95 * The function takes a pointer to the current hash values, a pointer to the 96 * input data, and a number of 64 byte blocks to process. Once all blocks have 97 * been processed, the digest pointer is updated with the resulting hash value. 98 * The function only processes complete blocks, there is no functionality to 99 * store partial blocks. All message padding and hash value initialization must 100 * be done outside the update function. 101 * 102 * The indented lines in the loop are instructions related to rounds processing. 103 * The non-indented lines are instructions related to the message schedule. 104 * 105 * void sha1_block_data_order(uint32_t *digest, const void *data, 106 uint32_t numBlocks) 107 * digest : pointer to digest 108 * data: pointer to input data 109 * numBlocks: Number of blocks to process 110 */ 111.text 112.align 32 113ENTRY_NP(sha1_block_data_order) 114 mov %rsp, RSPSAVE 115 sub $FRAME_SIZE, %rsp 116 and $~0xF, %rsp 117 118 shl $6, NUM_BLKS /* convert to bytes */ 119 jz .Ldone_hash 120 add DATA_PTR, NUM_BLKS /* pointer to end of data */ 121 122 /* load initial hash values */ 123 pinsrd $3, 1*16(DIGEST_PTR), E0 124 movdqu 0*16(DIGEST_PTR), ABCD 125 pand UPPER_WORD_MASK(%rip), E0 126 pshufd $0x1B, ABCD, ABCD 127 128 movdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK 129 130.Lloop0: 131 /* Save hash values for addition after rounds */ 132 movdqa E0, (0*16)(%rsp) 133 movdqa ABCD, (1*16)(%rsp) 134 135 /* Rounds 0-3 */ 136 movdqu 0*16(DATA_PTR), MSG0 137 pshufb SHUF_MASK, MSG0 138 paddd MSG0, E0 139 movdqa ABCD, E1 140 sha1rnds4 $0, E0, ABCD 141 142 /* Rounds 4-7 */ 143 movdqu 1*16(DATA_PTR), MSG1 144 pshufb SHUF_MASK, MSG1 145 sha1nexte MSG1, E1 146 movdqa ABCD, E0 147 sha1rnds4 $0, E1, ABCD 148 sha1msg1 MSG1, MSG0 149 150 /* Rounds 8-11 */ 151 movdqu 2*16(DATA_PTR), MSG2 152 pshufb SHUF_MASK, MSG2 153 sha1nexte MSG2, E0 154 movdqa ABCD, E1 155 sha1rnds4 $0, E0, ABCD 156 sha1msg1 MSG2, MSG1 157 pxor MSG2, MSG0 158 159 /* Rounds 12-15 */ 160 movdqu 3*16(DATA_PTR), MSG3 161 pshufb SHUF_MASK, MSG3 162 sha1nexte MSG3, E1 163 movdqa ABCD, E0 164 sha1msg2 MSG3, MSG0 165 sha1rnds4 $0, E1, ABCD 166 sha1msg1 MSG3, MSG2 167 pxor MSG3, MSG1 168 169 /* Rounds 16-19 */ 170 sha1nexte MSG0, E0 171 movdqa ABCD, E1 172 sha1msg2 MSG0, MSG1 173 sha1rnds4 $0, E0, ABCD 174 sha1msg1 MSG0, MSG3 175 pxor MSG0, MSG2 176 177 /* Rounds 20-23 */ 178 sha1nexte MSG1, E1 179 movdqa ABCD, E0 180 sha1msg2 MSG1, MSG2 181 sha1rnds4 $1, E1, ABCD 182 sha1msg1 MSG1, MSG0 183 pxor MSG1, MSG3 184 185 /* Rounds 24-27 */ 186 sha1nexte MSG2, E0 187 movdqa ABCD, E1 188 sha1msg2 MSG2, MSG3 189 sha1rnds4 $1, E0, ABCD 190 sha1msg1 MSG2, MSG1 191 pxor MSG2, MSG0 192 193 /* Rounds 28-31 */ 194 sha1nexte MSG3, E1 195 movdqa ABCD, E0 196 sha1msg2 MSG3, MSG0 197 sha1rnds4 $1, E1, ABCD 198 sha1msg1 MSG3, MSG2 199 pxor MSG3, MSG1 200 201 /* Rounds 32-35 */ 202 sha1nexte MSG0, E0 203 movdqa ABCD, E1 204 sha1msg2 MSG0, MSG1 205 sha1rnds4 $1, E0, ABCD 206 sha1msg1 MSG0, MSG3 207 pxor MSG0, MSG2 208 209 /* Rounds 36-39 */ 210 sha1nexte MSG1, E1 211 movdqa ABCD, E0 212 sha1msg2 MSG1, MSG2 213 sha1rnds4 $1, E1, ABCD 214 sha1msg1 MSG1, MSG0 215 pxor MSG1, MSG3 216 217 /* Rounds 40-43 */ 218 sha1nexte MSG2, E0 219 movdqa ABCD, E1 220 sha1msg2 MSG2, MSG3 221 sha1rnds4 $2, E0, ABCD 222 sha1msg1 MSG2, MSG1 223 pxor MSG2, MSG0 224 225 /* Rounds 44-47 */ 226 sha1nexte MSG3, E1 227 movdqa ABCD, E0 228 sha1msg2 MSG3, MSG0 229 sha1rnds4 $2, E1, ABCD 230 sha1msg1 MSG3, MSG2 231 pxor MSG3, MSG1 232 233 /* Rounds 48-51 */ 234 sha1nexte MSG0, E0 235 movdqa ABCD, E1 236 sha1msg2 MSG0, MSG1 237 sha1rnds4 $2, E0, ABCD 238 sha1msg1 MSG0, MSG3 239 pxor MSG0, MSG2 240 241 /* Rounds 52-55 */ 242 sha1nexte MSG1, E1 243 movdqa ABCD, E0 244 sha1msg2 MSG1, MSG2 245 sha1rnds4 $2, E1, ABCD 246 sha1msg1 MSG1, MSG0 247 pxor MSG1, MSG3 248 249 /* Rounds 56-59 */ 250 sha1nexte MSG2, E0 251 movdqa ABCD, E1 252 sha1msg2 MSG2, MSG3 253 sha1rnds4 $2, E0, ABCD 254 sha1msg1 MSG2, MSG1 255 pxor MSG2, MSG0 256 257 /* Rounds 60-63 */ 258 sha1nexte MSG3, E1 259 movdqa ABCD, E0 260 sha1msg2 MSG3, MSG0 261 sha1rnds4 $3, E1, ABCD 262 sha1msg1 MSG3, MSG2 263 pxor MSG3, MSG1 264 265 /* Rounds 64-67 */ 266 sha1nexte MSG0, E0 267 movdqa ABCD, E1 268 sha1msg2 MSG0, MSG1 269 sha1rnds4 $3, E0, ABCD 270 sha1msg1 MSG0, MSG3 271 pxor MSG0, MSG2 272 273 /* Rounds 68-71 */ 274 sha1nexte MSG1, E1 275 movdqa ABCD, E0 276 sha1msg2 MSG1, MSG2 277 sha1rnds4 $3, E1, ABCD 278 pxor MSG1, MSG3 279 280 /* Rounds 72-75 */ 281 sha1nexte MSG2, E0 282 movdqa ABCD, E1 283 sha1msg2 MSG2, MSG3 284 sha1rnds4 $3, E0, ABCD 285 286 /* Rounds 76-79 */ 287 sha1nexte MSG3, E1 288 movdqa ABCD, E0 289 sha1rnds4 $3, E1, ABCD 290 291 /* Add current hash values with previously saved */ 292 sha1nexte (0*16)(%rsp), E0 293 paddd (1*16)(%rsp), ABCD 294 295 /* Increment data pointer and loop if more to process */ 296 add $64, DATA_PTR 297 cmp NUM_BLKS, DATA_PTR 298 jne .Lloop0 299 300 /* Write hash values back in the correct order */ 301 pshufd $0x1B, ABCD, ABCD 302 movdqu ABCD, 0*16(DIGEST_PTR) 303 pextrd $3, E0, 1*16(DIGEST_PTR) 304 305.Ldone_hash: 306 mov RSPSAVE, %rsp 307 308 ret 309SET_SIZE(sha1_block_data_order) 310 311.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16 312.align 16 313PSHUFFLE_BYTE_FLIP_MASK: 314 .octa 0x000102030405060708090a0b0c0d0e0f 315 316.section .rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16 317.align 16 318UPPER_WORD_MASK: 319 .octa 0xFFFFFFFF000000000000000000000000 320