1/* 2 * Twofish Cipher 8-way parallel algorithm (AVX/x86_64) 3 * 4 * Copyright (C) 2012 Johannes Goetzfried 5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 6 * 7 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software 21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 22 * USA 23 * 24 */ 25 26#include <linux/linkage.h> 27#include <asm/frame.h> 28#include "glue_helper-asm-avx.S" 29 30.file "twofish-avx-x86_64-asm_64.S" 31 32.section .rodata.cst16.bswap128_mask, "aM", @progbits, 16 33.align 16 34.Lbswap128_mask: 35 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 36 37.section .rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16 38.align 16 39.Lxts_gf128mul_and_shl1_mask: 40 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 41 42.text 43 44/* structure of crypto context */ 45#define s0 0 46#define s1 1024 47#define s2 2048 48#define s3 3072 49#define w 4096 50#define k 4128 51 52/********************************************************************** 53 8-way AVX twofish 54 **********************************************************************/ 55#define CTX %rdi 56 57#define RA1 %xmm0 58#define RB1 %xmm1 59#define RC1 %xmm2 60#define RD1 %xmm3 61 62#define RA2 %xmm4 63#define RB2 %xmm5 64#define RC2 %xmm6 65#define RD2 %xmm7 66 67#define RX0 %xmm8 68#define RY0 %xmm9 69 70#define RX1 %xmm10 71#define RY1 %xmm11 72 73#define RK1 %xmm12 74#define RK2 %xmm13 75 76#define RT %xmm14 77#define RR %xmm15 78 79#define RID1 %rbp 80#define RID1d %ebp 81#define RID2 %rsi 82#define RID2d %esi 83 84#define RGI1 %rdx 85#define RGI1bl %dl 86#define RGI1bh %dh 87#define RGI2 %rcx 88#define RGI2bl %cl 89#define RGI2bh %ch 90 91#define RGI3 %rax 92#define RGI3bl %al 93#define RGI3bh %ah 94#define RGI4 %rbx 95#define RGI4bl %bl 96#define RGI4bh %bh 97 98#define RGS1 %r8 99#define RGS1d %r8d 100#define RGS2 %r9 101#define RGS2d %r9d 102#define RGS3 %r10 103#define RGS3d %r10d 104 105 106#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \ 107 movzbl src ## bl, RID1d; \ 108 movzbl src ## bh, RID2d; \ 109 shrq $16, src; \ 110 movl t0(CTX, RID1, 4), dst ## d; \ 111 movl t1(CTX, RID2, 4), RID2d; \ 112 movzbl src ## bl, RID1d; \ 113 xorl RID2d, dst ## d; \ 114 movzbl src ## bh, RID2d; \ 115 interleave_op(il_reg); \ 116 xorl t2(CTX, RID1, 4), dst ## d; \ 117 xorl t3(CTX, RID2, 4), dst ## d; 118 119#define dummy(d) /* do nothing */ 120 121#define shr_next(reg) \ 122 shrq $16, reg; 123 124#define G(gi1, gi2, x, t0, t1, t2, t3) \ 125 lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1); \ 126 lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2); \ 127 \ 128 lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none); \ 129 shlq $32, RGS2; \ 130 orq RGS1, RGS2; \ 131 lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none); \ 132 shlq $32, RGS1; \ 133 orq RGS1, RGS3; 134 135#define round_head_2(a, b, x1, y1, x2, y2) \ 136 vmovq b ## 1, RGI3; \ 137 vpextrq $1, b ## 1, RGI4; \ 138 \ 139 G(RGI1, RGI2, x1, s0, s1, s2, s3); \ 140 vmovq a ## 2, RGI1; \ 141 vpextrq $1, a ## 2, RGI2; \ 142 vmovq RGS2, x1; \ 143 vpinsrq $1, RGS3, x1, x1; \ 144 \ 145 G(RGI3, RGI4, y1, s1, s2, s3, s0); \ 146 vmovq b ## 2, RGI3; \ 147 vpextrq $1, b ## 2, RGI4; \ 148 vmovq RGS2, y1; \ 149 vpinsrq $1, RGS3, y1, y1; \ 150 \ 151 G(RGI1, RGI2, x2, s0, s1, s2, s3); \ 152 vmovq RGS2, x2; \ 153 vpinsrq $1, RGS3, x2, x2; \ 154 \ 155 G(RGI3, RGI4, y2, s1, s2, s3, s0); \ 156 vmovq RGS2, y2; \ 157 vpinsrq $1, RGS3, y2, y2; 158 159#define encround_tail(a, b, c, d, x, y, prerotate) \ 160 vpaddd x, y, x; \ 161 vpaddd x, RK1, RT;\ 162 prerotate(b); \ 163 vpxor RT, c, c; \ 164 vpaddd y, x, y; \ 165 vpaddd y, RK2, y; \ 166 vpsrld $1, c, RT; \ 167 vpslld $(32 - 1), c, c; \ 168 vpor c, RT, c; \ 169 vpxor d, y, d; \ 170 171#define decround_tail(a, b, c, d, x, y, prerotate) \ 172 vpaddd x, y, x; \ 173 vpaddd x, RK1, RT;\ 174 prerotate(a); \ 175 vpxor RT, c, c; \ 176 vpaddd y, x, y; \ 177 vpaddd y, RK2, y; \ 178 vpxor d, y, d; \ 179 vpsrld $1, d, y; \ 180 vpslld $(32 - 1), d, d; \ 181 vpor d, y, d; \ 182 183#define rotate_1l(x) \ 184 vpslld $1, x, RR; \ 185 vpsrld $(32 - 1), x, x; \ 186 vpor x, RR, x; 187 188#define preload_rgi(c) \ 189 vmovq c, RGI1; \ 190 vpextrq $1, c, RGI2; 191 192#define encrypt_round(n, a, b, c, d, preload, prerotate) \ 193 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ 194 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ 195 round_head_2(a, b, RX0, RY0, RX1, RY1); \ 196 encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \ 197 preload(c ## 1); \ 198 encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate); 199 200#define decrypt_round(n, a, b, c, d, preload, prerotate) \ 201 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ 202 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ 203 round_head_2(a, b, RX0, RY0, RX1, RY1); \ 204 decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \ 205 preload(c ## 1); \ 206 decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate); 207 208#define encrypt_cycle(n) \ 209 encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \ 210 encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); 211 212#define encrypt_cycle_last(n) \ 213 encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \ 214 encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy); 215 216#define decrypt_cycle(n) \ 217 decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \ 218 decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); 219 220#define decrypt_cycle_last(n) \ 221 decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \ 222 decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy); 223 224#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ 225 vpunpckldq x1, x0, t0; \ 226 vpunpckhdq x1, x0, t2; \ 227 vpunpckldq x3, x2, t1; \ 228 vpunpckhdq x3, x2, x3; \ 229 \ 230 vpunpcklqdq t1, t0, x0; \ 231 vpunpckhqdq t1, t0, x1; \ 232 vpunpcklqdq x3, t2, x2; \ 233 vpunpckhqdq x3, t2, x3; 234 235#define inpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \ 236 vpxor x0, wkey, x0; \ 237 vpxor x1, wkey, x1; \ 238 vpxor x2, wkey, x2; \ 239 vpxor x3, wkey, x3; \ 240 \ 241 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) 242 243#define outunpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \ 244 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ 245 \ 246 vpxor x0, wkey, x0; \ 247 vpxor x1, wkey, x1; \ 248 vpxor x2, wkey, x2; \ 249 vpxor x3, wkey, x3; 250 251.align 8 252__twofish_enc_blk8: 253 /* input: 254 * %rdi: ctx, CTX 255 * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks 256 * output: 257 * RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks 258 */ 259 260 vmovdqu w(CTX), RK1; 261 262 pushq %rbp; 263 pushq %rbx; 264 pushq %rcx; 265 266 inpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); 267 preload_rgi(RA1); 268 rotate_1l(RD1); 269 inpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); 270 rotate_1l(RD2); 271 272 encrypt_cycle(0); 273 encrypt_cycle(1); 274 encrypt_cycle(2); 275 encrypt_cycle(3); 276 encrypt_cycle(4); 277 encrypt_cycle(5); 278 encrypt_cycle(6); 279 encrypt_cycle_last(7); 280 281 vmovdqu (w+4*4)(CTX), RK1; 282 283 popq %rcx; 284 popq %rbx; 285 popq %rbp; 286 287 outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); 288 outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); 289 290 ret; 291ENDPROC(__twofish_enc_blk8) 292 293.align 8 294__twofish_dec_blk8: 295 /* input: 296 * %rdi: ctx, CTX 297 * RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks 298 * output: 299 * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: decrypted blocks 300 */ 301 302 vmovdqu (w+4*4)(CTX), RK1; 303 304 pushq %rbp; 305 pushq %rbx; 306 307 inpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); 308 preload_rgi(RC1); 309 rotate_1l(RA1); 310 inpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); 311 rotate_1l(RA2); 312 313 decrypt_cycle(7); 314 decrypt_cycle(6); 315 decrypt_cycle(5); 316 decrypt_cycle(4); 317 decrypt_cycle(3); 318 decrypt_cycle(2); 319 decrypt_cycle(1); 320 decrypt_cycle_last(0); 321 322 vmovdqu (w)(CTX), RK1; 323 324 popq %rbx; 325 popq %rbp; 326 327 outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); 328 outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); 329 330 ret; 331ENDPROC(__twofish_dec_blk8) 332 333ENTRY(twofish_ecb_enc_8way) 334 /* input: 335 * %rdi: ctx, CTX 336 * %rsi: dst 337 * %rdx: src 338 */ 339 FRAME_BEGIN 340 341 movq %rsi, %r11; 342 343 load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 344 345 call __twofish_enc_blk8; 346 347 store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); 348 349 FRAME_END 350 ret; 351ENDPROC(twofish_ecb_enc_8way) 352 353ENTRY(twofish_ecb_dec_8way) 354 /* input: 355 * %rdi: ctx, CTX 356 * %rsi: dst 357 * %rdx: src 358 */ 359 FRAME_BEGIN 360 361 movq %rsi, %r11; 362 363 load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); 364 365 call __twofish_dec_blk8; 366 367 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 368 369 FRAME_END 370 ret; 371ENDPROC(twofish_ecb_dec_8way) 372 373ENTRY(twofish_cbc_dec_8way) 374 /* input: 375 * %rdi: ctx, CTX 376 * %rsi: dst 377 * %rdx: src 378 */ 379 FRAME_BEGIN 380 381 pushq %r12; 382 383 movq %rsi, %r11; 384 movq %rdx, %r12; 385 386 load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); 387 388 call __twofish_dec_blk8; 389 390 store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 391 392 popq %r12; 393 394 FRAME_END 395 ret; 396ENDPROC(twofish_cbc_dec_8way) 397 398ENTRY(twofish_ctr_8way) 399 /* input: 400 * %rdi: ctx, CTX 401 * %rsi: dst 402 * %rdx: src 403 * %rcx: iv (little endian, 128bit) 404 */ 405 FRAME_BEGIN 406 407 pushq %r12; 408 409 movq %rsi, %r11; 410 movq %rdx, %r12; 411 412 load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, 413 RD2, RX0, RX1, RY0); 414 415 call __twofish_enc_blk8; 416 417 store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); 418 419 popq %r12; 420 421 FRAME_END 422 ret; 423ENDPROC(twofish_ctr_8way) 424 425ENTRY(twofish_xts_enc_8way) 426 /* input: 427 * %rdi: ctx, CTX 428 * %rsi: dst 429 * %rdx: src 430 * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) 431 */ 432 FRAME_BEGIN 433 434 movq %rsi, %r11; 435 436 /* regs <= src, dst <= IVs, regs <= regs xor IVs */ 437 load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, 438 RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); 439 440 call __twofish_enc_blk8; 441 442 /* dst <= regs xor IVs(in dst) */ 443 store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); 444 445 FRAME_END 446 ret; 447ENDPROC(twofish_xts_enc_8way) 448 449ENTRY(twofish_xts_dec_8way) 450 /* input: 451 * %rdi: ctx, CTX 452 * %rsi: dst 453 * %rdx: src 454 * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) 455 */ 456 FRAME_BEGIN 457 458 movq %rsi, %r11; 459 460 /* regs <= src, dst <= IVs, regs <= regs xor IVs */ 461 load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2, 462 RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); 463 464 call __twofish_dec_blk8; 465 466 /* dst <= regs xor IVs(in dst) */ 467 store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 468 469 FRAME_END 470 ret; 471ENDPROC(twofish_xts_dec_8way) 472