1*2b1ef7aeSEric Biggers/* SPDX-License-Identifier: GPL-2.0-only */ 2*2b1ef7aeSEric Biggers/* 3*2b1ef7aeSEric Biggers * Scalar AES core transform 4*2b1ef7aeSEric Biggers * 5*2b1ef7aeSEric Biggers * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6*2b1ef7aeSEric Biggers */ 7*2b1ef7aeSEric Biggers 8*2b1ef7aeSEric Biggers#include <linux/linkage.h> 9*2b1ef7aeSEric Biggers#include <asm/assembler.h> 10*2b1ef7aeSEric Biggers#include <asm/cache.h> 11*2b1ef7aeSEric Biggers 12*2b1ef7aeSEric Biggers .text 13*2b1ef7aeSEric Biggers 14*2b1ef7aeSEric Biggers rk .req x0 15*2b1ef7aeSEric Biggers out .req x1 16*2b1ef7aeSEric Biggers in .req x2 17*2b1ef7aeSEric Biggers rounds .req x3 18*2b1ef7aeSEric Biggers tt .req x2 19*2b1ef7aeSEric Biggers 20*2b1ef7aeSEric Biggers .macro __pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift 21*2b1ef7aeSEric Biggers .ifc \op\shift, b0 22*2b1ef7aeSEric Biggers ubfiz \reg0, \in0, #2, #8 23*2b1ef7aeSEric Biggers ubfiz \reg1, \in1e, #2, #8 24*2b1ef7aeSEric Biggers .else 25*2b1ef7aeSEric Biggers ubfx \reg0, \in0, #\shift, #8 26*2b1ef7aeSEric Biggers ubfx \reg1, \in1e, #\shift, #8 27*2b1ef7aeSEric Biggers .endif 28*2b1ef7aeSEric Biggers 29*2b1ef7aeSEric Biggers /* 30*2b1ef7aeSEric Biggers * AArch64 cannot do byte size indexed loads from a table containing 31*2b1ef7aeSEric Biggers * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a 32*2b1ef7aeSEric Biggers * valid instruction. So perform the shift explicitly first for the 33*2b1ef7aeSEric Biggers * high bytes (the low byte is shifted implicitly by using ubfiz rather 34*2b1ef7aeSEric Biggers * than ubfx above) 35*2b1ef7aeSEric Biggers */ 36*2b1ef7aeSEric Biggers .ifnc \op, b 37*2b1ef7aeSEric Biggers ldr \reg0, [tt, \reg0, uxtw #2] 38*2b1ef7aeSEric Biggers ldr \reg1, [tt, \reg1, uxtw #2] 39*2b1ef7aeSEric Biggers .else 40*2b1ef7aeSEric Biggers .if \shift > 0 41*2b1ef7aeSEric Biggers lsl \reg0, \reg0, #2 42*2b1ef7aeSEric Biggers lsl \reg1, \reg1, #2 43*2b1ef7aeSEric Biggers .endif 44*2b1ef7aeSEric Biggers ldrb \reg0, [tt, \reg0, uxtw] 45*2b1ef7aeSEric Biggers ldrb \reg1, [tt, \reg1, uxtw] 46*2b1ef7aeSEric Biggers .endif 47*2b1ef7aeSEric Biggers .endm 48*2b1ef7aeSEric Biggers 49*2b1ef7aeSEric Biggers .macro __pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift 50*2b1ef7aeSEric Biggers ubfx \reg0, \in0, #\shift, #8 51*2b1ef7aeSEric Biggers ubfx \reg1, \in1d, #\shift, #8 52*2b1ef7aeSEric Biggers ldr\op \reg0, [tt, \reg0, uxtw #\sz] 53*2b1ef7aeSEric Biggers ldr\op \reg1, [tt, \reg1, uxtw #\sz] 54*2b1ef7aeSEric Biggers .endm 55*2b1ef7aeSEric Biggers 56*2b1ef7aeSEric Biggers .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op 57*2b1ef7aeSEric Biggers ldp \out0, \out1, [rk], #8 58*2b1ef7aeSEric Biggers 59*2b1ef7aeSEric Biggers __pair\enc \sz, \op, w12, w13, \in0, \in1, \in3, 0 60*2b1ef7aeSEric Biggers __pair\enc \sz, \op, w14, w15, \in1, \in2, \in0, 8 61*2b1ef7aeSEric Biggers __pair\enc \sz, \op, w16, w17, \in2, \in3, \in1, 16 62*2b1ef7aeSEric Biggers __pair\enc \sz, \op, \t0, \t1, \in3, \in0, \in2, 24 63*2b1ef7aeSEric Biggers 64*2b1ef7aeSEric Biggers eor \out0, \out0, w12 65*2b1ef7aeSEric Biggers eor \out1, \out1, w13 66*2b1ef7aeSEric Biggers eor \out0, \out0, w14, ror #24 67*2b1ef7aeSEric Biggers eor \out1, \out1, w15, ror #24 68*2b1ef7aeSEric Biggers eor \out0, \out0, w16, ror #16 69*2b1ef7aeSEric Biggers eor \out1, \out1, w17, ror #16 70*2b1ef7aeSEric Biggers eor \out0, \out0, \t0, ror #8 71*2b1ef7aeSEric Biggers eor \out1, \out1, \t1, ror #8 72*2b1ef7aeSEric Biggers .endm 73*2b1ef7aeSEric Biggers 74*2b1ef7aeSEric Biggers .macro fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op 75*2b1ef7aeSEric Biggers __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op 76*2b1ef7aeSEric Biggers __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op 77*2b1ef7aeSEric Biggers .endm 78*2b1ef7aeSEric Biggers 79*2b1ef7aeSEric Biggers .macro iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op 80*2b1ef7aeSEric Biggers __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op 81*2b1ef7aeSEric Biggers __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op 82*2b1ef7aeSEric Biggers .endm 83*2b1ef7aeSEric Biggers 84*2b1ef7aeSEric Biggers .macro do_crypt, round, ttab, ltab, bsz 85*2b1ef7aeSEric Biggers ldp w4, w5, [in] 86*2b1ef7aeSEric Biggers ldp w6, w7, [in, #8] 87*2b1ef7aeSEric Biggers ldp w8, w9, [rk], #16 88*2b1ef7aeSEric Biggers ldp w10, w11, [rk, #-8] 89*2b1ef7aeSEric Biggers 90*2b1ef7aeSEric BiggersCPU_BE( rev w4, w4 ) 91*2b1ef7aeSEric BiggersCPU_BE( rev w5, w5 ) 92*2b1ef7aeSEric BiggersCPU_BE( rev w6, w6 ) 93*2b1ef7aeSEric BiggersCPU_BE( rev w7, w7 ) 94*2b1ef7aeSEric Biggers 95*2b1ef7aeSEric Biggers eor w4, w4, w8 96*2b1ef7aeSEric Biggers eor w5, w5, w9 97*2b1ef7aeSEric Biggers eor w6, w6, w10 98*2b1ef7aeSEric Biggers eor w7, w7, w11 99*2b1ef7aeSEric Biggers 100*2b1ef7aeSEric Biggers adr_l tt, \ttab 101*2b1ef7aeSEric Biggers 102*2b1ef7aeSEric Biggers tbnz rounds, #1, 1f 103*2b1ef7aeSEric Biggers 104*2b1ef7aeSEric Biggers0: \round w8, w9, w10, w11, w4, w5, w6, w7 105*2b1ef7aeSEric Biggers \round w4, w5, w6, w7, w8, w9, w10, w11 106*2b1ef7aeSEric Biggers 107*2b1ef7aeSEric Biggers1: subs rounds, rounds, #4 108*2b1ef7aeSEric Biggers \round w8, w9, w10, w11, w4, w5, w6, w7 109*2b1ef7aeSEric Biggers b.ls 3f 110*2b1ef7aeSEric Biggers2: \round w4, w5, w6, w7, w8, w9, w10, w11 111*2b1ef7aeSEric Biggers b 0b 112*2b1ef7aeSEric Biggers3: adr_l tt, \ltab 113*2b1ef7aeSEric Biggers \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b 114*2b1ef7aeSEric Biggers 115*2b1ef7aeSEric BiggersCPU_BE( rev w4, w4 ) 116*2b1ef7aeSEric BiggersCPU_BE( rev w5, w5 ) 117*2b1ef7aeSEric BiggersCPU_BE( rev w6, w6 ) 118*2b1ef7aeSEric BiggersCPU_BE( rev w7, w7 ) 119*2b1ef7aeSEric Biggers 120*2b1ef7aeSEric Biggers stp w4, w5, [out] 121*2b1ef7aeSEric Biggers stp w6, w7, [out, #8] 122*2b1ef7aeSEric Biggers ret 123*2b1ef7aeSEric Biggers .endm 124*2b1ef7aeSEric Biggers 125*2b1ef7aeSEric BiggersSYM_FUNC_START(__aes_arm64_encrypt) 126*2b1ef7aeSEric Biggers do_crypt fround, aes_enc_tab, aes_enc_tab + 1, 2 127*2b1ef7aeSEric BiggersSYM_FUNC_END(__aes_arm64_encrypt) 128*2b1ef7aeSEric Biggers 129*2b1ef7aeSEric Biggers .align 5 130*2b1ef7aeSEric BiggersSYM_FUNC_START(__aes_arm64_decrypt) 131*2b1ef7aeSEric Biggers do_crypt iround, aes_dec_tab, crypto_aes_inv_sbox, 0 132*2b1ef7aeSEric BiggersSYM_FUNC_END(__aes_arm64_decrypt) 133