1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * Fast AES implementation for SPE instruction set (PPC) 4 * 5 * This code makes use of the SPE SIMD instruction set as defined in 6 * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf 7 * Implementation is based on optimization guide notes from 8 * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf 9 * 10 * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> 11 */ 12 13#include <asm/ppc_asm.h> 14#include "aes-spe-regs.h" 15 16#define EAD(in, bpos) \ 17 rlwimi rT0,in,28-((bpos+3)%4)*8,20,27; 18 19#define DAD(in, bpos) \ 20 rlwimi rT1,in,24-((bpos+3)%4)*8,24,31; 21 22#define LWH(out, off) \ 23 evlwwsplat out,off(rT0); /* load word high */ 24 25#define LWL(out, off) \ 26 lwz out,off(rT0); /* load word low */ 27 28#define LBZ(out, tab, off) \ 29 lbz out,off(tab); /* load byte */ 30 31#define LAH(out, in, bpos, off) \ 32 EAD(in, bpos) /* calc addr + load word high */ \ 33 LWH(out, off) 34 35#define LAL(out, in, bpos, off) \ 36 EAD(in, bpos) /* calc addr + load word low */ \ 37 LWL(out, off) 38 39#define LAE(out, in, bpos) \ 40 EAD(in, bpos) /* calc addr + load enc byte */ \ 41 LBZ(out, rT0, 8) 42 43#define LBE(out) \ 44 LBZ(out, rT0, 8) /* load enc byte */ 45 46#define LAD(out, in, bpos) \ 47 DAD(in, bpos) /* calc addr + load dec byte */ \ 48 LBZ(out, rT1, 0) 49 50#define LBD(out) \ 51 LBZ(out, rT1, 0) 52 53/* 54 * ppc_encrypt_block: The central encryption function for a single 16 bytes 55 * block. It does no stack handling or register saving to support fast calls 56 * via bl/blr. It expects that caller has pre-xored input data with first 57 * 4 words of encryption key into rD0-rD3. Pointer/counter registers must 58 * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3 59 * and rW0-rW3 and caller must execute a final xor on the output registers. 60 * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing. 61 * 62 */ 63_GLOBAL(ppc_encrypt_block) 64 LAH(rW4, rD1, 2, 4) 65 LAH(rW6, rD0, 3, 0) 66 LAH(rW3, rD0, 1, 8) 67ppc_encrypt_block_loop: 68 LAH(rW0, rD3, 0, 12) 69 LAL(rW0, rD0, 0, 12) 70 LAH(rW1, rD1, 0, 12) 71 LAH(rW2, rD2, 1, 8) 72 LAL(rW2, rD3, 1, 8) 73 LAL(rW3, rD1, 1, 8) 74 LAL(rW4, rD2, 2, 4) 75 LAL(rW6, rD1, 3, 0) 76 LAH(rW5, rD3, 2, 4) 77 LAL(rW5, rD0, 2, 4) 78 LAH(rW7, rD2, 3, 0) 79 evldw rD1,16(rKP) 80 EAD(rD3, 3) 81 evxor rW2,rW2,rW4 82 LWL(rW7, 0) 83 evxor rW2,rW2,rW6 84 EAD(rD2, 0) 85 evxor rD1,rD1,rW2 86 LWL(rW1, 12) 87 evxor rD1,rD1,rW0 88 evldw rD3,24(rKP) 89 evmergehi rD0,rD0,rD1 90 EAD(rD1, 2) 91 evxor rW3,rW3,rW5 92 LWH(rW4, 4) 93 evxor rW3,rW3,rW7 94 EAD(rD0, 3) 95 evxor rD3,rD3,rW3 96 LWH(rW6, 0) 97 evxor rD3,rD3,rW1 98 EAD(rD0, 1) 99 evmergehi rD2,rD2,rD3 100 LWH(rW3, 8) 101 LAH(rW0, rD3, 0, 12) 102 LAL(rW0, rD0, 0, 12) 103 LAH(rW1, rD1, 0, 12) 104 LAH(rW2, rD2, 1, 8) 105 LAL(rW2, rD3, 1, 8) 106 LAL(rW3, rD1, 1, 8) 107 LAL(rW4, rD2, 2, 4) 108 LAL(rW6, rD1, 3, 0) 109 LAH(rW5, rD3, 2, 4) 110 LAL(rW5, rD0, 2, 4) 111 LAH(rW7, rD2, 3, 0) 112 evldw rD1,32(rKP) 113 EAD(rD3, 3) 114 evxor rW2,rW2,rW4 115 LWL(rW7, 0) 116 evxor rW2,rW2,rW6 117 EAD(rD2, 0) 118 evxor rD1,rD1,rW2 119 LWL(rW1, 12) 120 evxor rD1,rD1,rW0 121 evldw rD3,40(rKP) 122 evmergehi rD0,rD0,rD1 123 EAD(rD1, 2) 124 evxor rW3,rW3,rW5 125 LWH(rW4, 4) 126 evxor rW3,rW3,rW7 127 EAD(rD0, 3) 128 evxor rD3,rD3,rW3 129 LWH(rW6, 0) 130 evxor rD3,rD3,rW1 131 EAD(rD0, 1) 132 evmergehi rD2,rD2,rD3 133 LWH(rW3, 8) 134 addi rKP,rKP,32 135 bdnz ppc_encrypt_block_loop 136 LAH(rW0, rD3, 0, 12) 137 LAL(rW0, rD0, 0, 12) 138 LAH(rW1, rD1, 0, 12) 139 LAH(rW2, rD2, 1, 8) 140 LAL(rW2, rD3, 1, 8) 141 LAL(rW3, rD1, 1, 8) 142 LAL(rW4, rD2, 2, 4) 143 LAH(rW5, rD3, 2, 4) 144 LAL(rW6, rD1, 3, 0) 145 LAL(rW5, rD0, 2, 4) 146 LAH(rW7, rD2, 3, 0) 147 evldw rD1,16(rKP) 148 EAD(rD3, 3) 149 evxor rW2,rW2,rW4 150 LWL(rW7, 0) 151 evxor rW2,rW2,rW6 152 EAD(rD2, 0) 153 evxor rD1,rD1,rW2 154 LWL(rW1, 12) 155 evxor rD1,rD1,rW0 156 evldw rD3,24(rKP) 157 evmergehi rD0,rD0,rD1 158 EAD(rD1, 0) 159 evxor rW3,rW3,rW5 160 LBE(rW2) 161 evxor rW3,rW3,rW7 162 EAD(rD0, 1) 163 evxor rD3,rD3,rW3 164 LBE(rW6) 165 evxor rD3,rD3,rW1 166 EAD(rD0, 0) 167 evmergehi rD2,rD2,rD3 168 LBE(rW1) 169 LAE(rW0, rD3, 0) 170 LAE(rW1, rD0, 0) 171 LAE(rW4, rD2, 1) 172 LAE(rW5, rD3, 1) 173 LAE(rW3, rD2, 0) 174 LAE(rW7, rD1, 1) 175 rlwimi rW0,rW4,8,16,23 176 rlwimi rW1,rW5,8,16,23 177 LAE(rW4, rD1, 2) 178 LAE(rW5, rD2, 2) 179 rlwimi rW2,rW6,8,16,23 180 rlwimi rW3,rW7,8,16,23 181 LAE(rW6, rD3, 2) 182 LAE(rW7, rD0, 2) 183 rlwimi rW0,rW4,16,8,15 184 rlwimi rW1,rW5,16,8,15 185 LAE(rW4, rD0, 3) 186 LAE(rW5, rD1, 3) 187 rlwimi rW2,rW6,16,8,15 188 lwz rD0,32(rKP) 189 rlwimi rW3,rW7,16,8,15 190 lwz rD1,36(rKP) 191 LAE(rW6, rD2, 3) 192 LAE(rW7, rD3, 3) 193 rlwimi rW0,rW4,24,0,7 194 lwz rD2,40(rKP) 195 rlwimi rW1,rW5,24,0,7 196 lwz rD3,44(rKP) 197 rlwimi rW2,rW6,24,0,7 198 rlwimi rW3,rW7,24,0,7 199 blr 200 201/* 202 * ppc_decrypt_block: The central decryption function for a single 16 bytes 203 * block. It does no stack handling or register saving to support fast calls 204 * via bl/blr. It expects that caller has pre-xored input data with first 205 * 4 words of encryption key into rD0-rD3. Pointer/counter registers must 206 * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3 207 * and rW0-rW3 and caller must execute a final xor on the output registers. 208 * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing. 209 * 210 */ 211_GLOBAL(ppc_decrypt_block) 212 LAH(rW0, rD1, 0, 12) 213 LAH(rW6, rD0, 3, 0) 214 LAH(rW3, rD0, 1, 8) 215ppc_decrypt_block_loop: 216 LAH(rW1, rD3, 0, 12) 217 LAL(rW0, rD2, 0, 12) 218 LAH(rW2, rD2, 1, 8) 219 LAL(rW2, rD3, 1, 8) 220 LAH(rW4, rD3, 2, 4) 221 LAL(rW4, rD0, 2, 4) 222 LAL(rW6, rD1, 3, 0) 223 LAH(rW5, rD1, 2, 4) 224 LAH(rW7, rD2, 3, 0) 225 LAL(rW7, rD3, 3, 0) 226 LAL(rW3, rD1, 1, 8) 227 evldw rD1,16(rKP) 228 EAD(rD0, 0) 229 evxor rW4,rW4,rW6 230 LWL(rW1, 12) 231 evxor rW0,rW0,rW4 232 EAD(rD2, 2) 233 evxor rW0,rW0,rW2 234 LWL(rW5, 4) 235 evxor rD1,rD1,rW0 236 evldw rD3,24(rKP) 237 evmergehi rD0,rD0,rD1 238 EAD(rD1, 0) 239 evxor rW3,rW3,rW7 240 LWH(rW0, 12) 241 evxor rW3,rW3,rW1 242 EAD(rD0, 3) 243 evxor rD3,rD3,rW3 244 LWH(rW6, 0) 245 evxor rD3,rD3,rW5 246 EAD(rD0, 1) 247 evmergehi rD2,rD2,rD3 248 LWH(rW3, 8) 249 LAH(rW1, rD3, 0, 12) 250 LAL(rW0, rD2, 0, 12) 251 LAH(rW2, rD2, 1, 8) 252 LAL(rW2, rD3, 1, 8) 253 LAH(rW4, rD3, 2, 4) 254 LAL(rW4, rD0, 2, 4) 255 LAL(rW6, rD1, 3, 0) 256 LAH(rW5, rD1, 2, 4) 257 LAH(rW7, rD2, 3, 0) 258 LAL(rW7, rD3, 3, 0) 259 LAL(rW3, rD1, 1, 8) 260 evldw rD1,32(rKP) 261 EAD(rD0, 0) 262 evxor rW4,rW4,rW6 263 LWL(rW1, 12) 264 evxor rW0,rW0,rW4 265 EAD(rD2, 2) 266 evxor rW0,rW0,rW2 267 LWL(rW5, 4) 268 evxor rD1,rD1,rW0 269 evldw rD3,40(rKP) 270 evmergehi rD0,rD0,rD1 271 EAD(rD1, 0) 272 evxor rW3,rW3,rW7 273 LWH(rW0, 12) 274 evxor rW3,rW3,rW1 275 EAD(rD0, 3) 276 evxor rD3,rD3,rW3 277 LWH(rW6, 0) 278 evxor rD3,rD3,rW5 279 EAD(rD0, 1) 280 evmergehi rD2,rD2,rD3 281 LWH(rW3, 8) 282 addi rKP,rKP,32 283 bdnz ppc_decrypt_block_loop 284 LAH(rW1, rD3, 0, 12) 285 LAL(rW0, rD2, 0, 12) 286 LAH(rW2, rD2, 1, 8) 287 LAL(rW2, rD3, 1, 8) 288 LAH(rW4, rD3, 2, 4) 289 LAL(rW4, rD0, 2, 4) 290 LAL(rW6, rD1, 3, 0) 291 LAH(rW5, rD1, 2, 4) 292 LAH(rW7, rD2, 3, 0) 293 LAL(rW7, rD3, 3, 0) 294 LAL(rW3, rD1, 1, 8) 295 evldw rD1,16(rKP) 296 EAD(rD0, 0) 297 evxor rW4,rW4,rW6 298 LWL(rW1, 12) 299 evxor rW0,rW0,rW4 300 EAD(rD2, 2) 301 evxor rW0,rW0,rW2 302 LWL(rW5, 4) 303 evxor rD1,rD1,rW0 304 evldw rD3,24(rKP) 305 evmergehi rD0,rD0,rD1 306 DAD(rD1, 0) 307 evxor rW3,rW3,rW7 308 LBD(rW0) 309 evxor rW3,rW3,rW1 310 DAD(rD0, 1) 311 evxor rD3,rD3,rW3 312 LBD(rW6) 313 evxor rD3,rD3,rW5 314 DAD(rD0, 0) 315 evmergehi rD2,rD2,rD3 316 LBD(rW3) 317 LAD(rW2, rD3, 0) 318 LAD(rW1, rD2, 0) 319 LAD(rW4, rD2, 1) 320 LAD(rW5, rD3, 1) 321 LAD(rW7, rD1, 1) 322 rlwimi rW0,rW4,8,16,23 323 rlwimi rW1,rW5,8,16,23 324 LAD(rW4, rD3, 2) 325 LAD(rW5, rD0, 2) 326 rlwimi rW2,rW6,8,16,23 327 rlwimi rW3,rW7,8,16,23 328 LAD(rW6, rD1, 2) 329 LAD(rW7, rD2, 2) 330 rlwimi rW0,rW4,16,8,15 331 rlwimi rW1,rW5,16,8,15 332 LAD(rW4, rD0, 3) 333 LAD(rW5, rD1, 3) 334 rlwimi rW2,rW6,16,8,15 335 lwz rD0,32(rKP) 336 rlwimi rW3,rW7,16,8,15 337 lwz rD1,36(rKP) 338 LAD(rW6, rD2, 3) 339 LAD(rW7, rD3, 3) 340 rlwimi rW0,rW4,24,0,7 341 lwz rD2,40(rKP) 342 rlwimi rW1,rW5,24,0,7 343 lwz rD3,44(rKP) 344 rlwimi rW2,rW6,24,0,7 345 rlwimi rW3,rW7,24,0,7 346 blr 347