1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue code for AES implementation for SPE instructions (PPC) 4 * 5 * Based on generic implementation. The assembler module takes care 6 * about the SPE registers so it can run from interrupt context. 7 * 8 * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> 9 */ 10 11 #include <crypto/aes.h> 12 #include <linux/module.h> 13 #include <linux/init.h> 14 #include <linux/types.h> 15 #include <linux/errno.h> 16 #include <linux/crypto.h> 17 #include <asm/byteorder.h> 18 #include <asm/switch_to.h> 19 #include <crypto/algapi.h> 20 #include <crypto/internal/skcipher.h> 21 #include <crypto/xts.h> 22 #include <crypto/gf128mul.h> 23 #include <crypto/scatterwalk.h> 24 25 /* 26 * MAX_BYTES defines the number of bytes that are allowed to be processed 27 * between preempt_disable() and preempt_enable(). e500 cores can issue two 28 * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32 29 * bit unit (SU2). One of these can be a memory access that is executed via 30 * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per 31 * 16 byte block or 25 cycles per byte. Thus 768 bytes of input data 32 * will need an estimated maximum of 20,000 cycles. Headroom for cache misses 33 * included. Even with the low end model clocked at 667 MHz this equals to a 34 * critical time window of less than 30us. The value has been chosen to 35 * process a 512 byte disk block in one or a large 1400 bytes IPsec network 36 * packet in two runs. 37 * 38 */ 39 #define MAX_BYTES 768 40 41 struct ppc_aes_ctx { 42 u32 key_enc[AES_MAX_KEYLENGTH_U32]; 43 u32 key_dec[AES_MAX_KEYLENGTH_U32]; 44 u32 rounds; 45 }; 46 47 struct ppc_xts_ctx { 48 u32 key_enc[AES_MAX_KEYLENGTH_U32]; 49 u32 key_dec[AES_MAX_KEYLENGTH_U32]; 50 u32 key_twk[AES_MAX_KEYLENGTH_U32]; 51 u32 rounds; 52 }; 53 54 static void spe_begin(void) 55 { 56 /* disable preemption and save users SPE registers if required */ 57 preempt_disable(); 58 enable_kernel_spe(); 59 } 60 61 static void spe_end(void) 62 { 63 disable_kernel_spe(); 64 /* reenable preemption */ 65 preempt_enable(); 66 } 67 68 static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm, 69 const u8 *in_key, unsigned int key_len) 70 { 71 struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 72 73 switch (key_len) { 74 case AES_KEYSIZE_128: 75 ctx->rounds = 4; 76 ppc_expand_key_128(ctx->key_enc, in_key); 77 break; 78 case AES_KEYSIZE_192: 79 ctx->rounds = 5; 80 ppc_expand_key_192(ctx->key_enc, in_key); 81 break; 82 case AES_KEYSIZE_256: 83 ctx->rounds = 6; 84 ppc_expand_key_256(ctx->key_enc, in_key); 85 break; 86 default: 87 return -EINVAL; 88 } 89 90 ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len); 91 92 return 0; 93 } 94 95 static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 96 unsigned int key_len) 97 { 98 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 99 int err; 100 101 err = xts_verify_key(tfm, in_key, key_len); 102 if (err) 103 return err; 104 105 key_len >>= 1; 106 107 switch (key_len) { 108 case AES_KEYSIZE_128: 109 ctx->rounds = 4; 110 ppc_expand_key_128(ctx->key_enc, in_key); 111 ppc_expand_key_128(ctx->key_twk, in_key + AES_KEYSIZE_128); 112 break; 113 case AES_KEYSIZE_192: 114 ctx->rounds = 5; 115 ppc_expand_key_192(ctx->key_enc, in_key); 116 ppc_expand_key_192(ctx->key_twk, in_key + AES_KEYSIZE_192); 117 break; 118 case AES_KEYSIZE_256: 119 ctx->rounds = 6; 120 ppc_expand_key_256(ctx->key_enc, in_key); 121 ppc_expand_key_256(ctx->key_twk, in_key + AES_KEYSIZE_256); 122 break; 123 default: 124 return -EINVAL; 125 } 126 127 ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len); 128 129 return 0; 130 } 131 132 static int ppc_ecb_crypt(struct skcipher_request *req, bool enc) 133 { 134 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 135 struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 136 struct skcipher_walk walk; 137 unsigned int nbytes; 138 int err; 139 140 err = skcipher_walk_virt(&walk, req, false); 141 142 while ((nbytes = walk.nbytes) != 0) { 143 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 144 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 145 146 spe_begin(); 147 if (enc) 148 ppc_encrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr, 149 ctx->key_enc, ctx->rounds, nbytes); 150 else 151 ppc_decrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr, 152 ctx->key_dec, ctx->rounds, nbytes); 153 spe_end(); 154 155 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 156 } 157 158 return err; 159 } 160 161 static int ppc_ecb_encrypt(struct skcipher_request *req) 162 { 163 return ppc_ecb_crypt(req, true); 164 } 165 166 static int ppc_ecb_decrypt(struct skcipher_request *req) 167 { 168 return ppc_ecb_crypt(req, false); 169 } 170 171 static int ppc_cbc_crypt(struct skcipher_request *req, bool enc) 172 { 173 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 174 struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 175 struct skcipher_walk walk; 176 unsigned int nbytes; 177 int err; 178 179 err = skcipher_walk_virt(&walk, req, false); 180 181 while ((nbytes = walk.nbytes) != 0) { 182 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 183 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 184 185 spe_begin(); 186 if (enc) 187 ppc_encrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr, 188 ctx->key_enc, ctx->rounds, nbytes, 189 walk.iv); 190 else 191 ppc_decrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr, 192 ctx->key_dec, ctx->rounds, nbytes, 193 walk.iv); 194 spe_end(); 195 196 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 197 } 198 199 return err; 200 } 201 202 static int ppc_cbc_encrypt(struct skcipher_request *req) 203 { 204 return ppc_cbc_crypt(req, true); 205 } 206 207 static int ppc_cbc_decrypt(struct skcipher_request *req) 208 { 209 return ppc_cbc_crypt(req, false); 210 } 211 212 static int ppc_ctr_crypt(struct skcipher_request *req) 213 { 214 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 215 struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 216 struct skcipher_walk walk; 217 unsigned int nbytes; 218 int err; 219 220 err = skcipher_walk_virt(&walk, req, false); 221 222 while ((nbytes = walk.nbytes) != 0) { 223 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 224 if (nbytes < walk.total) 225 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 226 227 spe_begin(); 228 ppc_crypt_ctr(walk.dst.virt.addr, walk.src.virt.addr, 229 ctx->key_enc, ctx->rounds, nbytes, walk.iv); 230 spe_end(); 231 232 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 233 } 234 235 return err; 236 } 237 238 static int ppc_xts_crypt(struct skcipher_request *req, bool enc) 239 { 240 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 241 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 242 struct skcipher_walk walk; 243 unsigned int nbytes; 244 int err; 245 u32 *twk; 246 247 err = skcipher_walk_virt(&walk, req, false); 248 twk = ctx->key_twk; 249 250 while ((nbytes = walk.nbytes) != 0) { 251 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 252 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 253 254 spe_begin(); 255 if (enc) 256 ppc_encrypt_xts(walk.dst.virt.addr, walk.src.virt.addr, 257 ctx->key_enc, ctx->rounds, nbytes, 258 walk.iv, twk); 259 else 260 ppc_decrypt_xts(walk.dst.virt.addr, walk.src.virt.addr, 261 ctx->key_dec, ctx->rounds, nbytes, 262 walk.iv, twk); 263 spe_end(); 264 265 twk = NULL; 266 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 267 } 268 269 return err; 270 } 271 272 static int ppc_xts_encrypt(struct skcipher_request *req) 273 { 274 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 275 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 276 int tail = req->cryptlen % AES_BLOCK_SIZE; 277 int offset = req->cryptlen - tail - AES_BLOCK_SIZE; 278 struct skcipher_request subreq; 279 u8 b[2][AES_BLOCK_SIZE]; 280 int err; 281 282 if (req->cryptlen < AES_BLOCK_SIZE) 283 return -EINVAL; 284 285 if (tail) { 286 subreq = *req; 287 skcipher_request_set_crypt(&subreq, req->src, req->dst, 288 req->cryptlen - tail, req->iv); 289 req = &subreq; 290 } 291 292 err = ppc_xts_crypt(req, true); 293 if (err || !tail) 294 return err; 295 296 scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE, 0); 297 memcpy(b[1], b[0], tail); 298 scatterwalk_map_and_copy(b[0], req->src, offset + AES_BLOCK_SIZE, tail, 0); 299 300 spe_begin(); 301 ppc_encrypt_xts(b[0], b[0], ctx->key_enc, ctx->rounds, AES_BLOCK_SIZE, 302 req->iv, NULL); 303 spe_end(); 304 305 scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1); 306 307 return 0; 308 } 309 310 static int ppc_xts_decrypt(struct skcipher_request *req) 311 { 312 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 313 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 314 int tail = req->cryptlen % AES_BLOCK_SIZE; 315 int offset = req->cryptlen - tail - AES_BLOCK_SIZE; 316 struct skcipher_request subreq; 317 u8 b[3][AES_BLOCK_SIZE]; 318 le128 twk; 319 int err; 320 321 if (req->cryptlen < AES_BLOCK_SIZE) 322 return -EINVAL; 323 324 if (tail) { 325 subreq = *req; 326 skcipher_request_set_crypt(&subreq, req->src, req->dst, 327 offset, req->iv); 328 req = &subreq; 329 } 330 331 err = ppc_xts_crypt(req, false); 332 if (err || !tail) 333 return err; 334 335 scatterwalk_map_and_copy(b[1], req->src, offset, AES_BLOCK_SIZE + tail, 0); 336 337 spe_begin(); 338 if (!offset) 339 ppc_encrypt_ecb(req->iv, req->iv, ctx->key_twk, ctx->rounds, 340 AES_BLOCK_SIZE); 341 342 gf128mul_x_ble(&twk, (le128 *)req->iv); 343 344 ppc_decrypt_xts(b[1], b[1], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE, 345 (u8 *)&twk, NULL); 346 memcpy(b[0], b[2], tail); 347 memcpy(b[0] + tail, b[1] + tail, AES_BLOCK_SIZE - tail); 348 ppc_decrypt_xts(b[0], b[0], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE, 349 req->iv, NULL); 350 spe_end(); 351 352 scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1); 353 354 return 0; 355 } 356 357 /* 358 * Algorithm definitions. Disabling alignment (cra_alignmask=0) was chosen 359 * because the e500 platform can handle unaligned reads/writes very efficiently. 360 * This improves IPsec thoughput by another few percent. Additionally we assume 361 * that AES context is always aligned to at least 8 bytes because it is created 362 * with kmalloc() in the crypto infrastructure 363 */ 364 365 static struct skcipher_alg aes_skcipher_algs[] = { 366 { 367 .base.cra_name = "ecb(aes)", 368 .base.cra_driver_name = "ecb-ppc-spe", 369 .base.cra_priority = 300, 370 .base.cra_blocksize = AES_BLOCK_SIZE, 371 .base.cra_ctxsize = sizeof(struct ppc_aes_ctx), 372 .base.cra_module = THIS_MODULE, 373 .min_keysize = AES_MIN_KEY_SIZE, 374 .max_keysize = AES_MAX_KEY_SIZE, 375 .setkey = ppc_aes_setkey_skcipher, 376 .encrypt = ppc_ecb_encrypt, 377 .decrypt = ppc_ecb_decrypt, 378 }, { 379 .base.cra_name = "cbc(aes)", 380 .base.cra_driver_name = "cbc-ppc-spe", 381 .base.cra_priority = 300, 382 .base.cra_blocksize = AES_BLOCK_SIZE, 383 .base.cra_ctxsize = sizeof(struct ppc_aes_ctx), 384 .base.cra_module = THIS_MODULE, 385 .min_keysize = AES_MIN_KEY_SIZE, 386 .max_keysize = AES_MAX_KEY_SIZE, 387 .ivsize = AES_BLOCK_SIZE, 388 .setkey = ppc_aes_setkey_skcipher, 389 .encrypt = ppc_cbc_encrypt, 390 .decrypt = ppc_cbc_decrypt, 391 }, { 392 .base.cra_name = "ctr(aes)", 393 .base.cra_driver_name = "ctr-ppc-spe", 394 .base.cra_priority = 300, 395 .base.cra_blocksize = 1, 396 .base.cra_ctxsize = sizeof(struct ppc_aes_ctx), 397 .base.cra_module = THIS_MODULE, 398 .min_keysize = AES_MIN_KEY_SIZE, 399 .max_keysize = AES_MAX_KEY_SIZE, 400 .ivsize = AES_BLOCK_SIZE, 401 .setkey = ppc_aes_setkey_skcipher, 402 .encrypt = ppc_ctr_crypt, 403 .decrypt = ppc_ctr_crypt, 404 .chunksize = AES_BLOCK_SIZE, 405 }, { 406 .base.cra_name = "xts(aes)", 407 .base.cra_driver_name = "xts-ppc-spe", 408 .base.cra_priority = 300, 409 .base.cra_blocksize = AES_BLOCK_SIZE, 410 .base.cra_ctxsize = sizeof(struct ppc_xts_ctx), 411 .base.cra_module = THIS_MODULE, 412 .min_keysize = AES_MIN_KEY_SIZE * 2, 413 .max_keysize = AES_MAX_KEY_SIZE * 2, 414 .ivsize = AES_BLOCK_SIZE, 415 .setkey = ppc_xts_setkey, 416 .encrypt = ppc_xts_encrypt, 417 .decrypt = ppc_xts_decrypt, 418 } 419 }; 420 421 static int __init ppc_aes_mod_init(void) 422 { 423 return crypto_register_skciphers(aes_skcipher_algs, 424 ARRAY_SIZE(aes_skcipher_algs)); 425 } 426 427 static void __exit ppc_aes_mod_fini(void) 428 { 429 crypto_unregister_skciphers(aes_skcipher_algs, 430 ARRAY_SIZE(aes_skcipher_algs)); 431 } 432 433 module_init(ppc_aes_mod_init); 434 module_exit(ppc_aes_mod_fini); 435 436 MODULE_LICENSE("GPL"); 437 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS, SPE optimized"); 438 439 MODULE_ALIAS_CRYPTO("aes"); 440 MODULE_ALIAS_CRYPTO("ecb(aes)"); 441 MODULE_ALIAS_CRYPTO("cbc(aes)"); 442 MODULE_ALIAS_CRYPTO("ctr(aes)"); 443 MODULE_ALIAS_CRYPTO("xts(aes)"); 444 MODULE_ALIAS_CRYPTO("aes-ppc-spe"); 445