12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
23265c4baSKim Phillips /*
33265c4baSKim Phillips * Glue code for AES implementation for SPE instructions (PPC)
43265c4baSKim Phillips *
53265c4baSKim Phillips * Based on generic implementation. The assembler module takes care
63265c4baSKim Phillips * about the SPE registers so it can run from interrupt context.
73265c4baSKim Phillips *
83265c4baSKim Phillips * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
93265c4baSKim Phillips */
103265c4baSKim Phillips
113265c4baSKim Phillips #include <crypto/aes.h>
123265c4baSKim Phillips #include <linux/module.h>
133265c4baSKim Phillips #include <linux/init.h>
143265c4baSKim Phillips #include <linux/types.h>
153265c4baSKim Phillips #include <linux/errno.h>
163265c4baSKim Phillips #include <linux/crypto.h>
173265c4baSKim Phillips #include <asm/byteorder.h>
183265c4baSKim Phillips #include <asm/switch_to.h>
193265c4baSKim Phillips #include <crypto/algapi.h>
207f725f41SEric Biggers #include <crypto/internal/skcipher.h>
2149abc0d2SStephan Mueller #include <crypto/xts.h>
22d0be0720SArd Biesheuvel #include <crypto/gf128mul.h>
23d0be0720SArd Biesheuvel #include <crypto/scatterwalk.h>
243265c4baSKim Phillips
253265c4baSKim Phillips /*
263265c4baSKim Phillips * MAX_BYTES defines the number of bytes that are allowed to be processed
273265c4baSKim Phillips * between preempt_disable() and preempt_enable(). e500 cores can issue two
283265c4baSKim Phillips * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32
293265c4baSKim Phillips * bit unit (SU2). One of these can be a memory access that is executed via
303265c4baSKim Phillips * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per
31*647c952eSshaom Deng * 16 byte block or 25 cycles per byte. Thus 768 bytes of input data
323265c4baSKim Phillips * will need an estimated maximum of 20,000 cycles. Headroom for cache misses
333265c4baSKim Phillips * included. Even with the low end model clocked at 667 MHz this equals to a
34446957baSAdam Buchbinder * critical time window of less than 30us. The value has been chosen to
353265c4baSKim Phillips * process a 512 byte disk block in one or a large 1400 bytes IPsec network
363265c4baSKim Phillips * packet in two runs.
373265c4baSKim Phillips *
383265c4baSKim Phillips */
393265c4baSKim Phillips #define MAX_BYTES 768
403265c4baSKim Phillips
413265c4baSKim Phillips struct ppc_aes_ctx {
423265c4baSKim Phillips u32 key_enc[AES_MAX_KEYLENGTH_U32];
433265c4baSKim Phillips u32 key_dec[AES_MAX_KEYLENGTH_U32];
443265c4baSKim Phillips u32 rounds;
453265c4baSKim Phillips };
463265c4baSKim Phillips
473265c4baSKim Phillips struct ppc_xts_ctx {
483265c4baSKim Phillips u32 key_enc[AES_MAX_KEYLENGTH_U32];
493265c4baSKim Phillips u32 key_dec[AES_MAX_KEYLENGTH_U32];
503265c4baSKim Phillips u32 key_twk[AES_MAX_KEYLENGTH_U32];
513265c4baSKim Phillips u32 rounds;
523265c4baSKim Phillips };
533265c4baSKim Phillips
543265c4baSKim Phillips extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
553265c4baSKim Phillips extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
563265c4baSKim Phillips extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
573265c4baSKim Phillips u32 bytes);
583265c4baSKim Phillips extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
593265c4baSKim Phillips u32 bytes);
603265c4baSKim Phillips extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
613265c4baSKim Phillips u32 bytes, u8 *iv);
623265c4baSKim Phillips extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
633265c4baSKim Phillips u32 bytes, u8 *iv);
643265c4baSKim Phillips extern void ppc_crypt_ctr (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
653265c4baSKim Phillips u32 bytes, u8 *iv);
663265c4baSKim Phillips extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
673265c4baSKim Phillips u32 bytes, u8 *iv, u32 *key_twk);
683265c4baSKim Phillips extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
693265c4baSKim Phillips u32 bytes, u8 *iv, u32 *key_twk);
703265c4baSKim Phillips
713265c4baSKim Phillips extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
723265c4baSKim Phillips extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
733265c4baSKim Phillips extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
743265c4baSKim Phillips
753265c4baSKim Phillips extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
763265c4baSKim Phillips unsigned int key_len);
773265c4baSKim Phillips
spe_begin(void)783265c4baSKim Phillips static void spe_begin(void)
793265c4baSKim Phillips {
803265c4baSKim Phillips /* disable preemption and save users SPE registers if required */
813265c4baSKim Phillips preempt_disable();
823265c4baSKim Phillips enable_kernel_spe();
833265c4baSKim Phillips }
843265c4baSKim Phillips
spe_end(void)853265c4baSKim Phillips static void spe_end(void)
863265c4baSKim Phillips {
87dc4fbba1SAnton Blanchard disable_kernel_spe();
883265c4baSKim Phillips /* reenable preemption */
893265c4baSKim Phillips preempt_enable();
903265c4baSKim Phillips }
913265c4baSKim Phillips
ppc_aes_setkey(struct crypto_tfm * tfm,const u8 * in_key,unsigned int key_len)923265c4baSKim Phillips static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
933265c4baSKim Phillips unsigned int key_len)
943265c4baSKim Phillips {
953265c4baSKim Phillips struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
963265c4baSKim Phillips
973265c4baSKim Phillips switch (key_len) {
983265c4baSKim Phillips case AES_KEYSIZE_128:
993265c4baSKim Phillips ctx->rounds = 4;
1003265c4baSKim Phillips ppc_expand_key_128(ctx->key_enc, in_key);
1013265c4baSKim Phillips break;
1023265c4baSKim Phillips case AES_KEYSIZE_192:
1033265c4baSKim Phillips ctx->rounds = 5;
1043265c4baSKim Phillips ppc_expand_key_192(ctx->key_enc, in_key);
1053265c4baSKim Phillips break;
1063265c4baSKim Phillips case AES_KEYSIZE_256:
1073265c4baSKim Phillips ctx->rounds = 6;
1083265c4baSKim Phillips ppc_expand_key_256(ctx->key_enc, in_key);
1093265c4baSKim Phillips break;
110674f368aSEric Biggers default:
111674f368aSEric Biggers return -EINVAL;
1123265c4baSKim Phillips }
1133265c4baSKim Phillips
1143265c4baSKim Phillips ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
1153265c4baSKim Phillips
1163265c4baSKim Phillips return 0;
1173265c4baSKim Phillips }
1183265c4baSKim Phillips
ppc_aes_setkey_skcipher(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)1197f725f41SEric Biggers static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
1207f725f41SEric Biggers const u8 *in_key, unsigned int key_len)
1217f725f41SEric Biggers {
1227f725f41SEric Biggers return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
1237f725f41SEric Biggers }
1247f725f41SEric Biggers
ppc_xts_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)1257f725f41SEric Biggers static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
1263265c4baSKim Phillips unsigned int key_len)
1273265c4baSKim Phillips {
1287f725f41SEric Biggers struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
12928856a9eSStephan Mueller int err;
13028856a9eSStephan Mueller
1317f725f41SEric Biggers err = xts_verify_key(tfm, in_key, key_len);
13228856a9eSStephan Mueller if (err)
13328856a9eSStephan Mueller return err;
1343265c4baSKim Phillips
1353265c4baSKim Phillips key_len >>= 1;
1363265c4baSKim Phillips
1373265c4baSKim Phillips switch (key_len) {
1383265c4baSKim Phillips case AES_KEYSIZE_128:
1393265c4baSKim Phillips ctx->rounds = 4;
1403265c4baSKim Phillips ppc_expand_key_128(ctx->key_enc, in_key);
1413265c4baSKim Phillips ppc_expand_key_128(ctx->key_twk, in_key + AES_KEYSIZE_128);
1423265c4baSKim Phillips break;
1433265c4baSKim Phillips case AES_KEYSIZE_192:
1443265c4baSKim Phillips ctx->rounds = 5;
1453265c4baSKim Phillips ppc_expand_key_192(ctx->key_enc, in_key);
1463265c4baSKim Phillips ppc_expand_key_192(ctx->key_twk, in_key + AES_KEYSIZE_192);
1473265c4baSKim Phillips break;
1483265c4baSKim Phillips case AES_KEYSIZE_256:
1493265c4baSKim Phillips ctx->rounds = 6;
1503265c4baSKim Phillips ppc_expand_key_256(ctx->key_enc, in_key);
1513265c4baSKim Phillips ppc_expand_key_256(ctx->key_twk, in_key + AES_KEYSIZE_256);
1523265c4baSKim Phillips break;
153674f368aSEric Biggers default:
154674f368aSEric Biggers return -EINVAL;
1553265c4baSKim Phillips }
1563265c4baSKim Phillips
1573265c4baSKim Phillips ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
1583265c4baSKim Phillips
1593265c4baSKim Phillips return 0;
1603265c4baSKim Phillips }
1613265c4baSKim Phillips
ppc_aes_encrypt(struct crypto_tfm * tfm,u8 * out,const u8 * in)1623265c4baSKim Phillips static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
1633265c4baSKim Phillips {
1643265c4baSKim Phillips struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
1653265c4baSKim Phillips
1663265c4baSKim Phillips spe_begin();
1673265c4baSKim Phillips ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
1683265c4baSKim Phillips spe_end();
1693265c4baSKim Phillips }
1703265c4baSKim Phillips
ppc_aes_decrypt(struct crypto_tfm * tfm,u8 * out,const u8 * in)1713265c4baSKim Phillips static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
1723265c4baSKim Phillips {
1733265c4baSKim Phillips struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
1743265c4baSKim Phillips
1753265c4baSKim Phillips spe_begin();
1763265c4baSKim Phillips ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
1773265c4baSKim Phillips spe_end();
1783265c4baSKim Phillips }
1793265c4baSKim Phillips
ppc_ecb_crypt(struct skcipher_request * req,bool enc)1807f725f41SEric Biggers static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
1813265c4baSKim Phillips {
1827f725f41SEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1837f725f41SEric Biggers struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
1847f725f41SEric Biggers struct skcipher_walk walk;
1857f725f41SEric Biggers unsigned int nbytes;
1863265c4baSKim Phillips int err;
1873265c4baSKim Phillips
1887f725f41SEric Biggers err = skcipher_walk_virt(&walk, req, false);
1893265c4baSKim Phillips
1907f725f41SEric Biggers while ((nbytes = walk.nbytes) != 0) {
1917f725f41SEric Biggers nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
1927f725f41SEric Biggers nbytes = round_down(nbytes, AES_BLOCK_SIZE);
1933265c4baSKim Phillips
1943265c4baSKim Phillips spe_begin();
1957f725f41SEric Biggers if (enc)
1963265c4baSKim Phillips ppc_encrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
1973265c4baSKim Phillips ctx->key_enc, ctx->rounds, nbytes);
1987f725f41SEric Biggers else
1993265c4baSKim Phillips ppc_decrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
2003265c4baSKim Phillips ctx->key_dec, ctx->rounds, nbytes);
2013265c4baSKim Phillips spe_end();
2023265c4baSKim Phillips
2037f725f41SEric Biggers err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
2043265c4baSKim Phillips }
2053265c4baSKim Phillips
2063265c4baSKim Phillips return err;
2073265c4baSKim Phillips }
2083265c4baSKim Phillips
ppc_ecb_encrypt(struct skcipher_request * req)2097f725f41SEric Biggers static int ppc_ecb_encrypt(struct skcipher_request *req)
2103265c4baSKim Phillips {
2117f725f41SEric Biggers return ppc_ecb_crypt(req, true);
2127f725f41SEric Biggers }
2137f725f41SEric Biggers
ppc_ecb_decrypt(struct skcipher_request * req)2147f725f41SEric Biggers static int ppc_ecb_decrypt(struct skcipher_request *req)
2157f725f41SEric Biggers {
2167f725f41SEric Biggers return ppc_ecb_crypt(req, false);
2177f725f41SEric Biggers }
2187f725f41SEric Biggers
ppc_cbc_crypt(struct skcipher_request * req,bool enc)2197f725f41SEric Biggers static int ppc_cbc_crypt(struct skcipher_request *req, bool enc)
2207f725f41SEric Biggers {
2217f725f41SEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2227f725f41SEric Biggers struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
2237f725f41SEric Biggers struct skcipher_walk walk;
2247f725f41SEric Biggers unsigned int nbytes;
2253265c4baSKim Phillips int err;
2263265c4baSKim Phillips
2277f725f41SEric Biggers err = skcipher_walk_virt(&walk, req, false);
2283265c4baSKim Phillips
2297f725f41SEric Biggers while ((nbytes = walk.nbytes) != 0) {
2307f725f41SEric Biggers nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
2317f725f41SEric Biggers nbytes = round_down(nbytes, AES_BLOCK_SIZE);
2323265c4baSKim Phillips
2333265c4baSKim Phillips spe_begin();
2347f725f41SEric Biggers if (enc)
2353265c4baSKim Phillips ppc_encrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
2367f725f41SEric Biggers ctx->key_enc, ctx->rounds, nbytes,
2377f725f41SEric Biggers walk.iv);
2387f725f41SEric Biggers else
2393265c4baSKim Phillips ppc_decrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
2407f725f41SEric Biggers ctx->key_dec, ctx->rounds, nbytes,
2417f725f41SEric Biggers walk.iv);
2423265c4baSKim Phillips spe_end();
2433265c4baSKim Phillips
2447f725f41SEric Biggers err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
2453265c4baSKim Phillips }
2463265c4baSKim Phillips
2473265c4baSKim Phillips return err;
2483265c4baSKim Phillips }
2493265c4baSKim Phillips
ppc_cbc_encrypt(struct skcipher_request * req)2507f725f41SEric Biggers static int ppc_cbc_encrypt(struct skcipher_request *req)
2513265c4baSKim Phillips {
2527f725f41SEric Biggers return ppc_cbc_crypt(req, true);
2537f725f41SEric Biggers }
2547f725f41SEric Biggers
ppc_cbc_decrypt(struct skcipher_request * req)2557f725f41SEric Biggers static int ppc_cbc_decrypt(struct skcipher_request *req)
2567f725f41SEric Biggers {
2577f725f41SEric Biggers return ppc_cbc_crypt(req, false);
2587f725f41SEric Biggers }
2597f725f41SEric Biggers
ppc_ctr_crypt(struct skcipher_request * req)2607f725f41SEric Biggers static int ppc_ctr_crypt(struct skcipher_request *req)
2617f725f41SEric Biggers {
2627f725f41SEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2637f725f41SEric Biggers struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
2647f725f41SEric Biggers struct skcipher_walk walk;
2657f725f41SEric Biggers unsigned int nbytes;
2663265c4baSKim Phillips int err;
2673265c4baSKim Phillips
2687f725f41SEric Biggers err = skcipher_walk_virt(&walk, req, false);
2693265c4baSKim Phillips
2707f725f41SEric Biggers while ((nbytes = walk.nbytes) != 0) {
2717f725f41SEric Biggers nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
2727f725f41SEric Biggers if (nbytes < walk.total)
2737f725f41SEric Biggers nbytes = round_down(nbytes, AES_BLOCK_SIZE);
2743265c4baSKim Phillips
2753265c4baSKim Phillips spe_begin();
2763265c4baSKim Phillips ppc_crypt_ctr(walk.dst.virt.addr, walk.src.virt.addr,
2777f725f41SEric Biggers ctx->key_enc, ctx->rounds, nbytes, walk.iv);
2783265c4baSKim Phillips spe_end();
2793265c4baSKim Phillips
2807f725f41SEric Biggers err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
2813265c4baSKim Phillips }
2823265c4baSKim Phillips
2833265c4baSKim Phillips return err;
2843265c4baSKim Phillips }
2853265c4baSKim Phillips
ppc_xts_crypt(struct skcipher_request * req,bool enc)2867f725f41SEric Biggers static int ppc_xts_crypt(struct skcipher_request *req, bool enc)
2873265c4baSKim Phillips {
2887f725f41SEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2897f725f41SEric Biggers struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
2907f725f41SEric Biggers struct skcipher_walk walk;
2917f725f41SEric Biggers unsigned int nbytes;
2923265c4baSKim Phillips int err;
2933265c4baSKim Phillips u32 *twk;
2943265c4baSKim Phillips
2957f725f41SEric Biggers err = skcipher_walk_virt(&walk, req, false);
2963265c4baSKim Phillips twk = ctx->key_twk;
2973265c4baSKim Phillips
2987f725f41SEric Biggers while ((nbytes = walk.nbytes) != 0) {
2997f725f41SEric Biggers nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
3007f725f41SEric Biggers nbytes = round_down(nbytes, AES_BLOCK_SIZE);
3013265c4baSKim Phillips
3023265c4baSKim Phillips spe_begin();
3037f725f41SEric Biggers if (enc)
3043265c4baSKim Phillips ppc_encrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
3057f725f41SEric Biggers ctx->key_enc, ctx->rounds, nbytes,
3067f725f41SEric Biggers walk.iv, twk);
3077f725f41SEric Biggers else
3083265c4baSKim Phillips ppc_decrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
3097f725f41SEric Biggers ctx->key_dec, ctx->rounds, nbytes,
3107f725f41SEric Biggers walk.iv, twk);
3113265c4baSKim Phillips spe_end();
3123265c4baSKim Phillips
3133265c4baSKim Phillips twk = NULL;
3147f725f41SEric Biggers err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
3153265c4baSKim Phillips }
3163265c4baSKim Phillips
3173265c4baSKim Phillips return err;
3183265c4baSKim Phillips }
3193265c4baSKim Phillips
ppc_xts_encrypt(struct skcipher_request * req)3207f725f41SEric Biggers static int ppc_xts_encrypt(struct skcipher_request *req)
3217f725f41SEric Biggers {
322d0be0720SArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
323d0be0720SArd Biesheuvel struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
324d0be0720SArd Biesheuvel int tail = req->cryptlen % AES_BLOCK_SIZE;
325d0be0720SArd Biesheuvel int offset = req->cryptlen - tail - AES_BLOCK_SIZE;
326d0be0720SArd Biesheuvel struct skcipher_request subreq;
327d0be0720SArd Biesheuvel u8 b[2][AES_BLOCK_SIZE];
328d0be0720SArd Biesheuvel int err;
329d0be0720SArd Biesheuvel
330d0be0720SArd Biesheuvel if (req->cryptlen < AES_BLOCK_SIZE)
331d0be0720SArd Biesheuvel return -EINVAL;
332d0be0720SArd Biesheuvel
333d0be0720SArd Biesheuvel if (tail) {
334d0be0720SArd Biesheuvel subreq = *req;
335d0be0720SArd Biesheuvel skcipher_request_set_crypt(&subreq, req->src, req->dst,
336d0be0720SArd Biesheuvel req->cryptlen - tail, req->iv);
337d0be0720SArd Biesheuvel req = &subreq;
338d0be0720SArd Biesheuvel }
339d0be0720SArd Biesheuvel
340d0be0720SArd Biesheuvel err = ppc_xts_crypt(req, true);
341d0be0720SArd Biesheuvel if (err || !tail)
342d0be0720SArd Biesheuvel return err;
343d0be0720SArd Biesheuvel
344d0be0720SArd Biesheuvel scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE, 0);
345d0be0720SArd Biesheuvel memcpy(b[1], b[0], tail);
346d0be0720SArd Biesheuvel scatterwalk_map_and_copy(b[0], req->src, offset + AES_BLOCK_SIZE, tail, 0);
347d0be0720SArd Biesheuvel
348d0be0720SArd Biesheuvel spe_begin();
349d0be0720SArd Biesheuvel ppc_encrypt_xts(b[0], b[0], ctx->key_enc, ctx->rounds, AES_BLOCK_SIZE,
350d0be0720SArd Biesheuvel req->iv, NULL);
351d0be0720SArd Biesheuvel spe_end();
352d0be0720SArd Biesheuvel
353d0be0720SArd Biesheuvel scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1);
354d0be0720SArd Biesheuvel
355d0be0720SArd Biesheuvel return 0;
3567f725f41SEric Biggers }
3577f725f41SEric Biggers
ppc_xts_decrypt(struct skcipher_request * req)3587f725f41SEric Biggers static int ppc_xts_decrypt(struct skcipher_request *req)
3597f725f41SEric Biggers {
360d0be0720SArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
361d0be0720SArd Biesheuvel struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
362d0be0720SArd Biesheuvel int tail = req->cryptlen % AES_BLOCK_SIZE;
363d0be0720SArd Biesheuvel int offset = req->cryptlen - tail - AES_BLOCK_SIZE;
364d0be0720SArd Biesheuvel struct skcipher_request subreq;
365d0be0720SArd Biesheuvel u8 b[3][AES_BLOCK_SIZE];
366d0be0720SArd Biesheuvel le128 twk;
367d0be0720SArd Biesheuvel int err;
368d0be0720SArd Biesheuvel
369d0be0720SArd Biesheuvel if (req->cryptlen < AES_BLOCK_SIZE)
370d0be0720SArd Biesheuvel return -EINVAL;
371d0be0720SArd Biesheuvel
372d0be0720SArd Biesheuvel if (tail) {
373d0be0720SArd Biesheuvel subreq = *req;
374d0be0720SArd Biesheuvel skcipher_request_set_crypt(&subreq, req->src, req->dst,
375d0be0720SArd Biesheuvel offset, req->iv);
376d0be0720SArd Biesheuvel req = &subreq;
377d0be0720SArd Biesheuvel }
378d0be0720SArd Biesheuvel
379d0be0720SArd Biesheuvel err = ppc_xts_crypt(req, false);
380d0be0720SArd Biesheuvel if (err || !tail)
381d0be0720SArd Biesheuvel return err;
382d0be0720SArd Biesheuvel
383d0be0720SArd Biesheuvel scatterwalk_map_and_copy(b[1], req->src, offset, AES_BLOCK_SIZE + tail, 0);
384d0be0720SArd Biesheuvel
385d0be0720SArd Biesheuvel spe_begin();
386d0be0720SArd Biesheuvel if (!offset)
387d0be0720SArd Biesheuvel ppc_encrypt_ecb(req->iv, req->iv, ctx->key_twk, ctx->rounds,
388d0be0720SArd Biesheuvel AES_BLOCK_SIZE);
389d0be0720SArd Biesheuvel
390d0be0720SArd Biesheuvel gf128mul_x_ble(&twk, (le128 *)req->iv);
391d0be0720SArd Biesheuvel
392d0be0720SArd Biesheuvel ppc_decrypt_xts(b[1], b[1], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE,
393d0be0720SArd Biesheuvel (u8 *)&twk, NULL);
394d0be0720SArd Biesheuvel memcpy(b[0], b[2], tail);
395d0be0720SArd Biesheuvel memcpy(b[0] + tail, b[1] + tail, AES_BLOCK_SIZE - tail);
396d0be0720SArd Biesheuvel ppc_decrypt_xts(b[0], b[0], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE,
397d0be0720SArd Biesheuvel req->iv, NULL);
398d0be0720SArd Biesheuvel spe_end();
399d0be0720SArd Biesheuvel
400d0be0720SArd Biesheuvel scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1);
401d0be0720SArd Biesheuvel
402d0be0720SArd Biesheuvel return 0;
4037f725f41SEric Biggers }
4047f725f41SEric Biggers
4053265c4baSKim Phillips /*
4063265c4baSKim Phillips * Algorithm definitions. Disabling alignment (cra_alignmask=0) was chosen
4071fd02f66SJulia Lawall * because the e500 platform can handle unaligned reads/writes very efficiently.
4083265c4baSKim Phillips * This improves IPsec thoughput by another few percent. Additionally we assume
4093265c4baSKim Phillips * that AES context is always aligned to at least 8 bytes because it is created
4103265c4baSKim Phillips * with kmalloc() in the crypto infrastructure
4113265c4baSKim Phillips */
4127f725f41SEric Biggers
4137f725f41SEric Biggers static struct crypto_alg aes_cipher_alg = {
4143265c4baSKim Phillips .cra_name = "aes",
4153265c4baSKim Phillips .cra_driver_name = "aes-ppc-spe",
4163265c4baSKim Phillips .cra_priority = 300,
4173265c4baSKim Phillips .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
4183265c4baSKim Phillips .cra_blocksize = AES_BLOCK_SIZE,
4193265c4baSKim Phillips .cra_ctxsize = sizeof(struct ppc_aes_ctx),
4203265c4baSKim Phillips .cra_alignmask = 0,
4213265c4baSKim Phillips .cra_module = THIS_MODULE,
4223265c4baSKim Phillips .cra_u = {
4233265c4baSKim Phillips .cipher = {
4243265c4baSKim Phillips .cia_min_keysize = AES_MIN_KEY_SIZE,
4253265c4baSKim Phillips .cia_max_keysize = AES_MAX_KEY_SIZE,
4263265c4baSKim Phillips .cia_setkey = ppc_aes_setkey,
4273265c4baSKim Phillips .cia_encrypt = ppc_aes_encrypt,
4283265c4baSKim Phillips .cia_decrypt = ppc_aes_decrypt
4293265c4baSKim Phillips }
4303265c4baSKim Phillips }
4317f725f41SEric Biggers };
4327f725f41SEric Biggers
4337f725f41SEric Biggers static struct skcipher_alg aes_skcipher_algs[] = {
4347f725f41SEric Biggers {
4357f725f41SEric Biggers .base.cra_name = "ecb(aes)",
4367f725f41SEric Biggers .base.cra_driver_name = "ecb-ppc-spe",
4377f725f41SEric Biggers .base.cra_priority = 300,
4387f725f41SEric Biggers .base.cra_blocksize = AES_BLOCK_SIZE,
4397f725f41SEric Biggers .base.cra_ctxsize = sizeof(struct ppc_aes_ctx),
4407f725f41SEric Biggers .base.cra_module = THIS_MODULE,
4413265c4baSKim Phillips .min_keysize = AES_MIN_KEY_SIZE,
4423265c4baSKim Phillips .max_keysize = AES_MAX_KEY_SIZE,
4437f725f41SEric Biggers .setkey = ppc_aes_setkey_skcipher,
4443265c4baSKim Phillips .encrypt = ppc_ecb_encrypt,
4453265c4baSKim Phillips .decrypt = ppc_ecb_decrypt,
4463265c4baSKim Phillips }, {
4477f725f41SEric Biggers .base.cra_name = "cbc(aes)",
4487f725f41SEric Biggers .base.cra_driver_name = "cbc-ppc-spe",
4497f725f41SEric Biggers .base.cra_priority = 300,
4507f725f41SEric Biggers .base.cra_blocksize = AES_BLOCK_SIZE,
4517f725f41SEric Biggers .base.cra_ctxsize = sizeof(struct ppc_aes_ctx),
4527f725f41SEric Biggers .base.cra_module = THIS_MODULE,
4533265c4baSKim Phillips .min_keysize = AES_MIN_KEY_SIZE,
4543265c4baSKim Phillips .max_keysize = AES_MAX_KEY_SIZE,
4553265c4baSKim Phillips .ivsize = AES_BLOCK_SIZE,
4567f725f41SEric Biggers .setkey = ppc_aes_setkey_skcipher,
4573265c4baSKim Phillips .encrypt = ppc_cbc_encrypt,
4583265c4baSKim Phillips .decrypt = ppc_cbc_decrypt,
4593265c4baSKim Phillips }, {
4607f725f41SEric Biggers .base.cra_name = "ctr(aes)",
4617f725f41SEric Biggers .base.cra_driver_name = "ctr-ppc-spe",
4627f725f41SEric Biggers .base.cra_priority = 300,
4637f725f41SEric Biggers .base.cra_blocksize = 1,
4647f725f41SEric Biggers .base.cra_ctxsize = sizeof(struct ppc_aes_ctx),
4657f725f41SEric Biggers .base.cra_module = THIS_MODULE,
4663265c4baSKim Phillips .min_keysize = AES_MIN_KEY_SIZE,
4673265c4baSKim Phillips .max_keysize = AES_MAX_KEY_SIZE,
4683265c4baSKim Phillips .ivsize = AES_BLOCK_SIZE,
4697f725f41SEric Biggers .setkey = ppc_aes_setkey_skcipher,
4703265c4baSKim Phillips .encrypt = ppc_ctr_crypt,
4713265c4baSKim Phillips .decrypt = ppc_ctr_crypt,
4727f725f41SEric Biggers .chunksize = AES_BLOCK_SIZE,
4733265c4baSKim Phillips }, {
4747f725f41SEric Biggers .base.cra_name = "xts(aes)",
4757f725f41SEric Biggers .base.cra_driver_name = "xts-ppc-spe",
4767f725f41SEric Biggers .base.cra_priority = 300,
4777f725f41SEric Biggers .base.cra_blocksize = AES_BLOCK_SIZE,
4787f725f41SEric Biggers .base.cra_ctxsize = sizeof(struct ppc_xts_ctx),
4797f725f41SEric Biggers .base.cra_module = THIS_MODULE,
4803265c4baSKim Phillips .min_keysize = AES_MIN_KEY_SIZE * 2,
4813265c4baSKim Phillips .max_keysize = AES_MAX_KEY_SIZE * 2,
4823265c4baSKim Phillips .ivsize = AES_BLOCK_SIZE,
4833265c4baSKim Phillips .setkey = ppc_xts_setkey,
4843265c4baSKim Phillips .encrypt = ppc_xts_encrypt,
4853265c4baSKim Phillips .decrypt = ppc_xts_decrypt,
4863265c4baSKim Phillips }
4877f725f41SEric Biggers };
4883265c4baSKim Phillips
ppc_aes_mod_init(void)4893265c4baSKim Phillips static int __init ppc_aes_mod_init(void)
4903265c4baSKim Phillips {
4917f725f41SEric Biggers int err;
4927f725f41SEric Biggers
4937f725f41SEric Biggers err = crypto_register_alg(&aes_cipher_alg);
4947f725f41SEric Biggers if (err)
4957f725f41SEric Biggers return err;
4967f725f41SEric Biggers
4977f725f41SEric Biggers err = crypto_register_skciphers(aes_skcipher_algs,
4987f725f41SEric Biggers ARRAY_SIZE(aes_skcipher_algs));
4997f725f41SEric Biggers if (err)
5007f725f41SEric Biggers crypto_unregister_alg(&aes_cipher_alg);
5017f725f41SEric Biggers return err;
5023265c4baSKim Phillips }
5033265c4baSKim Phillips
ppc_aes_mod_fini(void)5043265c4baSKim Phillips static void __exit ppc_aes_mod_fini(void)
5053265c4baSKim Phillips {
5067f725f41SEric Biggers crypto_unregister_alg(&aes_cipher_alg);
5077f725f41SEric Biggers crypto_unregister_skciphers(aes_skcipher_algs,
5087f725f41SEric Biggers ARRAY_SIZE(aes_skcipher_algs));
5093265c4baSKim Phillips }
5103265c4baSKim Phillips
5113265c4baSKim Phillips module_init(ppc_aes_mod_init);
5123265c4baSKim Phillips module_exit(ppc_aes_mod_fini);
5133265c4baSKim Phillips
5143265c4baSKim Phillips MODULE_LICENSE("GPL");
5153265c4baSKim Phillips MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS, SPE optimized");
5163265c4baSKim Phillips
5173265c4baSKim Phillips MODULE_ALIAS_CRYPTO("aes");
5183265c4baSKim Phillips MODULE_ALIAS_CRYPTO("ecb(aes)");
5193265c4baSKim Phillips MODULE_ALIAS_CRYPTO("cbc(aes)");
5203265c4baSKim Phillips MODULE_ALIAS_CRYPTO("ctr(aes)");
5213265c4baSKim Phillips MODULE_ALIAS_CRYPTO("xts(aes)");
5223265c4baSKim Phillips MODULE_ALIAS_CRYPTO("aes-ppc-spe");
523