1*10930333SDanny Tsen // SPDX-License-Identifier: GPL-2.0-only
2*10930333SDanny Tsen /*
3*10930333SDanny Tsen * AES XTS routines supporting VMX In-core instructions on Power 8
4*10930333SDanny Tsen *
5*10930333SDanny Tsen * Copyright (C) 2015 International Business Machines Inc.
6*10930333SDanny Tsen *
7*10930333SDanny Tsen * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
8*10930333SDanny Tsen */
9*10930333SDanny Tsen
10*10930333SDanny Tsen #include <asm/simd.h>
11*10930333SDanny Tsen #include <asm/switch_to.h>
12*10930333SDanny Tsen #include <crypto/aes.h>
13*10930333SDanny Tsen #include <crypto/internal/simd.h>
14*10930333SDanny Tsen #include <crypto/internal/skcipher.h>
15*10930333SDanny Tsen #include <crypto/xts.h>
16*10930333SDanny Tsen
17*10930333SDanny Tsen #include "aesp8-ppc.h"
18*10930333SDanny Tsen
19*10930333SDanny Tsen struct p8_aes_xts_ctx {
20*10930333SDanny Tsen struct crypto_skcipher *fallback;
21*10930333SDanny Tsen struct aes_key enc_key;
22*10930333SDanny Tsen struct aes_key dec_key;
23*10930333SDanny Tsen struct aes_key tweak_key;
24*10930333SDanny Tsen };
25*10930333SDanny Tsen
p8_aes_xts_init(struct crypto_skcipher * tfm)26*10930333SDanny Tsen static int p8_aes_xts_init(struct crypto_skcipher *tfm)
27*10930333SDanny Tsen {
28*10930333SDanny Tsen struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
29*10930333SDanny Tsen struct crypto_skcipher *fallback;
30*10930333SDanny Tsen
31*10930333SDanny Tsen fallback = crypto_alloc_skcipher("xts(aes)", 0,
32*10930333SDanny Tsen CRYPTO_ALG_NEED_FALLBACK |
33*10930333SDanny Tsen CRYPTO_ALG_ASYNC);
34*10930333SDanny Tsen if (IS_ERR(fallback)) {
35*10930333SDanny Tsen pr_err("Failed to allocate xts(aes) fallback: %ld\n",
36*10930333SDanny Tsen PTR_ERR(fallback));
37*10930333SDanny Tsen return PTR_ERR(fallback);
38*10930333SDanny Tsen }
39*10930333SDanny Tsen
40*10930333SDanny Tsen crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
41*10930333SDanny Tsen crypto_skcipher_reqsize(fallback));
42*10930333SDanny Tsen ctx->fallback = fallback;
43*10930333SDanny Tsen return 0;
44*10930333SDanny Tsen }
45*10930333SDanny Tsen
p8_aes_xts_exit(struct crypto_skcipher * tfm)46*10930333SDanny Tsen static void p8_aes_xts_exit(struct crypto_skcipher *tfm)
47*10930333SDanny Tsen {
48*10930333SDanny Tsen struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
49*10930333SDanny Tsen
50*10930333SDanny Tsen crypto_free_skcipher(ctx->fallback);
51*10930333SDanny Tsen }
52*10930333SDanny Tsen
p8_aes_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)53*10930333SDanny Tsen static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
54*10930333SDanny Tsen unsigned int keylen)
55*10930333SDanny Tsen {
56*10930333SDanny Tsen struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
57*10930333SDanny Tsen int ret;
58*10930333SDanny Tsen
59*10930333SDanny Tsen ret = xts_verify_key(tfm, key, keylen);
60*10930333SDanny Tsen if (ret)
61*10930333SDanny Tsen return ret;
62*10930333SDanny Tsen
63*10930333SDanny Tsen preempt_disable();
64*10930333SDanny Tsen pagefault_disable();
65*10930333SDanny Tsen enable_kernel_vsx();
66*10930333SDanny Tsen ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key);
67*10930333SDanny Tsen ret |= aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key);
68*10930333SDanny Tsen ret |= aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key);
69*10930333SDanny Tsen disable_kernel_vsx();
70*10930333SDanny Tsen pagefault_enable();
71*10930333SDanny Tsen preempt_enable();
72*10930333SDanny Tsen
73*10930333SDanny Tsen ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
74*10930333SDanny Tsen
75*10930333SDanny Tsen return ret ? -EINVAL : 0;
76*10930333SDanny Tsen }
77*10930333SDanny Tsen
p8_aes_xts_crypt(struct skcipher_request * req,int enc)78*10930333SDanny Tsen static int p8_aes_xts_crypt(struct skcipher_request *req, int enc)
79*10930333SDanny Tsen {
80*10930333SDanny Tsen struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
81*10930333SDanny Tsen const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
82*10930333SDanny Tsen struct skcipher_walk walk;
83*10930333SDanny Tsen unsigned int nbytes;
84*10930333SDanny Tsen u8 tweak[AES_BLOCK_SIZE];
85*10930333SDanny Tsen int ret;
86*10930333SDanny Tsen
87*10930333SDanny Tsen if (req->cryptlen < AES_BLOCK_SIZE)
88*10930333SDanny Tsen return -EINVAL;
89*10930333SDanny Tsen
90*10930333SDanny Tsen if (!crypto_simd_usable() || (req->cryptlen % XTS_BLOCK_SIZE) != 0) {
91*10930333SDanny Tsen struct skcipher_request *subreq = skcipher_request_ctx(req);
92*10930333SDanny Tsen
93*10930333SDanny Tsen *subreq = *req;
94*10930333SDanny Tsen skcipher_request_set_tfm(subreq, ctx->fallback);
95*10930333SDanny Tsen return enc ? crypto_skcipher_encrypt(subreq) :
96*10930333SDanny Tsen crypto_skcipher_decrypt(subreq);
97*10930333SDanny Tsen }
98*10930333SDanny Tsen
99*10930333SDanny Tsen ret = skcipher_walk_virt(&walk, req, false);
100*10930333SDanny Tsen if (ret)
101*10930333SDanny Tsen return ret;
102*10930333SDanny Tsen
103*10930333SDanny Tsen preempt_disable();
104*10930333SDanny Tsen pagefault_disable();
105*10930333SDanny Tsen enable_kernel_vsx();
106*10930333SDanny Tsen
107*10930333SDanny Tsen aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key);
108*10930333SDanny Tsen
109*10930333SDanny Tsen disable_kernel_vsx();
110*10930333SDanny Tsen pagefault_enable();
111*10930333SDanny Tsen preempt_enable();
112*10930333SDanny Tsen
113*10930333SDanny Tsen while ((nbytes = walk.nbytes) != 0) {
114*10930333SDanny Tsen preempt_disable();
115*10930333SDanny Tsen pagefault_disable();
116*10930333SDanny Tsen enable_kernel_vsx();
117*10930333SDanny Tsen if (enc)
118*10930333SDanny Tsen aes_p8_xts_encrypt(walk.src.virt.addr,
119*10930333SDanny Tsen walk.dst.virt.addr,
120*10930333SDanny Tsen round_down(nbytes, AES_BLOCK_SIZE),
121*10930333SDanny Tsen &ctx->enc_key, NULL, tweak);
122*10930333SDanny Tsen else
123*10930333SDanny Tsen aes_p8_xts_decrypt(walk.src.virt.addr,
124*10930333SDanny Tsen walk.dst.virt.addr,
125*10930333SDanny Tsen round_down(nbytes, AES_BLOCK_SIZE),
126*10930333SDanny Tsen &ctx->dec_key, NULL, tweak);
127*10930333SDanny Tsen disable_kernel_vsx();
128*10930333SDanny Tsen pagefault_enable();
129*10930333SDanny Tsen preempt_enable();
130*10930333SDanny Tsen
131*10930333SDanny Tsen ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
132*10930333SDanny Tsen }
133*10930333SDanny Tsen return ret;
134*10930333SDanny Tsen }
135*10930333SDanny Tsen
p8_aes_xts_encrypt(struct skcipher_request * req)136*10930333SDanny Tsen static int p8_aes_xts_encrypt(struct skcipher_request *req)
137*10930333SDanny Tsen {
138*10930333SDanny Tsen return p8_aes_xts_crypt(req, 1);
139*10930333SDanny Tsen }
140*10930333SDanny Tsen
p8_aes_xts_decrypt(struct skcipher_request * req)141*10930333SDanny Tsen static int p8_aes_xts_decrypt(struct skcipher_request *req)
142*10930333SDanny Tsen {
143*10930333SDanny Tsen return p8_aes_xts_crypt(req, 0);
144*10930333SDanny Tsen }
145*10930333SDanny Tsen
146*10930333SDanny Tsen struct skcipher_alg p8_aes_xts_alg = {
147*10930333SDanny Tsen .base.cra_name = "xts(aes)",
148*10930333SDanny Tsen .base.cra_driver_name = "p8_aes_xts",
149*10930333SDanny Tsen .base.cra_module = THIS_MODULE,
150*10930333SDanny Tsen .base.cra_priority = 2000,
151*10930333SDanny Tsen .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
152*10930333SDanny Tsen .base.cra_blocksize = AES_BLOCK_SIZE,
153*10930333SDanny Tsen .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
154*10930333SDanny Tsen .setkey = p8_aes_xts_setkey,
155*10930333SDanny Tsen .encrypt = p8_aes_xts_encrypt,
156*10930333SDanny Tsen .decrypt = p8_aes_xts_decrypt,
157*10930333SDanny Tsen .init = p8_aes_xts_init,
158*10930333SDanny Tsen .exit = p8_aes_xts_exit,
159*10930333SDanny Tsen .min_keysize = 2 * AES_MIN_KEY_SIZE,
160*10930333SDanny Tsen .max_keysize = 2 * AES_MAX_KEY_SIZE,
161*10930333SDanny Tsen .ivsize = AES_BLOCK_SIZE,
162*10930333SDanny Tsen };
163