xref: /linux/arch/sparc/crypto/aes_glue.c (revision 8dd06ef34b6e2f41b29fbf5fc1663780f2524285)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Glue code for AES encryption optimized for sparc64 crypto opcodes.
3  *
4  * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
5  *
6  * Copyright (C) 2008, Intel Corp.
7  *    Author: Huang Ying <ying.huang@intel.com>
8  *
9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10  * interface for 64-bit kernels.
11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
13  *             Tadeusz Struk (tadeusz.struk@intel.com)
14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
15  *    Copyright (c) 2010, Intel Corporation.
16  */
17 
18 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
19 
20 #include <linux/crypto.h>
21 #include <linux/init.h>
22 #include <linux/module.h>
23 #include <linux/mm.h>
24 #include <linux/types.h>
25 #include <crypto/algapi.h>
26 #include <crypto/aes.h>
27 #include <crypto/internal/skcipher.h>
28 
29 #include <asm/fpumacro.h>
30 #include <asm/pstate.h>
31 #include <asm/elf.h>
32 
33 #include "opcodes.h"
34 
35 struct aes_ops {
36 	void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
37 	void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
38 	void (*load_encrypt_keys)(const u64 *key);
39 	void (*load_decrypt_keys)(const u64 *key);
40 	void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
41 			    unsigned int len);
42 	void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
43 			    unsigned int len);
44 	void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
45 			    unsigned int len, u64 *iv);
46 	void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
47 			    unsigned int len, u64 *iv);
48 	void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
49 			  unsigned int len, u64 *iv);
50 };
51 
52 struct crypto_sparc64_aes_ctx {
53 	struct aes_ops *ops;
54 	u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
55 	u32 key_length;
56 	u32 expanded_key_length;
57 };
58 
59 extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
60 				    u32 *output);
61 extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
62 				    u32 *output);
63 extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
64 				    u32 *output);
65 
66 extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
67 				    u32 *output);
68 extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
69 				    u32 *output);
70 extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
71 				    u32 *output);
72 
73 extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
74 extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
75 extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
76 
77 extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
78 extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
79 extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
80 
81 extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
82 					u64 *output, unsigned int len);
83 extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
84 					u64 *output, unsigned int len);
85 extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
86 					u64 *output, unsigned int len);
87 
88 extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
89 					u64 *output, unsigned int len);
90 extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
91 					u64 *output, unsigned int len);
92 extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
93 					u64 *output, unsigned int len);
94 
95 extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
96 					u64 *output, unsigned int len,
97 					u64 *iv);
98 
99 extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
100 					u64 *output, unsigned int len,
101 					u64 *iv);
102 
103 extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
104 					u64 *output, unsigned int len,
105 					u64 *iv);
106 
107 extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
108 					u64 *output, unsigned int len,
109 					u64 *iv);
110 
111 extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
112 					u64 *output, unsigned int len,
113 					u64 *iv);
114 
115 extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
116 					u64 *output, unsigned int len,
117 					u64 *iv);
118 
119 extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
120 				      u64 *output, unsigned int len,
121 				      u64 *iv);
122 extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
123 				      u64 *output, unsigned int len,
124 				      u64 *iv);
125 extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
126 				      u64 *output, unsigned int len,
127 				      u64 *iv);
128 
129 static struct aes_ops aes128_ops = {
130 	.encrypt		= aes_sparc64_encrypt_128,
131 	.decrypt		= aes_sparc64_decrypt_128,
132 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_128,
133 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_128,
134 	.ecb_encrypt		= aes_sparc64_ecb_encrypt_128,
135 	.ecb_decrypt		= aes_sparc64_ecb_decrypt_128,
136 	.cbc_encrypt		= aes_sparc64_cbc_encrypt_128,
137 	.cbc_decrypt		= aes_sparc64_cbc_decrypt_128,
138 	.ctr_crypt		= aes_sparc64_ctr_crypt_128,
139 };
140 
141 static struct aes_ops aes192_ops = {
142 	.encrypt		= aes_sparc64_encrypt_192,
143 	.decrypt		= aes_sparc64_decrypt_192,
144 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_192,
145 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_192,
146 	.ecb_encrypt		= aes_sparc64_ecb_encrypt_192,
147 	.ecb_decrypt		= aes_sparc64_ecb_decrypt_192,
148 	.cbc_encrypt		= aes_sparc64_cbc_encrypt_192,
149 	.cbc_decrypt		= aes_sparc64_cbc_decrypt_192,
150 	.ctr_crypt		= aes_sparc64_ctr_crypt_192,
151 };
152 
153 static struct aes_ops aes256_ops = {
154 	.encrypt		= aes_sparc64_encrypt_256,
155 	.decrypt		= aes_sparc64_decrypt_256,
156 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_256,
157 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_256,
158 	.ecb_encrypt		= aes_sparc64_ecb_encrypt_256,
159 	.ecb_decrypt		= aes_sparc64_ecb_decrypt_256,
160 	.cbc_encrypt		= aes_sparc64_cbc_encrypt_256,
161 	.cbc_decrypt		= aes_sparc64_cbc_decrypt_256,
162 	.ctr_crypt		= aes_sparc64_ctr_crypt_256,
163 };
164 
165 extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
166 				   unsigned int key_len);
167 
aes_set_key(struct crypto_tfm * tfm,const u8 * in_key,unsigned int key_len)168 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
169 		       unsigned int key_len)
170 {
171 	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
172 
173 	switch (key_len) {
174 	case AES_KEYSIZE_128:
175 		ctx->expanded_key_length = 0xb0;
176 		ctx->ops = &aes128_ops;
177 		break;
178 
179 	case AES_KEYSIZE_192:
180 		ctx->expanded_key_length = 0xd0;
181 		ctx->ops = &aes192_ops;
182 		break;
183 
184 	case AES_KEYSIZE_256:
185 		ctx->expanded_key_length = 0xf0;
186 		ctx->ops = &aes256_ops;
187 		break;
188 
189 	default:
190 		return -EINVAL;
191 	}
192 
193 	aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
194 	ctx->key_length = key_len;
195 
196 	return 0;
197 }
198 
aes_set_key_skcipher(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)199 static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
200 				unsigned int key_len)
201 {
202 	return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len);
203 }
204 
crypto_aes_encrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)205 static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
206 {
207 	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
208 
209 	ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
210 }
211 
crypto_aes_decrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)212 static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
213 {
214 	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
215 
216 	ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
217 }
218 
ecb_encrypt(struct skcipher_request * req)219 static int ecb_encrypt(struct skcipher_request *req)
220 {
221 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
222 	const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
223 	struct skcipher_walk walk;
224 	unsigned int nbytes;
225 	int err;
226 
227 	err = skcipher_walk_virt(&walk, req, true);
228 	if (err)
229 		return err;
230 
231 	ctx->ops->load_encrypt_keys(&ctx->key[0]);
232 	while ((nbytes = walk.nbytes) != 0) {
233 		ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr,
234 				      walk.dst.virt.addr,
235 				      round_down(nbytes, AES_BLOCK_SIZE));
236 		err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
237 	}
238 	fprs_write(0);
239 	return err;
240 }
241 
ecb_decrypt(struct skcipher_request * req)242 static int ecb_decrypt(struct skcipher_request *req)
243 {
244 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
245 	const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
246 	const u64 *key_end;
247 	struct skcipher_walk walk;
248 	unsigned int nbytes;
249 	int err;
250 
251 	err = skcipher_walk_virt(&walk, req, true);
252 	if (err)
253 		return err;
254 
255 	ctx->ops->load_decrypt_keys(&ctx->key[0]);
256 	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
257 	while ((nbytes = walk.nbytes) != 0) {
258 		ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr,
259 				      walk.dst.virt.addr,
260 				      round_down(nbytes, AES_BLOCK_SIZE));
261 		err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
262 	}
263 	fprs_write(0);
264 
265 	return err;
266 }
267 
cbc_encrypt(struct skcipher_request * req)268 static int cbc_encrypt(struct skcipher_request *req)
269 {
270 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
271 	const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
272 	struct skcipher_walk walk;
273 	unsigned int nbytes;
274 	int err;
275 
276 	err = skcipher_walk_virt(&walk, req, true);
277 	if (err)
278 		return err;
279 
280 	ctx->ops->load_encrypt_keys(&ctx->key[0]);
281 	while ((nbytes = walk.nbytes) != 0) {
282 		ctx->ops->cbc_encrypt(&ctx->key[0], walk.src.virt.addr,
283 				      walk.dst.virt.addr,
284 				      round_down(nbytes, AES_BLOCK_SIZE),
285 				      walk.iv);
286 		err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
287 	}
288 	fprs_write(0);
289 	return err;
290 }
291 
cbc_decrypt(struct skcipher_request * req)292 static int cbc_decrypt(struct skcipher_request *req)
293 {
294 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
295 	const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
296 	const u64 *key_end;
297 	struct skcipher_walk walk;
298 	unsigned int nbytes;
299 	int err;
300 
301 	err = skcipher_walk_virt(&walk, req, true);
302 	if (err)
303 		return err;
304 
305 	ctx->ops->load_decrypt_keys(&ctx->key[0]);
306 	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
307 	while ((nbytes = walk.nbytes) != 0) {
308 		ctx->ops->cbc_decrypt(key_end, walk.src.virt.addr,
309 				      walk.dst.virt.addr,
310 				      round_down(nbytes, AES_BLOCK_SIZE),
311 				      walk.iv);
312 		err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
313 	}
314 	fprs_write(0);
315 
316 	return err;
317 }
318 
ctr_crypt_final(const struct crypto_sparc64_aes_ctx * ctx,struct skcipher_walk * walk)319 static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx,
320 			    struct skcipher_walk *walk)
321 {
322 	u8 *ctrblk = walk->iv;
323 	u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
324 	u8 *src = walk->src.virt.addr;
325 	u8 *dst = walk->dst.virt.addr;
326 	unsigned int nbytes = walk->nbytes;
327 
328 	ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
329 			      keystream, AES_BLOCK_SIZE);
330 	crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
331 	crypto_inc(ctrblk, AES_BLOCK_SIZE);
332 }
333 
ctr_crypt(struct skcipher_request * req)334 static int ctr_crypt(struct skcipher_request *req)
335 {
336 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
337 	const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
338 	struct skcipher_walk walk;
339 	unsigned int nbytes;
340 	int err;
341 
342 	err = skcipher_walk_virt(&walk, req, true);
343 	if (err)
344 		return err;
345 
346 	ctx->ops->load_encrypt_keys(&ctx->key[0]);
347 	while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
348 		ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr,
349 				    walk.dst.virt.addr,
350 				    round_down(nbytes, AES_BLOCK_SIZE),
351 				    walk.iv);
352 		err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
353 	}
354 	if (walk.nbytes) {
355 		ctr_crypt_final(ctx, &walk);
356 		err = skcipher_walk_done(&walk, 0);
357 	}
358 	fprs_write(0);
359 	return err;
360 }
361 
362 static struct crypto_alg cipher_alg = {
363 	.cra_name		= "aes",
364 	.cra_driver_name	= "aes-sparc64",
365 	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
366 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
367 	.cra_blocksize		= AES_BLOCK_SIZE,
368 	.cra_ctxsize		= sizeof(struct crypto_sparc64_aes_ctx),
369 	.cra_alignmask		= 3,
370 	.cra_module		= THIS_MODULE,
371 	.cra_u	= {
372 		.cipher	= {
373 			.cia_min_keysize	= AES_MIN_KEY_SIZE,
374 			.cia_max_keysize	= AES_MAX_KEY_SIZE,
375 			.cia_setkey		= aes_set_key,
376 			.cia_encrypt		= crypto_aes_encrypt,
377 			.cia_decrypt		= crypto_aes_decrypt
378 		}
379 	}
380 };
381 
382 static struct skcipher_alg skcipher_algs[] = {
383 	{
384 		.base.cra_name		= "ecb(aes)",
385 		.base.cra_driver_name	= "ecb-aes-sparc64",
386 		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
387 		.base.cra_blocksize	= AES_BLOCK_SIZE,
388 		.base.cra_ctxsize	= sizeof(struct crypto_sparc64_aes_ctx),
389 		.base.cra_alignmask	= 7,
390 		.base.cra_module	= THIS_MODULE,
391 		.min_keysize		= AES_MIN_KEY_SIZE,
392 		.max_keysize		= AES_MAX_KEY_SIZE,
393 		.setkey			= aes_set_key_skcipher,
394 		.encrypt		= ecb_encrypt,
395 		.decrypt		= ecb_decrypt,
396 	}, {
397 		.base.cra_name		= "cbc(aes)",
398 		.base.cra_driver_name	= "cbc-aes-sparc64",
399 		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
400 		.base.cra_blocksize	= AES_BLOCK_SIZE,
401 		.base.cra_ctxsize	= sizeof(struct crypto_sparc64_aes_ctx),
402 		.base.cra_alignmask	= 7,
403 		.base.cra_module	= THIS_MODULE,
404 		.min_keysize		= AES_MIN_KEY_SIZE,
405 		.max_keysize		= AES_MAX_KEY_SIZE,
406 		.ivsize			= AES_BLOCK_SIZE,
407 		.setkey			= aes_set_key_skcipher,
408 		.encrypt		= cbc_encrypt,
409 		.decrypt		= cbc_decrypt,
410 	}, {
411 		.base.cra_name		= "ctr(aes)",
412 		.base.cra_driver_name	= "ctr-aes-sparc64",
413 		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
414 		.base.cra_blocksize	= 1,
415 		.base.cra_ctxsize	= sizeof(struct crypto_sparc64_aes_ctx),
416 		.base.cra_alignmask	= 7,
417 		.base.cra_module	= THIS_MODULE,
418 		.min_keysize		= AES_MIN_KEY_SIZE,
419 		.max_keysize		= AES_MAX_KEY_SIZE,
420 		.ivsize			= AES_BLOCK_SIZE,
421 		.setkey			= aes_set_key_skcipher,
422 		.encrypt		= ctr_crypt,
423 		.decrypt		= ctr_crypt,
424 		.chunksize		= AES_BLOCK_SIZE,
425 	}
426 };
427 
sparc64_has_aes_opcode(void)428 static bool __init sparc64_has_aes_opcode(void)
429 {
430 	unsigned long cfr;
431 
432 	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
433 		return false;
434 
435 	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
436 	if (!(cfr & CFR_AES))
437 		return false;
438 
439 	return true;
440 }
441 
aes_sparc64_mod_init(void)442 static int __init aes_sparc64_mod_init(void)
443 {
444 	int err;
445 
446 	if (!sparc64_has_aes_opcode()) {
447 		pr_info("sparc64 aes opcodes not available.\n");
448 		return -ENODEV;
449 	}
450 	pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
451 	err = crypto_register_alg(&cipher_alg);
452 	if (err)
453 		return err;
454 	err = crypto_register_skciphers(skcipher_algs,
455 					ARRAY_SIZE(skcipher_algs));
456 	if (err)
457 		crypto_unregister_alg(&cipher_alg);
458 	return err;
459 }
460 
aes_sparc64_mod_fini(void)461 static void __exit aes_sparc64_mod_fini(void)
462 {
463 	crypto_unregister_alg(&cipher_alg);
464 	crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
465 }
466 
467 module_init(aes_sparc64_mod_init);
468 module_exit(aes_sparc64_mod_fini);
469 
470 MODULE_LICENSE("GPL");
471 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated");
472 
473 MODULE_ALIAS_CRYPTO("aes");
474 
475 #include "crop_devid.c"
476