xref: /linux/arch/sparc/crypto/des_glue.c (revision 2ba9268dd603d23e17643437b2246acb6844953b) 
1 /* Glue code for DES encryption optimized for sparc64 crypto opcodes.
2  *
3  * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
4  */
5 
6 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
7 
8 #include <linux/crypto.h>
9 #include <linux/init.h>
10 #include <linux/module.h>
11 #include <linux/mm.h>
12 #include <linux/types.h>
13 #include <crypto/algapi.h>
14 #include <crypto/des.h>
15 
16 #include <asm/fpumacro.h>
17 #include <asm/pstate.h>
18 #include <asm/elf.h>
19 
20 #include "opcodes.h"
21 
22 struct des_sparc64_ctx {
23 	u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
24 	u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
25 };
26 
27 struct des3_ede_sparc64_ctx {
28 	u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
29 	u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
30 };
31 
32 static void encrypt_to_decrypt(u64 *d, const u64 *e)
33 {
34 	const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
35 	int i;
36 
37 	for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
38 		*d++ = *s--;
39 }
40 
41 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
42 
43 static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
44 		       unsigned int keylen)
45 {
46 	struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
47 	u32 *flags = &tfm->crt_flags;
48 	u32 tmp[DES_EXPKEY_WORDS];
49 	int ret;
50 
51 	/* Even though we have special instructions for key expansion,
52 	 * we call des_ekey() so that we don't have to write our own
53 	 * weak key detection code.
54 	 */
55 	ret = des_ekey(tmp, key);
56 	if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
57 		*flags |= CRYPTO_TFM_RES_WEAK_KEY;
58 		return -EINVAL;
59 	}
60 
61 	des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
62 	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
63 
64 	return 0;
65 }
66 
67 extern void des_sparc64_crypt(const u64 *key, const u64 *input,
68 			      u64 *output);
69 
70 static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
71 {
72 	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
73 	const u64 *K = ctx->encrypt_expkey;
74 
75 	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
76 }
77 
78 static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
79 {
80 	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
81 	const u64 *K = ctx->decrypt_expkey;
82 
83 	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
84 }
85 
86 extern void des_sparc64_load_keys(const u64 *key);
87 
88 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
89 				  unsigned int len);
90 
91 #define DES_BLOCK_MASK	(~(DES_BLOCK_SIZE - 1))
92 
93 static int __ecb_crypt(struct blkcipher_desc *desc,
94 		       struct scatterlist *dst, struct scatterlist *src,
95 		       unsigned int nbytes, bool encrypt)
96 {
97 	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
98 	struct blkcipher_walk walk;
99 	int err;
100 
101 	blkcipher_walk_init(&walk, dst, src, nbytes);
102 	err = blkcipher_walk_virt(desc, &walk);
103 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
104 
105 	if (encrypt)
106 		des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
107 	else
108 		des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
109 	while ((nbytes = walk.nbytes)) {
110 		unsigned int block_len = nbytes & DES_BLOCK_MASK;
111 
112 		if (likely(block_len)) {
113 			des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
114 					      (u64 *) walk.dst.virt.addr,
115 					      block_len);
116 		}
117 		nbytes &= DES_BLOCK_SIZE - 1;
118 		err = blkcipher_walk_done(desc, &walk, nbytes);
119 	}
120 	fprs_write(0);
121 	return err;
122 }
123 
124 static int ecb_encrypt(struct blkcipher_desc *desc,
125 		       struct scatterlist *dst, struct scatterlist *src,
126 		       unsigned int nbytes)
127 {
128 	return __ecb_crypt(desc, dst, src, nbytes, true);
129 }
130 
131 static int ecb_decrypt(struct blkcipher_desc *desc,
132 		       struct scatterlist *dst, struct scatterlist *src,
133 		       unsigned int nbytes)
134 {
135 	return __ecb_crypt(desc, dst, src, nbytes, false);
136 }
137 
138 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
139 				    unsigned int len, u64 *iv);
140 
141 static int cbc_encrypt(struct blkcipher_desc *desc,
142 		       struct scatterlist *dst, struct scatterlist *src,
143 		       unsigned int nbytes)
144 {
145 	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
146 	struct blkcipher_walk walk;
147 	int err;
148 
149 	blkcipher_walk_init(&walk, dst, src, nbytes);
150 	err = blkcipher_walk_virt(desc, &walk);
151 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
152 
153 	des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
154 	while ((nbytes = walk.nbytes)) {
155 		unsigned int block_len = nbytes & DES_BLOCK_MASK;
156 
157 		if (likely(block_len)) {
158 			des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
159 						(u64 *) walk.dst.virt.addr,
160 						block_len, (u64 *) walk.iv);
161 		}
162 		nbytes &= DES_BLOCK_SIZE - 1;
163 		err = blkcipher_walk_done(desc, &walk, nbytes);
164 	}
165 	fprs_write(0);
166 	return err;
167 }
168 
169 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
170 				    unsigned int len, u64 *iv);
171 
172 static int cbc_decrypt(struct blkcipher_desc *desc,
173 		       struct scatterlist *dst, struct scatterlist *src,
174 		       unsigned int nbytes)
175 {
176 	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
177 	struct blkcipher_walk walk;
178 	int err;
179 
180 	blkcipher_walk_init(&walk, dst, src, nbytes);
181 	err = blkcipher_walk_virt(desc, &walk);
182 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
183 
184 	des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
185 	while ((nbytes = walk.nbytes)) {
186 		unsigned int block_len = nbytes & DES_BLOCK_MASK;
187 
188 		if (likely(block_len)) {
189 			des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
190 						(u64 *) walk.dst.virt.addr,
191 						block_len, (u64 *) walk.iv);
192 		}
193 		nbytes &= DES_BLOCK_SIZE - 1;
194 		err = blkcipher_walk_done(desc, &walk, nbytes);
195 	}
196 	fprs_write(0);
197 	return err;
198 }
199 
200 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
201 			    unsigned int keylen)
202 {
203 	struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
204 	const u32 *K = (const u32 *)key;
205 	u32 *flags = &tfm->crt_flags;
206 	u64 k1[DES_EXPKEY_WORDS / 2];
207 	u64 k2[DES_EXPKEY_WORDS / 2];
208 	u64 k3[DES_EXPKEY_WORDS / 2];
209 
210 	if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
211 		     !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
212 		     (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
213 		*flags |= CRYPTO_TFM_RES_WEAK_KEY;
214 		return -EINVAL;
215 	}
216 
217 	des_sparc64_key_expand((const u32 *)key, k1);
218 	key += DES_KEY_SIZE;
219 	des_sparc64_key_expand((const u32 *)key, k2);
220 	key += DES_KEY_SIZE;
221 	des_sparc64_key_expand((const u32 *)key, k3);
222 
223 	memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
224 	encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
225 	memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
226 	       &k3[0], sizeof(k3));
227 
228 	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
229 	memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
230 	       &k2[0], sizeof(k2));
231 	encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
232 			   &k1[0]);
233 
234 	return 0;
235 }
236 
237 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
238 				   u64 *output);
239 
240 static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
241 {
242 	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
243 	const u64 *K = ctx->encrypt_expkey;
244 
245 	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
246 }
247 
248 static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
249 {
250 	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
251 	const u64 *K = ctx->decrypt_expkey;
252 
253 	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
254 }
255 
256 extern void des3_ede_sparc64_load_keys(const u64 *key);
257 
258 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
259 				       u64 *output, unsigned int len);
260 
261 static int __ecb3_crypt(struct blkcipher_desc *desc,
262 			struct scatterlist *dst, struct scatterlist *src,
263 			unsigned int nbytes, bool encrypt)
264 {
265 	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
266 	struct blkcipher_walk walk;
267 	const u64 *K;
268 	int err;
269 
270 	blkcipher_walk_init(&walk, dst, src, nbytes);
271 	err = blkcipher_walk_virt(desc, &walk);
272 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
273 
274 	if (encrypt)
275 		K = &ctx->encrypt_expkey[0];
276 	else
277 		K = &ctx->decrypt_expkey[0];
278 	des3_ede_sparc64_load_keys(K);
279 	while ((nbytes = walk.nbytes)) {
280 		unsigned int block_len = nbytes & DES_BLOCK_MASK;
281 
282 		if (likely(block_len)) {
283 			const u64 *src64 = (const u64 *)walk.src.virt.addr;
284 			des3_ede_sparc64_ecb_crypt(K, src64,
285 						   (u64 *) walk.dst.virt.addr,
286 						   block_len);
287 		}
288 		nbytes &= DES_BLOCK_SIZE - 1;
289 		err = blkcipher_walk_done(desc, &walk, nbytes);
290 	}
291 	fprs_write(0);
292 	return err;
293 }
294 
295 static int ecb3_encrypt(struct blkcipher_desc *desc,
296 		       struct scatterlist *dst, struct scatterlist *src,
297 		       unsigned int nbytes)
298 {
299 	return __ecb3_crypt(desc, dst, src, nbytes, true);
300 }
301 
302 static int ecb3_decrypt(struct blkcipher_desc *desc,
303 		       struct scatterlist *dst, struct scatterlist *src,
304 		       unsigned int nbytes)
305 {
306 	return __ecb3_crypt(desc, dst, src, nbytes, false);
307 }
308 
309 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
310 					 u64 *output, unsigned int len,
311 					 u64 *iv);
312 
313 static int cbc3_encrypt(struct blkcipher_desc *desc,
314 			struct scatterlist *dst, struct scatterlist *src,
315 			unsigned int nbytes)
316 {
317 	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
318 	struct blkcipher_walk walk;
319 	const u64 *K;
320 	int err;
321 
322 	blkcipher_walk_init(&walk, dst, src, nbytes);
323 	err = blkcipher_walk_virt(desc, &walk);
324 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
325 
326 	K = &ctx->encrypt_expkey[0];
327 	des3_ede_sparc64_load_keys(K);
328 	while ((nbytes = walk.nbytes)) {
329 		unsigned int block_len = nbytes & DES_BLOCK_MASK;
330 
331 		if (likely(block_len)) {
332 			const u64 *src64 = (const u64 *)walk.src.virt.addr;
333 			des3_ede_sparc64_cbc_encrypt(K, src64,
334 						     (u64 *) walk.dst.virt.addr,
335 						     block_len,
336 						     (u64 *) walk.iv);
337 		}
338 		nbytes &= DES_BLOCK_SIZE - 1;
339 		err = blkcipher_walk_done(desc, &walk, nbytes);
340 	}
341 	fprs_write(0);
342 	return err;
343 }
344 
345 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
346 					 u64 *output, unsigned int len,
347 					 u64 *iv);
348 
349 static int cbc3_decrypt(struct blkcipher_desc *desc,
350 			struct scatterlist *dst, struct scatterlist *src,
351 			unsigned int nbytes)
352 {
353 	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
354 	struct blkcipher_walk walk;
355 	const u64 *K;
356 	int err;
357 
358 	blkcipher_walk_init(&walk, dst, src, nbytes);
359 	err = blkcipher_walk_virt(desc, &walk);
360 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
361 
362 	K = &ctx->decrypt_expkey[0];
363 	des3_ede_sparc64_load_keys(K);
364 	while ((nbytes = walk.nbytes)) {
365 		unsigned int block_len = nbytes & DES_BLOCK_MASK;
366 
367 		if (likely(block_len)) {
368 			const u64 *src64 = (const u64 *)walk.src.virt.addr;
369 			des3_ede_sparc64_cbc_decrypt(K, src64,
370 						     (u64 *) walk.dst.virt.addr,
371 						     block_len,
372 						     (u64 *) walk.iv);
373 		}
374 		nbytes &= DES_BLOCK_SIZE - 1;
375 		err = blkcipher_walk_done(desc, &walk, nbytes);
376 	}
377 	fprs_write(0);
378 	return err;
379 }
380 
381 static struct crypto_alg algs[] = { {
382 	.cra_name		= "des",
383 	.cra_driver_name	= "des-sparc64",
384 	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
385 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
386 	.cra_blocksize		= DES_BLOCK_SIZE,
387 	.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
388 	.cra_alignmask		= 7,
389 	.cra_module		= THIS_MODULE,
390 	.cra_u	= {
391 		.cipher	= {
392 			.cia_min_keysize	= DES_KEY_SIZE,
393 			.cia_max_keysize	= DES_KEY_SIZE,
394 			.cia_setkey		= des_set_key,
395 			.cia_encrypt		= des_encrypt,
396 			.cia_decrypt		= des_decrypt
397 		}
398 	}
399 }, {
400 	.cra_name		= "ecb(des)",
401 	.cra_driver_name	= "ecb-des-sparc64",
402 	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
403 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
404 	.cra_blocksize		= DES_BLOCK_SIZE,
405 	.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
406 	.cra_alignmask		= 7,
407 	.cra_type		= &crypto_blkcipher_type,
408 	.cra_module		= THIS_MODULE,
409 	.cra_u = {
410 		.blkcipher = {
411 			.min_keysize	= DES_KEY_SIZE,
412 			.max_keysize	= DES_KEY_SIZE,
413 			.setkey		= des_set_key,
414 			.encrypt	= ecb_encrypt,
415 			.decrypt	= ecb_decrypt,
416 		},
417 	},
418 }, {
419 	.cra_name		= "cbc(des)",
420 	.cra_driver_name	= "cbc-des-sparc64",
421 	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
422 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
423 	.cra_blocksize		= DES_BLOCK_SIZE,
424 	.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
425 	.cra_alignmask		= 7,
426 	.cra_type		= &crypto_blkcipher_type,
427 	.cra_module		= THIS_MODULE,
428 	.cra_u = {
429 		.blkcipher = {
430 			.min_keysize	= DES_KEY_SIZE,
431 			.max_keysize	= DES_KEY_SIZE,
432 			.setkey		= des_set_key,
433 			.encrypt	= cbc_encrypt,
434 			.decrypt	= cbc_decrypt,
435 		},
436 	},
437 }, {
438 	.cra_name		= "des3_ede",
439 	.cra_driver_name	= "des3_ede-sparc64",
440 	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
441 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
442 	.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
443 	.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
444 	.cra_alignmask		= 7,
445 	.cra_module		= THIS_MODULE,
446 	.cra_u	= {
447 		.cipher	= {
448 			.cia_min_keysize	= DES3_EDE_KEY_SIZE,
449 			.cia_max_keysize	= DES3_EDE_KEY_SIZE,
450 			.cia_setkey		= des3_ede_set_key,
451 			.cia_encrypt		= des3_ede_encrypt,
452 			.cia_decrypt		= des3_ede_decrypt
453 		}
454 	}
455 }, {
456 	.cra_name		= "ecb(des3_ede)",
457 	.cra_driver_name	= "ecb-des3_ede-sparc64",
458 	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
459 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
460 	.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
461 	.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
462 	.cra_alignmask		= 7,
463 	.cra_type		= &crypto_blkcipher_type,
464 	.cra_module		= THIS_MODULE,
465 	.cra_u = {
466 		.blkcipher = {
467 			.min_keysize	= DES3_EDE_KEY_SIZE,
468 			.max_keysize	= DES3_EDE_KEY_SIZE,
469 			.setkey		= des3_ede_set_key,
470 			.encrypt	= ecb3_encrypt,
471 			.decrypt	= ecb3_decrypt,
472 		},
473 	},
474 }, {
475 	.cra_name		= "cbc(des3_ede)",
476 	.cra_driver_name	= "cbc-des3_ede-sparc64",
477 	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
478 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
479 	.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
480 	.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
481 	.cra_alignmask		= 7,
482 	.cra_type		= &crypto_blkcipher_type,
483 	.cra_module		= THIS_MODULE,
484 	.cra_u = {
485 		.blkcipher = {
486 			.min_keysize	= DES3_EDE_KEY_SIZE,
487 			.max_keysize	= DES3_EDE_KEY_SIZE,
488 			.setkey		= des3_ede_set_key,
489 			.encrypt	= cbc3_encrypt,
490 			.decrypt	= cbc3_decrypt,
491 		},
492 	},
493 } };
494 
495 static bool __init sparc64_has_des_opcode(void)
496 {
497 	unsigned long cfr;
498 
499 	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
500 		return false;
501 
502 	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
503 	if (!(cfr & CFR_DES))
504 		return false;
505 
506 	return true;
507 }
508 
509 static int __init des_sparc64_mod_init(void)
510 {
511 	int i;
512 
513 	for (i = 0; i < ARRAY_SIZE(algs); i++)
514 		INIT_LIST_HEAD(&algs[i].cra_list);
515 
516 	if (sparc64_has_des_opcode()) {
517 		pr_info("Using sparc64 des opcodes optimized DES implementation\n");
518 		return crypto_register_algs(algs, ARRAY_SIZE(algs));
519 	}
520 	pr_info("sparc64 des opcodes not available.\n");
521 	return -ENODEV;
522 }
523 
524 static void __exit des_sparc64_mod_fini(void)
525 {
526 	crypto_unregister_algs(algs, ARRAY_SIZE(algs));
527 }
528 
529 module_init(des_sparc64_mod_init);
530 module_exit(des_sparc64_mod_fini);
531 
532 MODULE_LICENSE("GPL");
533 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
534 
535 MODULE_ALIAS_CRYPTO("des");
536 MODULE_ALIAS_CRYPTO("des3_ede");
537 
538 #include "crop_devid.c"
539