xref: /linux/arch/s390/crypto/paes_s390.c (revision d163d60258c755845cbc9cfe0e45fca71e649488)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Cryptographic API.
4  *
5  * s390 implementation of the AES Cipher Algorithm with protected keys.
6  *
7  * s390 Version:
8  *   Copyright IBM Corp. 2017, 2023
9  *   Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
10  *		Harald Freudenberger <freude@de.ibm.com>
11  */
12 
13 #define KMSG_COMPONENT "paes_s390"
14 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
15 
16 #include <crypto/aes.h>
17 #include <crypto/algapi.h>
18 #include <linux/bug.h>
19 #include <linux/err.h>
20 #include <linux/module.h>
21 #include <linux/cpufeature.h>
22 #include <linux/init.h>
23 #include <linux/mutex.h>
24 #include <linux/spinlock.h>
25 #include <linux/delay.h>
26 #include <crypto/internal/skcipher.h>
27 #include <crypto/xts.h>
28 #include <asm/cpacf.h>
29 #include <asm/pkey.h>
30 
31 /*
32  * Key blobs smaller/bigger than these defines are rejected
33  * by the common code even before the individual setkey function
34  * is called. As paes can handle different kinds of key blobs
35  * and padding is also possible, the limits need to be generous.
36  */
37 #define PAES_MIN_KEYSIZE 16
38 #define PAES_MAX_KEYSIZE MAXEP11AESKEYBLOBSIZE
39 
40 static u8 *ctrblk;
41 static DEFINE_MUTEX(ctrblk_lock);
42 
43 static cpacf_mask_t km_functions, kmc_functions, kmctr_functions;
44 
45 struct key_blob {
46 	/*
47 	 * Small keys will be stored in the keybuf. Larger keys are
48 	 * stored in extra allocated memory. In both cases does
49 	 * key point to the memory where the key is stored.
50 	 * The code distinguishes by checking keylen against
51 	 * sizeof(keybuf). See the two following helper functions.
52 	 */
53 	u8 *key;
54 	u8 keybuf[128];
55 	unsigned int keylen;
56 };
57 
58 static inline int _key_to_kb(struct key_blob *kb,
59 			     const u8 *key,
60 			     unsigned int keylen)
61 {
62 	struct clearkey_header {
63 		u8  type;
64 		u8  res0[3];
65 		u8  version;
66 		u8  res1[3];
67 		u32 keytype;
68 		u32 len;
69 	} __packed * h;
70 
71 	switch (keylen) {
72 	case 16:
73 	case 24:
74 	case 32:
75 		/* clear key value, prepare pkey clear key token in keybuf */
76 		memset(kb->keybuf, 0, sizeof(kb->keybuf));
77 		h = (struct clearkey_header *) kb->keybuf;
78 		h->version = 0x02; /* TOKVER_CLEAR_KEY */
79 		h->keytype = (keylen - 8) >> 3;
80 		h->len = keylen;
81 		memcpy(kb->keybuf + sizeof(*h), key, keylen);
82 		kb->keylen = sizeof(*h) + keylen;
83 		kb->key = kb->keybuf;
84 		break;
85 	default:
86 		/* other key material, let pkey handle this */
87 		if (keylen <= sizeof(kb->keybuf))
88 			kb->key = kb->keybuf;
89 		else {
90 			kb->key = kmalloc(keylen, GFP_KERNEL);
91 			if (!kb->key)
92 				return -ENOMEM;
93 		}
94 		memcpy(kb->key, key, keylen);
95 		kb->keylen = keylen;
96 		break;
97 	}
98 
99 	return 0;
100 }
101 
102 static inline void _free_kb_keybuf(struct key_blob *kb)
103 {
104 	if (kb->key && kb->key != kb->keybuf
105 	    && kb->keylen > sizeof(kb->keybuf)) {
106 		kfree_sensitive(kb->key);
107 		kb->key = NULL;
108 	}
109 }
110 
111 struct s390_paes_ctx {
112 	struct key_blob kb;
113 	struct pkey_protkey pk;
114 	spinlock_t pk_lock;
115 	unsigned long fc;
116 };
117 
118 struct s390_pxts_ctx {
119 	struct key_blob kb[2];
120 	struct pkey_protkey pk[2];
121 	spinlock_t pk_lock;
122 	unsigned long fc;
123 };
124 
125 static inline int __paes_keyblob2pkey(struct key_blob *kb,
126 				     struct pkey_protkey *pk)
127 {
128 	int i, ret = -EIO;
129 
130 	/* try three times in case of busy card */
131 	for (i = 0; ret && i < 3; i++) {
132 		if (ret == -EBUSY && in_task()) {
133 			if (msleep_interruptible(1000))
134 				return -EINTR;
135 		}
136 		ret = pkey_keyblob2pkey(kb->key, kb->keylen,
137 					pk->protkey, &pk->len, &pk->type);
138 	}
139 
140 	return ret;
141 }
142 
143 static inline int __paes_convert_key(struct s390_paes_ctx *ctx)
144 {
145 	int ret;
146 	struct pkey_protkey pkey;
147 
148 	pkey.len = sizeof(pkey.protkey);
149 	ret = __paes_keyblob2pkey(&ctx->kb, &pkey);
150 	if (ret)
151 		return ret;
152 
153 	spin_lock_bh(&ctx->pk_lock);
154 	memcpy(&ctx->pk, &pkey, sizeof(pkey));
155 	spin_unlock_bh(&ctx->pk_lock);
156 
157 	return 0;
158 }
159 
160 static int ecb_paes_init(struct crypto_skcipher *tfm)
161 {
162 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
163 
164 	ctx->kb.key = NULL;
165 	spin_lock_init(&ctx->pk_lock);
166 
167 	return 0;
168 }
169 
170 static void ecb_paes_exit(struct crypto_skcipher *tfm)
171 {
172 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
173 
174 	_free_kb_keybuf(&ctx->kb);
175 }
176 
177 static inline int __ecb_paes_set_key(struct s390_paes_ctx *ctx)
178 {
179 	int rc;
180 	unsigned long fc;
181 
182 	rc = __paes_convert_key(ctx);
183 	if (rc)
184 		return rc;
185 
186 	/* Pick the correct function code based on the protected key type */
187 	fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 :
188 		(ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 :
189 		(ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0;
190 
191 	/* Check if the function code is available */
192 	ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
193 
194 	return ctx->fc ? 0 : -EINVAL;
195 }
196 
197 static int ecb_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
198 			    unsigned int key_len)
199 {
200 	int rc;
201 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
202 
203 	_free_kb_keybuf(&ctx->kb);
204 	rc = _key_to_kb(&ctx->kb, in_key, key_len);
205 	if (rc)
206 		return rc;
207 
208 	return __ecb_paes_set_key(ctx);
209 }
210 
211 static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier)
212 {
213 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
214 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
215 	struct skcipher_walk walk;
216 	unsigned int nbytes, n, k;
217 	int ret;
218 	struct {
219 		u8 key[MAXPROTKEYSIZE];
220 	} param;
221 
222 	ret = skcipher_walk_virt(&walk, req, false);
223 	if (ret)
224 		return ret;
225 
226 	spin_lock_bh(&ctx->pk_lock);
227 	memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
228 	spin_unlock_bh(&ctx->pk_lock);
229 
230 	while ((nbytes = walk.nbytes) != 0) {
231 		/* only use complete blocks */
232 		n = nbytes & ~(AES_BLOCK_SIZE - 1);
233 		k = cpacf_km(ctx->fc | modifier, &param,
234 			     walk.dst.virt.addr, walk.src.virt.addr, n);
235 		if (k)
236 			ret = skcipher_walk_done(&walk, nbytes - k);
237 		if (k < n) {
238 			if (__paes_convert_key(ctx))
239 				return skcipher_walk_done(&walk, -EIO);
240 			spin_lock_bh(&ctx->pk_lock);
241 			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
242 			spin_unlock_bh(&ctx->pk_lock);
243 		}
244 	}
245 	return ret;
246 }
247 
248 static int ecb_paes_encrypt(struct skcipher_request *req)
249 {
250 	return ecb_paes_crypt(req, 0);
251 }
252 
253 static int ecb_paes_decrypt(struct skcipher_request *req)
254 {
255 	return ecb_paes_crypt(req, CPACF_DECRYPT);
256 }
257 
258 static struct skcipher_alg ecb_paes_alg = {
259 	.base.cra_name		=	"ecb(paes)",
260 	.base.cra_driver_name	=	"ecb-paes-s390",
261 	.base.cra_priority	=	401,	/* combo: aes + ecb + 1 */
262 	.base.cra_blocksize	=	AES_BLOCK_SIZE,
263 	.base.cra_ctxsize	=	sizeof(struct s390_paes_ctx),
264 	.base.cra_module	=	THIS_MODULE,
265 	.base.cra_list		=	LIST_HEAD_INIT(ecb_paes_alg.base.cra_list),
266 	.init			=	ecb_paes_init,
267 	.exit			=	ecb_paes_exit,
268 	.min_keysize		=	PAES_MIN_KEYSIZE,
269 	.max_keysize		=	PAES_MAX_KEYSIZE,
270 	.setkey			=	ecb_paes_set_key,
271 	.encrypt		=	ecb_paes_encrypt,
272 	.decrypt		=	ecb_paes_decrypt,
273 };
274 
275 static int cbc_paes_init(struct crypto_skcipher *tfm)
276 {
277 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
278 
279 	ctx->kb.key = NULL;
280 	spin_lock_init(&ctx->pk_lock);
281 
282 	return 0;
283 }
284 
285 static void cbc_paes_exit(struct crypto_skcipher *tfm)
286 {
287 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
288 
289 	_free_kb_keybuf(&ctx->kb);
290 }
291 
292 static inline int __cbc_paes_set_key(struct s390_paes_ctx *ctx)
293 {
294 	int rc;
295 	unsigned long fc;
296 
297 	rc = __paes_convert_key(ctx);
298 	if (rc)
299 		return rc;
300 
301 	/* Pick the correct function code based on the protected key type */
302 	fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMC_PAES_128 :
303 		(ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMC_PAES_192 :
304 		(ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KMC_PAES_256 : 0;
305 
306 	/* Check if the function code is available */
307 	ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0;
308 
309 	return ctx->fc ? 0 : -EINVAL;
310 }
311 
312 static int cbc_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
313 			    unsigned int key_len)
314 {
315 	int rc;
316 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
317 
318 	_free_kb_keybuf(&ctx->kb);
319 	rc = _key_to_kb(&ctx->kb, in_key, key_len);
320 	if (rc)
321 		return rc;
322 
323 	return __cbc_paes_set_key(ctx);
324 }
325 
326 static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier)
327 {
328 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
329 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
330 	struct skcipher_walk walk;
331 	unsigned int nbytes, n, k;
332 	int ret;
333 	struct {
334 		u8 iv[AES_BLOCK_SIZE];
335 		u8 key[MAXPROTKEYSIZE];
336 	} param;
337 
338 	ret = skcipher_walk_virt(&walk, req, false);
339 	if (ret)
340 		return ret;
341 
342 	memcpy(param.iv, walk.iv, AES_BLOCK_SIZE);
343 	spin_lock_bh(&ctx->pk_lock);
344 	memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
345 	spin_unlock_bh(&ctx->pk_lock);
346 
347 	while ((nbytes = walk.nbytes) != 0) {
348 		/* only use complete blocks */
349 		n = nbytes & ~(AES_BLOCK_SIZE - 1);
350 		k = cpacf_kmc(ctx->fc | modifier, &param,
351 			      walk.dst.virt.addr, walk.src.virt.addr, n);
352 		if (k) {
353 			memcpy(walk.iv, param.iv, AES_BLOCK_SIZE);
354 			ret = skcipher_walk_done(&walk, nbytes - k);
355 		}
356 		if (k < n) {
357 			if (__paes_convert_key(ctx))
358 				return skcipher_walk_done(&walk, -EIO);
359 			spin_lock_bh(&ctx->pk_lock);
360 			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
361 			spin_unlock_bh(&ctx->pk_lock);
362 		}
363 	}
364 	return ret;
365 }
366 
367 static int cbc_paes_encrypt(struct skcipher_request *req)
368 {
369 	return cbc_paes_crypt(req, 0);
370 }
371 
372 static int cbc_paes_decrypt(struct skcipher_request *req)
373 {
374 	return cbc_paes_crypt(req, CPACF_DECRYPT);
375 }
376 
377 static struct skcipher_alg cbc_paes_alg = {
378 	.base.cra_name		=	"cbc(paes)",
379 	.base.cra_driver_name	=	"cbc-paes-s390",
380 	.base.cra_priority	=	402,	/* ecb-paes-s390 + 1 */
381 	.base.cra_blocksize	=	AES_BLOCK_SIZE,
382 	.base.cra_ctxsize	=	sizeof(struct s390_paes_ctx),
383 	.base.cra_module	=	THIS_MODULE,
384 	.base.cra_list		=	LIST_HEAD_INIT(cbc_paes_alg.base.cra_list),
385 	.init			=	cbc_paes_init,
386 	.exit			=	cbc_paes_exit,
387 	.min_keysize		=	PAES_MIN_KEYSIZE,
388 	.max_keysize		=	PAES_MAX_KEYSIZE,
389 	.ivsize			=	AES_BLOCK_SIZE,
390 	.setkey			=	cbc_paes_set_key,
391 	.encrypt		=	cbc_paes_encrypt,
392 	.decrypt		=	cbc_paes_decrypt,
393 };
394 
395 static int xts_paes_init(struct crypto_skcipher *tfm)
396 {
397 	struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
398 
399 	ctx->kb[0].key = NULL;
400 	ctx->kb[1].key = NULL;
401 	spin_lock_init(&ctx->pk_lock);
402 
403 	return 0;
404 }
405 
406 static void xts_paes_exit(struct crypto_skcipher *tfm)
407 {
408 	struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
409 
410 	_free_kb_keybuf(&ctx->kb[0]);
411 	_free_kb_keybuf(&ctx->kb[1]);
412 }
413 
414 static inline int __xts_paes_convert_key(struct s390_pxts_ctx *ctx)
415 {
416 	struct pkey_protkey pkey0, pkey1;
417 
418 	pkey0.len = sizeof(pkey0.protkey);
419 	pkey1.len = sizeof(pkey1.protkey);
420 
421 	if (__paes_keyblob2pkey(&ctx->kb[0], &pkey0) ||
422 	    __paes_keyblob2pkey(&ctx->kb[1], &pkey1))
423 		return -EINVAL;
424 
425 	spin_lock_bh(&ctx->pk_lock);
426 	memcpy(&ctx->pk[0], &pkey0, sizeof(pkey0));
427 	memcpy(&ctx->pk[1], &pkey1, sizeof(pkey1));
428 	spin_unlock_bh(&ctx->pk_lock);
429 
430 	return 0;
431 }
432 
433 static inline int __xts_paes_set_key(struct s390_pxts_ctx *ctx)
434 {
435 	unsigned long fc;
436 
437 	if (__xts_paes_convert_key(ctx))
438 		return -EINVAL;
439 
440 	if (ctx->pk[0].type != ctx->pk[1].type)
441 		return -EINVAL;
442 
443 	/* Pick the correct function code based on the protected key type */
444 	fc = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PXTS_128 :
445 		(ctx->pk[0].type == PKEY_KEYTYPE_AES_256) ?
446 		CPACF_KM_PXTS_256 : 0;
447 
448 	/* Check if the function code is available */
449 	ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
450 
451 	return ctx->fc ? 0 : -EINVAL;
452 }
453 
454 static int xts_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
455 			    unsigned int xts_key_len)
456 {
457 	int rc;
458 	struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
459 	u8 ckey[2 * AES_MAX_KEY_SIZE];
460 	unsigned int ckey_len, key_len;
461 
462 	if (xts_key_len % 2)
463 		return -EINVAL;
464 
465 	key_len = xts_key_len / 2;
466 
467 	_free_kb_keybuf(&ctx->kb[0]);
468 	_free_kb_keybuf(&ctx->kb[1]);
469 	rc = _key_to_kb(&ctx->kb[0], in_key, key_len);
470 	if (rc)
471 		return rc;
472 	rc = _key_to_kb(&ctx->kb[1], in_key + key_len, key_len);
473 	if (rc)
474 		return rc;
475 
476 	rc = __xts_paes_set_key(ctx);
477 	if (rc)
478 		return rc;
479 
480 	/*
481 	 * xts_verify_key verifies the key length is not odd and makes
482 	 * sure that the two keys are not the same. This can be done
483 	 * on the two protected keys as well
484 	 */
485 	ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ?
486 		AES_KEYSIZE_128 : AES_KEYSIZE_256;
487 	memcpy(ckey, ctx->pk[0].protkey, ckey_len);
488 	memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len);
489 	return xts_verify_key(tfm, ckey, 2*ckey_len);
490 }
491 
492 static int xts_paes_crypt(struct skcipher_request *req, unsigned long modifier)
493 {
494 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
495 	struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
496 	struct skcipher_walk walk;
497 	unsigned int keylen, offset, nbytes, n, k;
498 	int ret;
499 	struct {
500 		u8 key[MAXPROTKEYSIZE];	/* key + verification pattern */
501 		u8 tweak[16];
502 		u8 block[16];
503 		u8 bit[16];
504 		u8 xts[16];
505 	} pcc_param;
506 	struct {
507 		u8 key[MAXPROTKEYSIZE];	/* key + verification pattern */
508 		u8 init[16];
509 	} xts_param;
510 
511 	ret = skcipher_walk_virt(&walk, req, false);
512 	if (ret)
513 		return ret;
514 
515 	keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64;
516 	offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0;
517 
518 	memset(&pcc_param, 0, sizeof(pcc_param));
519 	memcpy(pcc_param.tweak, walk.iv, sizeof(pcc_param.tweak));
520 	spin_lock_bh(&ctx->pk_lock);
521 	memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen);
522 	memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen);
523 	spin_unlock_bh(&ctx->pk_lock);
524 	cpacf_pcc(ctx->fc, pcc_param.key + offset);
525 	memcpy(xts_param.init, pcc_param.xts, 16);
526 
527 	while ((nbytes = walk.nbytes) != 0) {
528 		/* only use complete blocks */
529 		n = nbytes & ~(AES_BLOCK_SIZE - 1);
530 		k = cpacf_km(ctx->fc | modifier, xts_param.key + offset,
531 			     walk.dst.virt.addr, walk.src.virt.addr, n);
532 		if (k)
533 			ret = skcipher_walk_done(&walk, nbytes - k);
534 		if (k < n) {
535 			if (__xts_paes_convert_key(ctx))
536 				return skcipher_walk_done(&walk, -EIO);
537 			spin_lock_bh(&ctx->pk_lock);
538 			memcpy(xts_param.key + offset,
539 			       ctx->pk[0].protkey, keylen);
540 			spin_unlock_bh(&ctx->pk_lock);
541 		}
542 	}
543 
544 	return ret;
545 }
546 
547 static int xts_paes_encrypt(struct skcipher_request *req)
548 {
549 	return xts_paes_crypt(req, 0);
550 }
551 
552 static int xts_paes_decrypt(struct skcipher_request *req)
553 {
554 	return xts_paes_crypt(req, CPACF_DECRYPT);
555 }
556 
557 static struct skcipher_alg xts_paes_alg = {
558 	.base.cra_name		=	"xts(paes)",
559 	.base.cra_driver_name	=	"xts-paes-s390",
560 	.base.cra_priority	=	402,	/* ecb-paes-s390 + 1 */
561 	.base.cra_blocksize	=	AES_BLOCK_SIZE,
562 	.base.cra_ctxsize	=	sizeof(struct s390_pxts_ctx),
563 	.base.cra_module	=	THIS_MODULE,
564 	.base.cra_list		=	LIST_HEAD_INIT(xts_paes_alg.base.cra_list),
565 	.init			=	xts_paes_init,
566 	.exit			=	xts_paes_exit,
567 	.min_keysize		=	2 * PAES_MIN_KEYSIZE,
568 	.max_keysize		=	2 * PAES_MAX_KEYSIZE,
569 	.ivsize			=	AES_BLOCK_SIZE,
570 	.setkey			=	xts_paes_set_key,
571 	.encrypt		=	xts_paes_encrypt,
572 	.decrypt		=	xts_paes_decrypt,
573 };
574 
575 static int ctr_paes_init(struct crypto_skcipher *tfm)
576 {
577 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
578 
579 	ctx->kb.key = NULL;
580 	spin_lock_init(&ctx->pk_lock);
581 
582 	return 0;
583 }
584 
585 static void ctr_paes_exit(struct crypto_skcipher *tfm)
586 {
587 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
588 
589 	_free_kb_keybuf(&ctx->kb);
590 }
591 
592 static inline int __ctr_paes_set_key(struct s390_paes_ctx *ctx)
593 {
594 	int rc;
595 	unsigned long fc;
596 
597 	rc = __paes_convert_key(ctx);
598 	if (rc)
599 		return rc;
600 
601 	/* Pick the correct function code based on the protected key type */
602 	fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMCTR_PAES_128 :
603 		(ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMCTR_PAES_192 :
604 		(ctx->pk.type == PKEY_KEYTYPE_AES_256) ?
605 		CPACF_KMCTR_PAES_256 : 0;
606 
607 	/* Check if the function code is available */
608 	ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0;
609 
610 	return ctx->fc ? 0 : -EINVAL;
611 }
612 
613 static int ctr_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
614 			    unsigned int key_len)
615 {
616 	int rc;
617 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
618 
619 	_free_kb_keybuf(&ctx->kb);
620 	rc = _key_to_kb(&ctx->kb, in_key, key_len);
621 	if (rc)
622 		return rc;
623 
624 	return __ctr_paes_set_key(ctx);
625 }
626 
627 static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes)
628 {
629 	unsigned int i, n;
630 
631 	/* only use complete blocks, max. PAGE_SIZE */
632 	memcpy(ctrptr, iv, AES_BLOCK_SIZE);
633 	n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1);
634 	for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) {
635 		memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE);
636 		crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE);
637 		ctrptr += AES_BLOCK_SIZE;
638 	}
639 	return n;
640 }
641 
642 static int ctr_paes_crypt(struct skcipher_request *req)
643 {
644 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
645 	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
646 	u8 buf[AES_BLOCK_SIZE], *ctrptr;
647 	struct skcipher_walk walk;
648 	unsigned int nbytes, n, k;
649 	int ret, locked;
650 	struct {
651 		u8 key[MAXPROTKEYSIZE];
652 	} param;
653 
654 	ret = skcipher_walk_virt(&walk, req, false);
655 	if (ret)
656 		return ret;
657 
658 	spin_lock_bh(&ctx->pk_lock);
659 	memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
660 	spin_unlock_bh(&ctx->pk_lock);
661 
662 	locked = mutex_trylock(&ctrblk_lock);
663 
664 	while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
665 		n = AES_BLOCK_SIZE;
666 		if (nbytes >= 2*AES_BLOCK_SIZE && locked)
667 			n = __ctrblk_init(ctrblk, walk.iv, nbytes);
668 		ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv;
669 		k = cpacf_kmctr(ctx->fc, &param, walk.dst.virt.addr,
670 				walk.src.virt.addr, n, ctrptr);
671 		if (k) {
672 			if (ctrptr == ctrblk)
673 				memcpy(walk.iv, ctrptr + k - AES_BLOCK_SIZE,
674 				       AES_BLOCK_SIZE);
675 			crypto_inc(walk.iv, AES_BLOCK_SIZE);
676 			ret = skcipher_walk_done(&walk, nbytes - k);
677 		}
678 		if (k < n) {
679 			if (__paes_convert_key(ctx)) {
680 				if (locked)
681 					mutex_unlock(&ctrblk_lock);
682 				return skcipher_walk_done(&walk, -EIO);
683 			}
684 			spin_lock_bh(&ctx->pk_lock);
685 			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
686 			spin_unlock_bh(&ctx->pk_lock);
687 		}
688 	}
689 	if (locked)
690 		mutex_unlock(&ctrblk_lock);
691 	/*
692 	 * final block may be < AES_BLOCK_SIZE, copy only nbytes
693 	 */
694 	if (nbytes) {
695 		memset(buf, 0, AES_BLOCK_SIZE);
696 		memcpy(buf, walk.src.virt.addr, nbytes);
697 		while (1) {
698 			if (cpacf_kmctr(ctx->fc, &param, buf,
699 					buf, AES_BLOCK_SIZE,
700 					walk.iv) == AES_BLOCK_SIZE)
701 				break;
702 			if (__paes_convert_key(ctx))
703 				return skcipher_walk_done(&walk, -EIO);
704 			spin_lock_bh(&ctx->pk_lock);
705 			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);
706 			spin_unlock_bh(&ctx->pk_lock);
707 		}
708 		memcpy(walk.dst.virt.addr, buf, nbytes);
709 		crypto_inc(walk.iv, AES_BLOCK_SIZE);
710 		ret = skcipher_walk_done(&walk, nbytes);
711 	}
712 
713 	return ret;
714 }
715 
716 static struct skcipher_alg ctr_paes_alg = {
717 	.base.cra_name		=	"ctr(paes)",
718 	.base.cra_driver_name	=	"ctr-paes-s390",
719 	.base.cra_priority	=	402,	/* ecb-paes-s390 + 1 */
720 	.base.cra_blocksize	=	1,
721 	.base.cra_ctxsize	=	sizeof(struct s390_paes_ctx),
722 	.base.cra_module	=	THIS_MODULE,
723 	.base.cra_list		=	LIST_HEAD_INIT(ctr_paes_alg.base.cra_list),
724 	.init			=	ctr_paes_init,
725 	.exit			=	ctr_paes_exit,
726 	.min_keysize		=	PAES_MIN_KEYSIZE,
727 	.max_keysize		=	PAES_MAX_KEYSIZE,
728 	.ivsize			=	AES_BLOCK_SIZE,
729 	.setkey			=	ctr_paes_set_key,
730 	.encrypt		=	ctr_paes_crypt,
731 	.decrypt		=	ctr_paes_crypt,
732 	.chunksize		=	AES_BLOCK_SIZE,
733 };
734 
735 static inline void __crypto_unregister_skcipher(struct skcipher_alg *alg)
736 {
737 	if (!list_empty(&alg->base.cra_list))
738 		crypto_unregister_skcipher(alg);
739 }
740 
741 static void paes_s390_fini(void)
742 {
743 	__crypto_unregister_skcipher(&ctr_paes_alg);
744 	__crypto_unregister_skcipher(&xts_paes_alg);
745 	__crypto_unregister_skcipher(&cbc_paes_alg);
746 	__crypto_unregister_skcipher(&ecb_paes_alg);
747 	if (ctrblk)
748 		free_page((unsigned long) ctrblk);
749 }
750 
751 static int __init paes_s390_init(void)
752 {
753 	int ret;
754 
755 	/* Query available functions for KM, KMC and KMCTR */
756 	cpacf_query(CPACF_KM, &km_functions);
757 	cpacf_query(CPACF_KMC, &kmc_functions);
758 	cpacf_query(CPACF_KMCTR, &kmctr_functions);
759 
760 	if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) ||
761 	    cpacf_test_func(&km_functions, CPACF_KM_PAES_192) ||
762 	    cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) {
763 		ret = crypto_register_skcipher(&ecb_paes_alg);
764 		if (ret)
765 			goto out_err;
766 	}
767 
768 	if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) ||
769 	    cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) ||
770 	    cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) {
771 		ret = crypto_register_skcipher(&cbc_paes_alg);
772 		if (ret)
773 			goto out_err;
774 	}
775 
776 	if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) ||
777 	    cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) {
778 		ret = crypto_register_skcipher(&xts_paes_alg);
779 		if (ret)
780 			goto out_err;
781 	}
782 
783 	if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) ||
784 	    cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) ||
785 	    cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) {
786 		ctrblk = (u8 *) __get_free_page(GFP_KERNEL);
787 		if (!ctrblk) {
788 			ret = -ENOMEM;
789 			goto out_err;
790 		}
791 		ret = crypto_register_skcipher(&ctr_paes_alg);
792 		if (ret)
793 			goto out_err;
794 	}
795 
796 	return 0;
797 out_err:
798 	paes_s390_fini();
799 	return ret;
800 }
801 
802 module_init(paes_s390_init);
803 module_exit(paes_s390_fini);
804 
805 MODULE_ALIAS_CRYPTO("paes");
806 
807 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys");
808 MODULE_LICENSE("GPL");
809