1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Cryptographic API.
4 *
5 * s390 implementation of the AES Cipher Algorithm with protected keys.
6 *
7 * s390 Version:
8 * Copyright IBM Corp. 2017, 2025
9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
10 * Harald Freudenberger <freude@de.ibm.com>
11 */
12
13 #define pr_fmt(fmt) "paes_s390: " fmt
14
15 #include <linux/atomic.h>
16 #include <linux/cpufeature.h>
17 #include <linux/delay.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/miscdevice.h>
21 #include <linux/module.h>
22 #include <linux/semaphore.h>
23 #include <linux/spinlock.h>
24 #include <crypto/aes.h>
25 #include <crypto/algapi.h>
26 #include <crypto/engine.h>
27 #include <crypto/internal/skcipher.h>
28 #include <crypto/xts.h>
29 #include <asm/cpacf.h>
30 #include <asm/pkey.h>
31
32 /*
33 * Key blobs smaller/bigger than these defines are rejected
34 * by the common code even before the individual setkey function
35 * is called. As paes can handle different kinds of key blobs
36 * and padding is also possible, the limits need to be generous.
37 */
38 #define PAES_MIN_KEYSIZE 16
39 #define PAES_MAX_KEYSIZE MAXEP11AESKEYBLOBSIZE
40 #define PAES_256_PROTKEY_SIZE (32 + 32) /* key + verification pattern */
41 #define PXTS_256_PROTKEY_SIZE (32 + 32 + 32) /* k1 + k2 + verification pattern */
42
43 static bool pkey_clrkey_allowed;
44 module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444);
45 MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)");
46
47 static u8 *ctrblk;
48 static DEFINE_SEMAPHORE(ctrblk_sem, 1);
49
50 static cpacf_mask_t km_functions, kmc_functions, kmctr_functions;
51
52 static struct crypto_engine *paes_crypto_engine;
53 #define MAX_QLEN 10
54
55 /*
56 * protected key specific stuff
57 */
58
59 struct paes_protkey {
60 u32 type;
61 u32 len;
62 u8 protkey[PXTS_256_PROTKEY_SIZE];
63 };
64
65 #define PK_STATE_NO_KEY 0
66 #define PK_STATE_CONVERT_IN_PROGRESS 1
67 #define PK_STATE_VALID 2
68
69 struct s390_paes_ctx {
70 /* source key material used to derive a protected key from */
71 u8 keybuf[PAES_MAX_KEYSIZE];
72 unsigned int keylen;
73
74 /* cpacf function code to use with this protected key type */
75 long fc;
76
77 /* nr of requests enqueued via crypto engine which use this tfm ctx */
78 atomic_t via_engine_ctr;
79
80 /* spinlock to atomic read/update all the following fields */
81 spinlock_t pk_lock;
82
83 /* see PK_STATE* defines above, < 0 holds convert failure rc */
84 int pk_state;
85 /* if state is valid, pk holds the protected key */
86 struct paes_protkey pk;
87 };
88
89 struct s390_pxts_ctx {
90 /* source key material used to derive a protected key from */
91 u8 keybuf[2 * PAES_MAX_KEYSIZE];
92 unsigned int keylen;
93
94 /* cpacf function code to use with this protected key type */
95 long fc;
96
97 /* nr of requests enqueued via crypto engine which use this tfm ctx */
98 atomic_t via_engine_ctr;
99
100 /* spinlock to atomic read/update all the following fields */
101 spinlock_t pk_lock;
102
103 /* see PK_STATE* defines above, < 0 holds convert failure rc */
104 int pk_state;
105 /* if state is valid, pk[] hold(s) the protected key(s) */
106 struct paes_protkey pk[2];
107 };
108
109 /*
110 * make_clrkey_token() - wrap the raw key ck with pkey clearkey token
111 * information.
112 * @returns the size of the clearkey token
113 */
make_clrkey_token(const u8 * ck,size_t cklen,u8 * dest)114 static inline u32 make_clrkey_token(const u8 *ck, size_t cklen, u8 *dest)
115 {
116 struct clrkey_token {
117 u8 type;
118 u8 res0[3];
119 u8 version;
120 u8 res1[3];
121 u32 keytype;
122 u32 len;
123 u8 key[];
124 } __packed *token = (struct clrkey_token *)dest;
125
126 token->type = 0x00;
127 token->version = 0x02;
128 token->keytype = (cklen - 8) >> 3;
129 token->len = cklen;
130 memcpy(token->key, ck, cklen);
131
132 return sizeof(*token) + cklen;
133 }
134
135 /*
136 * paes_ctx_setkey() - Set key value into context, maybe construct
137 * a clear key token digestible by pkey from a clear key value.
138 */
paes_ctx_setkey(struct s390_paes_ctx * ctx,const u8 * key,unsigned int keylen)139 static inline int paes_ctx_setkey(struct s390_paes_ctx *ctx,
140 const u8 *key, unsigned int keylen)
141 {
142 if (keylen > sizeof(ctx->keybuf))
143 return -EINVAL;
144
145 switch (keylen) {
146 case 16:
147 case 24:
148 case 32:
149 /* clear key value, prepare pkey clear key token in keybuf */
150 memset(ctx->keybuf, 0, sizeof(ctx->keybuf));
151 ctx->keylen = make_clrkey_token(key, keylen, ctx->keybuf);
152 break;
153 default:
154 /* other key material, let pkey handle this */
155 memcpy(ctx->keybuf, key, keylen);
156 ctx->keylen = keylen;
157 break;
158 }
159
160 return 0;
161 }
162
163 /*
164 * pxts_ctx_setkey() - Set key value into context, maybe construct
165 * a clear key token digestible by pkey from a clear key value.
166 */
pxts_ctx_setkey(struct s390_pxts_ctx * ctx,const u8 * key,unsigned int keylen)167 static inline int pxts_ctx_setkey(struct s390_pxts_ctx *ctx,
168 const u8 *key, unsigned int keylen)
169 {
170 size_t cklen = keylen / 2;
171
172 if (keylen > sizeof(ctx->keybuf))
173 return -EINVAL;
174
175 switch (keylen) {
176 case 32:
177 case 64:
178 /* clear key value, prepare pkey clear key tokens in keybuf */
179 memset(ctx->keybuf, 0, sizeof(ctx->keybuf));
180 ctx->keylen = make_clrkey_token(key, cklen, ctx->keybuf);
181 ctx->keylen += make_clrkey_token(key + cklen, cklen,
182 ctx->keybuf + ctx->keylen);
183 break;
184 default:
185 /* other key material, let pkey handle this */
186 memcpy(ctx->keybuf, key, keylen);
187 ctx->keylen = keylen;
188 break;
189 }
190
191 return 0;
192 }
193
194 /*
195 * Convert the raw key material into a protected key via PKEY api.
196 * This function may sleep - don't call in non-sleeping context.
197 */
convert_key(const u8 * key,unsigned int keylen,struct paes_protkey * pk,bool tested)198 static inline int convert_key(const u8 *key, unsigned int keylen,
199 struct paes_protkey *pk, bool tested)
200 {
201 u32 xflags = PKEY_XFLAG_NOMEMALLOC;
202 int rc, i;
203
204 if (tested && !pkey_clrkey_allowed)
205 xflags |= PKEY_XFLAG_NOCLEARKEY;
206
207 pk->len = sizeof(pk->protkey);
208
209 /*
210 * In case of a busy card retry with increasing delay
211 * of 200, 400, 800 and 1600 ms - in total 3 s.
212 */
213 for (rc = -EIO, i = 0; rc && i < 5; i++) {
214 if (rc == -EBUSY && msleep_interruptible((1 << i) * 100)) {
215 rc = -EINTR;
216 goto out;
217 }
218 rc = pkey_key2protkey(key, keylen,
219 pk->protkey, &pk->len, &pk->type,
220 xflags);
221 }
222
223 /* But finally map -EBUSY to -EIO to indicate an IO failure */
224 if (rc == -EBUSY)
225 rc = -EIO;
226
227 out:
228 pr_debug("rc=%d\n", rc);
229 return rc;
230 }
231
232 /*
233 * (Re-)Convert the raw key material from the ctx into a protected key
234 * via convert_key() function. Update the pk_state, pk_type, pk_len
235 * and the protected key in the tfm context.
236 * Please note this function may be invoked concurrently with the very
237 * same tfm context. The pk_lock spinlock in the context ensures an
238 * atomic update of the pk and the pk state but does not guarantee any
239 * order of update. So a fresh converted valid protected key may get
240 * updated with an 'old' expired key value. As the cpacf instructions
241 * detect this, refuse to operate with an invalid key and the calling
242 * code triggers a (re-)conversion this does no harm. This may lead to
243 * unnecessary additional conversion but never to invalid data on en-
244 * or decrypt operations.
245 */
paes_convert_key(struct s390_paes_ctx * ctx,bool tested)246 static int paes_convert_key(struct s390_paes_ctx *ctx, bool tested)
247 {
248 struct paes_protkey pk;
249 int rc;
250
251 spin_lock_bh(&ctx->pk_lock);
252 ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS;
253 spin_unlock_bh(&ctx->pk_lock);
254
255 rc = convert_key(ctx->keybuf, ctx->keylen, &pk, tested);
256
257 /* update context */
258 spin_lock_bh(&ctx->pk_lock);
259 if (rc) {
260 ctx->pk_state = rc;
261 } else {
262 ctx->pk_state = PK_STATE_VALID;
263 ctx->pk = pk;
264 }
265 spin_unlock_bh(&ctx->pk_lock);
266
267 memzero_explicit(&pk, sizeof(pk));
268 pr_debug("rc=%d\n", rc);
269 return rc;
270 }
271
272 /*
273 * (Re-)Convert the raw xts key material from the ctx into a
274 * protected key via convert_key() function. Update the pk_state,
275 * pk_type, pk_len and the protected key in the tfm context.
276 * See also comments on function paes_convert_key.
277 */
pxts_convert_key(struct s390_pxts_ctx * ctx,bool tested)278 static int pxts_convert_key(struct s390_pxts_ctx *ctx, bool tested)
279 {
280 struct paes_protkey pk0, pk1;
281 size_t split_keylen;
282 int rc;
283
284 spin_lock_bh(&ctx->pk_lock);
285 ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS;
286 spin_unlock_bh(&ctx->pk_lock);
287
288 rc = convert_key(ctx->keybuf, ctx->keylen, &pk0, tested);
289 if (rc)
290 goto out;
291
292 switch (pk0.type) {
293 case PKEY_KEYTYPE_AES_128:
294 case PKEY_KEYTYPE_AES_256:
295 /* second keytoken required */
296 if (ctx->keylen % 2) {
297 rc = -EINVAL;
298 goto out;
299 }
300 split_keylen = ctx->keylen / 2;
301 rc = convert_key(ctx->keybuf + split_keylen,
302 split_keylen, &pk1, tested);
303 if (rc)
304 goto out;
305 if (pk0.type != pk1.type) {
306 rc = -EINVAL;
307 goto out;
308 }
309 break;
310 case PKEY_KEYTYPE_AES_XTS_128:
311 case PKEY_KEYTYPE_AES_XTS_256:
312 /* single key */
313 pk1.type = 0;
314 break;
315 default:
316 /* unsupported protected keytype */
317 rc = -EINVAL;
318 goto out;
319 }
320
321 out:
322 /* update context */
323 spin_lock_bh(&ctx->pk_lock);
324 if (rc) {
325 ctx->pk_state = rc;
326 } else {
327 ctx->pk_state = PK_STATE_VALID;
328 ctx->pk[0] = pk0;
329 ctx->pk[1] = pk1;
330 }
331 spin_unlock_bh(&ctx->pk_lock);
332
333 memzero_explicit(&pk0, sizeof(pk0));
334 memzero_explicit(&pk1, sizeof(pk1));
335 pr_debug("rc=%d\n", rc);
336 return rc;
337 }
338
339 /*
340 * PAES ECB implementation
341 */
342
343 struct ecb_param {
344 u8 key[PAES_256_PROTKEY_SIZE];
345 } __packed;
346
347 struct s390_pecb_req_ctx {
348 unsigned long modifier;
349 struct skcipher_walk walk;
350 bool param_init_done;
351 struct ecb_param param;
352 };
353
ecb_paes_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)354 static int ecb_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
355 unsigned int key_len)
356 {
357 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
358 bool tested = crypto_skcipher_tested(tfm);
359 long fc;
360 int rc;
361
362 /* set raw key into context */
363 rc = paes_ctx_setkey(ctx, in_key, key_len);
364 if (rc)
365 goto out;
366
367 /* convert key into protected key */
368 rc = paes_convert_key(ctx, tested);
369 if (rc)
370 goto out;
371
372 /* Pick the correct function code based on the protected key type */
373 switch (ctx->pk.type) {
374 case PKEY_KEYTYPE_AES_128:
375 fc = CPACF_KM_PAES_128;
376 break;
377 case PKEY_KEYTYPE_AES_192:
378 fc = CPACF_KM_PAES_192;
379 break;
380 case PKEY_KEYTYPE_AES_256:
381 fc = CPACF_KM_PAES_256;
382 break;
383 default:
384 fc = 0;
385 break;
386 }
387 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
388
389 rc = fc ? 0 : -EINVAL;
390
391 out:
392 pr_debug("rc=%d\n", rc);
393 return rc;
394 }
395
ecb_paes_do_crypt(struct s390_paes_ctx * ctx,struct s390_pecb_req_ctx * req_ctx,bool tested,bool maysleep)396 static int ecb_paes_do_crypt(struct s390_paes_ctx *ctx,
397 struct s390_pecb_req_ctx *req_ctx,
398 bool tested, bool maysleep)
399 {
400 struct ecb_param *param = &req_ctx->param;
401 struct skcipher_walk *walk = &req_ctx->walk;
402 unsigned int nbytes, n, k;
403 int pk_state, rc = 0;
404
405 if (!req_ctx->param_init_done) {
406 /* fetch and check protected key state */
407 spin_lock_bh(&ctx->pk_lock);
408 pk_state = ctx->pk_state;
409 switch (pk_state) {
410 case PK_STATE_NO_KEY:
411 rc = -ENOKEY;
412 break;
413 case PK_STATE_CONVERT_IN_PROGRESS:
414 rc = -EKEYEXPIRED;
415 break;
416 case PK_STATE_VALID:
417 memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
418 req_ctx->param_init_done = true;
419 break;
420 default:
421 rc = pk_state < 0 ? pk_state : -EIO;
422 break;
423 }
424 spin_unlock_bh(&ctx->pk_lock);
425 }
426 if (rc)
427 goto out;
428
429 /*
430 * Note that in case of partial processing or failure the walk
431 * is NOT unmapped here. So a follow up task may reuse the walk
432 * or in case of unrecoverable failure needs to unmap it.
433 */
434 while ((nbytes = walk->nbytes) != 0) {
435 /* only use complete blocks */
436 n = nbytes & ~(AES_BLOCK_SIZE - 1);
437 k = cpacf_km(ctx->fc | req_ctx->modifier, param,
438 walk->dst.virt.addr, walk->src.virt.addr, n);
439 if (k) {
440 rc = skcipher_walk_done(walk, nbytes - k);
441 if (rc)
442 goto out;
443 }
444 if (k < n) {
445 if (!maysleep) {
446 rc = -EKEYEXPIRED;
447 goto out;
448 }
449 rc = paes_convert_key(ctx, tested);
450 if (rc)
451 goto out;
452 spin_lock_bh(&ctx->pk_lock);
453 memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
454 spin_unlock_bh(&ctx->pk_lock);
455 }
456 }
457
458 out:
459 pr_debug("rc=%d\n", rc);
460 return rc;
461 }
462
ecb_paes_crypt(struct skcipher_request * req,unsigned long modifier)463 static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier)
464 {
465 struct s390_pecb_req_ctx *req_ctx = skcipher_request_ctx(req);
466 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
467 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
468 struct skcipher_walk *walk = &req_ctx->walk;
469 bool tested = crypto_skcipher_tested(tfm);
470 bool cleanup = true;
471 int rc;
472
473 /*
474 * Attempt synchronous encryption first. If it fails, schedule the request
475 * asynchronously via the crypto engine. To preserve execution order,
476 * once a request is queued to the engine, further requests using the same
477 * tfm will also be routed through the engine.
478 */
479
480 rc = skcipher_walk_virt(walk, req, false);
481 if (rc)
482 goto out;
483
484 req_ctx->modifier = modifier;
485 req_ctx->param_init_done = false;
486
487 /* Try synchronous operation if no active engine usage */
488 if (!atomic_read(&ctx->via_engine_ctr)) {
489 rc = ecb_paes_do_crypt(ctx, req_ctx, tested, false);
490 if (rc == 0)
491 goto out;
492 }
493
494 /*
495 * If sync operation failed or key expired or there are already
496 * requests enqueued via engine, fallback to async. Mark tfm as
497 * using engine to serialize requests.
498 */
499 if (rc == 0 || rc == -EKEYEXPIRED) {
500 atomic_inc(&ctx->via_engine_ctr);
501 rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
502 if (rc == -EINPROGRESS || rc == -EBUSY)
503 cleanup = false;
504 else
505 atomic_dec(&ctx->via_engine_ctr);
506 }
507
508 if (cleanup && walk->nbytes)
509 skcipher_walk_done(walk, rc);
510
511 out:
512 if (cleanup)
513 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
514 pr_debug("rc=%d\n", rc);
515 return rc;
516 }
517
ecb_paes_encrypt(struct skcipher_request * req)518 static int ecb_paes_encrypt(struct skcipher_request *req)
519 {
520 return ecb_paes_crypt(req, 0);
521 }
522
ecb_paes_decrypt(struct skcipher_request * req)523 static int ecb_paes_decrypt(struct skcipher_request *req)
524 {
525 return ecb_paes_crypt(req, CPACF_DECRYPT);
526 }
527
ecb_paes_init(struct crypto_skcipher * tfm)528 static int ecb_paes_init(struct crypto_skcipher *tfm)
529 {
530 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
531
532 memset(ctx, 0, sizeof(*ctx));
533 spin_lock_init(&ctx->pk_lock);
534
535 crypto_skcipher_set_reqsize(tfm, sizeof(struct s390_pecb_req_ctx));
536
537 return 0;
538 }
539
ecb_paes_exit(struct crypto_skcipher * tfm)540 static void ecb_paes_exit(struct crypto_skcipher *tfm)
541 {
542 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
543
544 memzero_explicit(ctx, sizeof(*ctx));
545 }
546
ecb_paes_do_one_request(struct crypto_engine * engine,void * areq)547 static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq)
548 {
549 struct skcipher_request *req = skcipher_request_cast(areq);
550 struct s390_pecb_req_ctx *req_ctx = skcipher_request_ctx(req);
551 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
552 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
553 struct skcipher_walk *walk = &req_ctx->walk;
554 bool tested = crypto_skcipher_tested(tfm);
555 int rc;
556
557 /* walk has already been prepared */
558
559 rc = ecb_paes_do_crypt(ctx, req_ctx, tested, true);
560 if (rc == -EKEYEXPIRED) {
561 return pkey_handle_expired();
562 } else if (rc && walk->nbytes) {
563 skcipher_walk_done(walk, rc);
564 }
565
566 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
567 pr_debug("request complete with rc=%d\n", rc);
568 local_bh_disable();
569 atomic_dec(&ctx->via_engine_ctr);
570 crypto_finalize_skcipher_request(engine, req, rc);
571 local_bh_enable();
572 return 0;
573 }
574
575 static struct skcipher_engine_alg ecb_paes_alg = {
576 .base = {
577 .base.cra_name = "ecb(paes)",
578 .base.cra_driver_name = "ecb-paes-s390",
579 .base.cra_priority = 401, /* combo: aes + ecb + 1 */
580 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
581 .base.cra_blocksize = AES_BLOCK_SIZE,
582 .base.cra_ctxsize = sizeof(struct s390_paes_ctx),
583 .base.cra_module = THIS_MODULE,
584 .base.cra_list = LIST_HEAD_INIT(ecb_paes_alg.base.base.cra_list),
585 .init = ecb_paes_init,
586 .exit = ecb_paes_exit,
587 .min_keysize = PAES_MIN_KEYSIZE,
588 .max_keysize = PAES_MAX_KEYSIZE,
589 .setkey = ecb_paes_setkey,
590 .encrypt = ecb_paes_encrypt,
591 .decrypt = ecb_paes_decrypt,
592 },
593 .op = {
594 .do_one_request = ecb_paes_do_one_request,
595 },
596 };
597
598 /*
599 * PAES CBC implementation
600 */
601
602 struct cbc_param {
603 u8 iv[AES_BLOCK_SIZE];
604 u8 key[PAES_256_PROTKEY_SIZE];
605 } __packed;
606
607 struct s390_pcbc_req_ctx {
608 unsigned long modifier;
609 struct skcipher_walk walk;
610 bool param_init_done;
611 struct cbc_param param;
612 };
613
cbc_paes_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)614 static int cbc_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
615 unsigned int key_len)
616 {
617 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
618 bool tested = crypto_skcipher_tested(tfm);
619 long fc;
620 int rc;
621
622 /* set raw key into context */
623 rc = paes_ctx_setkey(ctx, in_key, key_len);
624 if (rc)
625 goto out;
626
627 /* convert raw key into protected key */
628 rc = paes_convert_key(ctx, tested);
629 if (rc)
630 goto out;
631
632 /* Pick the correct function code based on the protected key type */
633 switch (ctx->pk.type) {
634 case PKEY_KEYTYPE_AES_128:
635 fc = CPACF_KMC_PAES_128;
636 break;
637 case PKEY_KEYTYPE_AES_192:
638 fc = CPACF_KMC_PAES_192;
639 break;
640 case PKEY_KEYTYPE_AES_256:
641 fc = CPACF_KMC_PAES_256;
642 break;
643 default:
644 fc = 0;
645 break;
646 }
647 ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0;
648
649 rc = fc ? 0 : -EINVAL;
650
651 out:
652 pr_debug("rc=%d\n", rc);
653 return rc;
654 }
655
cbc_paes_do_crypt(struct s390_paes_ctx * ctx,struct s390_pcbc_req_ctx * req_ctx,bool tested,bool maysleep)656 static int cbc_paes_do_crypt(struct s390_paes_ctx *ctx,
657 struct s390_pcbc_req_ctx *req_ctx,
658 bool tested, bool maysleep)
659 {
660 struct cbc_param *param = &req_ctx->param;
661 struct skcipher_walk *walk = &req_ctx->walk;
662 unsigned int nbytes, n, k;
663 int pk_state, rc = 0;
664
665 if (!req_ctx->param_init_done) {
666 /* fetch and check protected key state */
667 spin_lock_bh(&ctx->pk_lock);
668 pk_state = ctx->pk_state;
669 switch (pk_state) {
670 case PK_STATE_NO_KEY:
671 rc = -ENOKEY;
672 break;
673 case PK_STATE_CONVERT_IN_PROGRESS:
674 rc = -EKEYEXPIRED;
675 break;
676 case PK_STATE_VALID:
677 memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
678 req_ctx->param_init_done = true;
679 break;
680 default:
681 rc = pk_state < 0 ? pk_state : -EIO;
682 break;
683 }
684 spin_unlock_bh(&ctx->pk_lock);
685 }
686 if (rc)
687 goto out;
688
689 memcpy(param->iv, walk->iv, AES_BLOCK_SIZE);
690
691 /*
692 * Note that in case of partial processing or failure the walk
693 * is NOT unmapped here. So a follow up task may reuse the walk
694 * or in case of unrecoverable failure needs to unmap it.
695 */
696 while ((nbytes = walk->nbytes) != 0) {
697 /* only use complete blocks */
698 n = nbytes & ~(AES_BLOCK_SIZE - 1);
699 k = cpacf_kmc(ctx->fc | req_ctx->modifier, param,
700 walk->dst.virt.addr, walk->src.virt.addr, n);
701 if (k) {
702 memcpy(walk->iv, param->iv, AES_BLOCK_SIZE);
703 rc = skcipher_walk_done(walk, nbytes - k);
704 if (rc)
705 goto out;
706 }
707 if (k < n) {
708 if (!maysleep) {
709 rc = -EKEYEXPIRED;
710 goto out;
711 }
712 rc = paes_convert_key(ctx, tested);
713 if (rc)
714 goto out;
715 spin_lock_bh(&ctx->pk_lock);
716 memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
717 spin_unlock_bh(&ctx->pk_lock);
718 }
719 }
720
721 out:
722 pr_debug("rc=%d\n", rc);
723 return rc;
724 }
725
cbc_paes_crypt(struct skcipher_request * req,unsigned long modifier)726 static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier)
727 {
728 struct s390_pcbc_req_ctx *req_ctx = skcipher_request_ctx(req);
729 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
730 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
731 struct skcipher_walk *walk = &req_ctx->walk;
732 bool tested = crypto_skcipher_tested(tfm);
733 bool cleanup = true;
734 int rc;
735
736 /*
737 * Attempt synchronous encryption first. If it fails, schedule the request
738 * asynchronously via the crypto engine. To preserve execution order,
739 * once a request is queued to the engine, further requests using the same
740 * tfm will also be routed through the engine.
741 */
742
743 rc = skcipher_walk_virt(walk, req, false);
744 if (rc)
745 goto out;
746
747 req_ctx->modifier = modifier;
748 req_ctx->param_init_done = false;
749
750 /* Try synchronous operation if no active engine usage */
751 if (!atomic_read(&ctx->via_engine_ctr)) {
752 rc = cbc_paes_do_crypt(ctx, req_ctx, tested, false);
753 if (rc == 0)
754 goto out;
755 }
756
757 /*
758 * If sync operation failed or key expired or there are already
759 * requests enqueued via engine, fallback to async. Mark tfm as
760 * using engine to serialize requests.
761 */
762 if (rc == 0 || rc == -EKEYEXPIRED) {
763 atomic_inc(&ctx->via_engine_ctr);
764 rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
765 if (rc == -EINPROGRESS || rc == -EBUSY)
766 cleanup = false;
767 else
768 atomic_dec(&ctx->via_engine_ctr);
769 }
770
771 if (cleanup && walk->nbytes)
772 skcipher_walk_done(walk, rc);
773
774 out:
775 if (cleanup)
776 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
777 pr_debug("rc=%d\n", rc);
778 return rc;
779 }
780
cbc_paes_encrypt(struct skcipher_request * req)781 static int cbc_paes_encrypt(struct skcipher_request *req)
782 {
783 return cbc_paes_crypt(req, 0);
784 }
785
cbc_paes_decrypt(struct skcipher_request * req)786 static int cbc_paes_decrypt(struct skcipher_request *req)
787 {
788 return cbc_paes_crypt(req, CPACF_DECRYPT);
789 }
790
cbc_paes_init(struct crypto_skcipher * tfm)791 static int cbc_paes_init(struct crypto_skcipher *tfm)
792 {
793 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
794
795 memset(ctx, 0, sizeof(*ctx));
796 spin_lock_init(&ctx->pk_lock);
797
798 crypto_skcipher_set_reqsize(tfm, sizeof(struct s390_pcbc_req_ctx));
799
800 return 0;
801 }
802
cbc_paes_exit(struct crypto_skcipher * tfm)803 static void cbc_paes_exit(struct crypto_skcipher *tfm)
804 {
805 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
806
807 memzero_explicit(ctx, sizeof(*ctx));
808 }
809
cbc_paes_do_one_request(struct crypto_engine * engine,void * areq)810 static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq)
811 {
812 struct skcipher_request *req = skcipher_request_cast(areq);
813 struct s390_pcbc_req_ctx *req_ctx = skcipher_request_ctx(req);
814 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
815 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
816 struct skcipher_walk *walk = &req_ctx->walk;
817 bool tested = crypto_skcipher_tested(tfm);
818 int rc;
819
820 /* walk has already been prepared */
821
822 rc = cbc_paes_do_crypt(ctx, req_ctx, tested, true);
823 if (rc == -EKEYEXPIRED) {
824 return pkey_handle_expired();
825 } else if (rc && walk->nbytes) {
826 skcipher_walk_done(walk, rc);
827 }
828
829 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
830 pr_debug("request complete with rc=%d\n", rc);
831 local_bh_disable();
832 atomic_dec(&ctx->via_engine_ctr);
833 crypto_finalize_skcipher_request(engine, req, rc);
834 local_bh_enable();
835 return 0;
836 }
837
838 static struct skcipher_engine_alg cbc_paes_alg = {
839 .base = {
840 .base.cra_name = "cbc(paes)",
841 .base.cra_driver_name = "cbc-paes-s390",
842 .base.cra_priority = 402, /* cbc-paes-s390 + 1 */
843 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
844 .base.cra_blocksize = AES_BLOCK_SIZE,
845 .base.cra_ctxsize = sizeof(struct s390_paes_ctx),
846 .base.cra_module = THIS_MODULE,
847 .base.cra_list = LIST_HEAD_INIT(cbc_paes_alg.base.base.cra_list),
848 .init = cbc_paes_init,
849 .exit = cbc_paes_exit,
850 .min_keysize = PAES_MIN_KEYSIZE,
851 .max_keysize = PAES_MAX_KEYSIZE,
852 .ivsize = AES_BLOCK_SIZE,
853 .setkey = cbc_paes_setkey,
854 .encrypt = cbc_paes_encrypt,
855 .decrypt = cbc_paes_decrypt,
856 },
857 .op = {
858 .do_one_request = cbc_paes_do_one_request,
859 },
860 };
861
862 /*
863 * PAES CTR implementation
864 */
865
866 struct ctr_param {
867 u8 key[PAES_256_PROTKEY_SIZE];
868 } __packed;
869
870 struct s390_pctr_req_ctx {
871 unsigned long modifier;
872 struct skcipher_walk walk;
873 bool param_init_done;
874 struct ctr_param param;
875 };
876
ctr_paes_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)877 static int ctr_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
878 unsigned int key_len)
879 {
880 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
881 bool tested = crypto_skcipher_tested(tfm);
882 long fc;
883 int rc;
884
885 /* set raw key into context */
886 rc = paes_ctx_setkey(ctx, in_key, key_len);
887 if (rc)
888 goto out;
889
890 /* convert raw key into protected key */
891 rc = paes_convert_key(ctx, tested);
892 if (rc)
893 goto out;
894
895 /* Pick the correct function code based on the protected key type */
896 switch (ctx->pk.type) {
897 case PKEY_KEYTYPE_AES_128:
898 fc = CPACF_KMCTR_PAES_128;
899 break;
900 case PKEY_KEYTYPE_AES_192:
901 fc = CPACF_KMCTR_PAES_192;
902 break;
903 case PKEY_KEYTYPE_AES_256:
904 fc = CPACF_KMCTR_PAES_256;
905 break;
906 default:
907 fc = 0;
908 break;
909 }
910 ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0;
911
912 rc = fc ? 0 : -EINVAL;
913
914 out:
915 pr_debug("rc=%d\n", rc);
916 return rc;
917 }
918
__ctrblk_init(u8 * ctrptr,u8 * iv,unsigned int nbytes)919 static inline unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes)
920 {
921 unsigned int i, n;
922
923 /* only use complete blocks, max. PAGE_SIZE */
924 memcpy(ctrptr, iv, AES_BLOCK_SIZE);
925 n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1);
926 for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) {
927 memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE);
928 crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE);
929 ctrptr += AES_BLOCK_SIZE;
930 }
931 return n;
932 }
933
__ctr_paes_do_crypt(struct s390_paes_ctx * ctx,struct ctr_param * param,struct skcipher_walk * walk,bool tested,bool maysleep,bool locked)934 static int __ctr_paes_do_crypt(struct s390_paes_ctx *ctx,
935 struct ctr_param *param,
936 struct skcipher_walk *walk,
937 bool tested, bool maysleep, bool locked)
938 {
939 unsigned int nbytes, n, k;
940 u8 *ctrptr;
941 int rc = 0;
942
943 /*
944 * Note that in case of partial processing or failure the walk
945 * is NOT unmapped here. So a follow up task may reuse the walk
946 * or in case of unrecoverable failure needs to unmap it.
947 */
948 while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) {
949 n = AES_BLOCK_SIZE;
950 if (nbytes >= 2 * AES_BLOCK_SIZE && locked)
951 n = __ctrblk_init(ctrblk, walk->iv, nbytes);
952 ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv;
953 k = cpacf_kmctr(ctx->fc, param, walk->dst.virt.addr,
954 walk->src.virt.addr, n, ctrptr);
955 if (k) {
956 if (ctrptr == ctrblk)
957 memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE,
958 AES_BLOCK_SIZE);
959 crypto_inc(walk->iv, AES_BLOCK_SIZE);
960 rc = skcipher_walk_done(walk, nbytes - k);
961 if (rc)
962 goto out;
963 }
964 if (k < n) {
965 if (!maysleep) {
966 rc = -EKEYEXPIRED;
967 goto out;
968 }
969 rc = paes_convert_key(ctx, tested);
970 if (rc)
971 goto out;
972 spin_lock_bh(&ctx->pk_lock);
973 memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
974 spin_unlock_bh(&ctx->pk_lock);
975 }
976 }
977
978 out:
979 return rc;
980 }
981
ctr_paes_do_crypt(struct s390_paes_ctx * ctx,struct s390_pctr_req_ctx * req_ctx,bool tested,bool maysleep)982 static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx,
983 struct s390_pctr_req_ctx *req_ctx,
984 bool tested, bool maysleep)
985 {
986 struct ctr_param *param = &req_ctx->param;
987 struct skcipher_walk *walk = &req_ctx->walk;
988 u8 buf[AES_BLOCK_SIZE];
989 int pk_state, rc = 0;
990
991 if (!req_ctx->param_init_done) {
992 /* fetch and check protected key state */
993 spin_lock_bh(&ctx->pk_lock);
994 pk_state = ctx->pk_state;
995 switch (pk_state) {
996 case PK_STATE_NO_KEY:
997 rc = -ENOKEY;
998 break;
999 case PK_STATE_CONVERT_IN_PROGRESS:
1000 rc = -EKEYEXPIRED;
1001 break;
1002 case PK_STATE_VALID:
1003 memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
1004 req_ctx->param_init_done = true;
1005 break;
1006 default:
1007 rc = pk_state < 0 ? pk_state : -EIO;
1008 break;
1009 }
1010 spin_unlock_bh(&ctx->pk_lock);
1011 }
1012 if (rc)
1013 goto out;
1014
1015 if (down_trylock(&ctrblk_sem) == 0) {
1016 rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, true);
1017 up(&ctrblk_sem);
1018 } else {
1019 rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, false);
1020 }
1021
1022 /* final block may be < AES_BLOCK_SIZE, copy only nbytes */
1023 if (!rc && walk->nbytes > 0) {
1024 memset(buf, 0, AES_BLOCK_SIZE);
1025 memcpy(buf, walk->src.virt.addr, walk->nbytes);
1026 while (1) {
1027 if (cpacf_kmctr(ctx->fc, param, buf,
1028 buf, AES_BLOCK_SIZE,
1029 walk->iv) == AES_BLOCK_SIZE)
1030 break;
1031 if (!maysleep) {
1032 rc = -EKEYEXPIRED;
1033 goto out;
1034 }
1035 rc = paes_convert_key(ctx, tested);
1036 if (rc)
1037 goto out;
1038 spin_lock_bh(&ctx->pk_lock);
1039 memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
1040 spin_unlock_bh(&ctx->pk_lock);
1041 }
1042 memcpy(walk->dst.virt.addr, buf, walk->nbytes);
1043 crypto_inc(walk->iv, AES_BLOCK_SIZE);
1044 rc = skcipher_walk_done(walk, 0);
1045 }
1046
1047 out:
1048 memzero_explicit(buf, sizeof(buf));
1049 pr_debug("rc=%d\n", rc);
1050 return rc;
1051 }
1052
ctr_paes_crypt(struct skcipher_request * req)1053 static int ctr_paes_crypt(struct skcipher_request *req)
1054 {
1055 struct s390_pctr_req_ctx *req_ctx = skcipher_request_ctx(req);
1056 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1057 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
1058 struct skcipher_walk *walk = &req_ctx->walk;
1059 bool tested = crypto_skcipher_tested(tfm);
1060 bool cleanup = true;
1061 int rc;
1062
1063 /*
1064 * Attempt synchronous encryption first. If it fails, schedule the request
1065 * asynchronously via the crypto engine. To preserve execution order,
1066 * once a request is queued to the engine, further requests using the same
1067 * tfm will also be routed through the engine.
1068 */
1069
1070 rc = skcipher_walk_virt(walk, req, false);
1071 if (rc)
1072 goto out;
1073
1074 req_ctx->param_init_done = false;
1075
1076 /* Try synchronous operation if no active engine usage */
1077 if (!atomic_read(&ctx->via_engine_ctr)) {
1078 rc = ctr_paes_do_crypt(ctx, req_ctx, tested, false);
1079 if (rc == 0)
1080 goto out;
1081 }
1082
1083 /*
1084 * If sync operation failed or key expired or there are already
1085 * requests enqueued via engine, fallback to async. Mark tfm as
1086 * using engine to serialize requests.
1087 */
1088 if (rc == 0 || rc == -EKEYEXPIRED) {
1089 atomic_inc(&ctx->via_engine_ctr);
1090 rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
1091 if (rc == -EINPROGRESS || rc == -EBUSY)
1092 cleanup = false;
1093 else
1094 atomic_dec(&ctx->via_engine_ctr);
1095 }
1096
1097 if (cleanup && walk->nbytes)
1098 skcipher_walk_done(walk, rc);
1099
1100 out:
1101 if (cleanup)
1102 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
1103 pr_debug("rc=%d\n", rc);
1104 return rc;
1105 }
1106
ctr_paes_init(struct crypto_skcipher * tfm)1107 static int ctr_paes_init(struct crypto_skcipher *tfm)
1108 {
1109 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
1110
1111 memset(ctx, 0, sizeof(*ctx));
1112 spin_lock_init(&ctx->pk_lock);
1113
1114 crypto_skcipher_set_reqsize(tfm, sizeof(struct s390_pctr_req_ctx));
1115
1116 return 0;
1117 }
1118
ctr_paes_exit(struct crypto_skcipher * tfm)1119 static void ctr_paes_exit(struct crypto_skcipher *tfm)
1120 {
1121 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
1122
1123 memzero_explicit(ctx, sizeof(*ctx));
1124 }
1125
ctr_paes_do_one_request(struct crypto_engine * engine,void * areq)1126 static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq)
1127 {
1128 struct skcipher_request *req = skcipher_request_cast(areq);
1129 struct s390_pctr_req_ctx *req_ctx = skcipher_request_ctx(req);
1130 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1131 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
1132 struct skcipher_walk *walk = &req_ctx->walk;
1133 bool tested = crypto_skcipher_tested(tfm);
1134 int rc;
1135
1136 /* walk has already been prepared */
1137
1138 rc = ctr_paes_do_crypt(ctx, req_ctx, tested, true);
1139 if (rc == -EKEYEXPIRED) {
1140 return pkey_handle_expired();
1141 } else if (rc && walk->nbytes) {
1142 skcipher_walk_done(walk, rc);
1143 }
1144
1145 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
1146 pr_debug("request complete with rc=%d\n", rc);
1147 local_bh_disable();
1148 atomic_dec(&ctx->via_engine_ctr);
1149 crypto_finalize_skcipher_request(engine, req, rc);
1150 local_bh_enable();
1151 return 0;
1152 }
1153
1154 static struct skcipher_engine_alg ctr_paes_alg = {
1155 .base = {
1156 .base.cra_name = "ctr(paes)",
1157 .base.cra_driver_name = "ctr-paes-s390",
1158 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */
1159 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
1160 .base.cra_blocksize = 1,
1161 .base.cra_ctxsize = sizeof(struct s390_paes_ctx),
1162 .base.cra_module = THIS_MODULE,
1163 .base.cra_list = LIST_HEAD_INIT(ctr_paes_alg.base.base.cra_list),
1164 .init = ctr_paes_init,
1165 .exit = ctr_paes_exit,
1166 .min_keysize = PAES_MIN_KEYSIZE,
1167 .max_keysize = PAES_MAX_KEYSIZE,
1168 .ivsize = AES_BLOCK_SIZE,
1169 .setkey = ctr_paes_setkey,
1170 .encrypt = ctr_paes_crypt,
1171 .decrypt = ctr_paes_crypt,
1172 .chunksize = AES_BLOCK_SIZE,
1173 },
1174 .op = {
1175 .do_one_request = ctr_paes_do_one_request,
1176 },
1177 };
1178
1179 /*
1180 * PAES XTS implementation
1181 */
1182
1183 struct xts_full_km_param {
1184 u8 key[64];
1185 u8 tweak[16];
1186 u8 nap[16];
1187 u8 wkvp[32];
1188 } __packed;
1189
1190 struct xts_km_param {
1191 u8 key[PAES_256_PROTKEY_SIZE];
1192 u8 init[16];
1193 } __packed;
1194
1195 struct xts_pcc_param {
1196 u8 key[PAES_256_PROTKEY_SIZE];
1197 u8 tweak[16];
1198 u8 block[16];
1199 u8 bit[16];
1200 u8 xts[16];
1201 } __packed;
1202
1203 struct s390_pxts_req_ctx {
1204 unsigned long modifier;
1205 struct skcipher_walk walk;
1206 bool param_init_done;
1207 union {
1208 struct xts_full_km_param full_km_param;
1209 struct xts_km_param km_param;
1210 } param;
1211 };
1212
xts_paes_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int in_keylen)1213 static int xts_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
1214 unsigned int in_keylen)
1215 {
1216 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
1217 bool tested = crypto_skcipher_tested(tfm);
1218 u8 ckey[2 * AES_MAX_KEY_SIZE];
1219 unsigned int ckey_len;
1220 long fc;
1221 int rc;
1222
1223 if ((in_keylen == 32 || in_keylen == 64) &&
1224 xts_verify_key(tfm, in_key, in_keylen))
1225 return -EINVAL;
1226
1227 /* set raw key into context */
1228 rc = pxts_ctx_setkey(ctx, in_key, in_keylen);
1229 if (rc)
1230 goto out;
1231
1232 /* convert raw key(s) into protected key(s) */
1233 rc = pxts_convert_key(ctx, tested);
1234 if (rc)
1235 goto out;
1236
1237 /*
1238 * xts_verify_key verifies the key length is not odd and makes
1239 * sure that the two keys are not the same. This can be done
1240 * on the two protected keys as well - but not for full xts keys.
1241 */
1242 if (ctx->pk[0].type == PKEY_KEYTYPE_AES_128 ||
1243 ctx->pk[0].type == PKEY_KEYTYPE_AES_256) {
1244 ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ?
1245 AES_KEYSIZE_128 : AES_KEYSIZE_256;
1246 memcpy(ckey, ctx->pk[0].protkey, ckey_len);
1247 memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len);
1248 rc = xts_verify_key(tfm, ckey, 2 * ckey_len);
1249 memzero_explicit(ckey, sizeof(ckey));
1250 if (rc)
1251 goto out;
1252 }
1253
1254 /* Pick the correct function code based on the protected key type */
1255 switch (ctx->pk[0].type) {
1256 case PKEY_KEYTYPE_AES_128:
1257 fc = CPACF_KM_PXTS_128;
1258 break;
1259 case PKEY_KEYTYPE_AES_256:
1260 fc = CPACF_KM_PXTS_256;
1261 break;
1262 case PKEY_KEYTYPE_AES_XTS_128:
1263 fc = CPACF_KM_PXTS_128_FULL;
1264 break;
1265 case PKEY_KEYTYPE_AES_XTS_256:
1266 fc = CPACF_KM_PXTS_256_FULL;
1267 break;
1268 default:
1269 fc = 0;
1270 break;
1271 }
1272 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
1273
1274 rc = fc ? 0 : -EINVAL;
1275
1276 out:
1277 pr_debug("rc=%d\n", rc);
1278 return rc;
1279 }
1280
xts_paes_do_crypt_fullkey(struct s390_pxts_ctx * ctx,struct s390_pxts_req_ctx * req_ctx,bool tested,bool maysleep)1281 static int xts_paes_do_crypt_fullkey(struct s390_pxts_ctx *ctx,
1282 struct s390_pxts_req_ctx *req_ctx,
1283 bool tested, bool maysleep)
1284 {
1285 struct xts_full_km_param *param = &req_ctx->param.full_km_param;
1286 struct skcipher_walk *walk = &req_ctx->walk;
1287 unsigned int keylen, offset, nbytes, n, k;
1288 int rc = 0;
1289
1290 /*
1291 * The calling function xts_paes_do_crypt() ensures the
1292 * protected key state is always PK_STATE_VALID when this
1293 * function is invoked.
1294 */
1295
1296 keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_XTS_128) ? 32 : 64;
1297 offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_XTS_128) ? 32 : 0;
1298
1299 if (!req_ctx->param_init_done) {
1300 memset(param, 0, sizeof(*param));
1301 spin_lock_bh(&ctx->pk_lock);
1302 memcpy(param->key + offset, ctx->pk[0].protkey, keylen);
1303 memcpy(param->wkvp, ctx->pk[0].protkey + keylen, sizeof(param->wkvp));
1304 spin_unlock_bh(&ctx->pk_lock);
1305 memcpy(param->tweak, walk->iv, sizeof(param->tweak));
1306 param->nap[0] = 0x01; /* initial alpha power (1, little-endian) */
1307 req_ctx->param_init_done = true;
1308 }
1309
1310 /*
1311 * Note that in case of partial processing or failure the walk
1312 * is NOT unmapped here. So a follow up task may reuse the walk
1313 * or in case of unrecoverable failure needs to unmap it.
1314 */
1315 while ((nbytes = walk->nbytes) != 0) {
1316 /* only use complete blocks */
1317 n = nbytes & ~(AES_BLOCK_SIZE - 1);
1318 k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset,
1319 walk->dst.virt.addr, walk->src.virt.addr, n);
1320 if (k) {
1321 rc = skcipher_walk_done(walk, nbytes - k);
1322 if (rc)
1323 goto out;
1324 }
1325 if (k < n) {
1326 if (!maysleep) {
1327 rc = -EKEYEXPIRED;
1328 goto out;
1329 }
1330 rc = pxts_convert_key(ctx, tested);
1331 if (rc)
1332 goto out;
1333 spin_lock_bh(&ctx->pk_lock);
1334 memcpy(param->key + offset, ctx->pk[0].protkey, keylen);
1335 memcpy(param->wkvp, ctx->pk[0].protkey + keylen, sizeof(param->wkvp));
1336 spin_unlock_bh(&ctx->pk_lock);
1337 }
1338 }
1339
1340 out:
1341 pr_debug("rc=%d\n", rc);
1342 return rc;
1343 }
1344
__xts_2keys_prep_param(struct s390_pxts_ctx * ctx,struct xts_km_param * param,struct skcipher_walk * walk,unsigned int keylen,unsigned int offset,bool tested,bool maysleep)1345 static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx,
1346 struct xts_km_param *param,
1347 struct skcipher_walk *walk,
1348 unsigned int keylen,
1349 unsigned int offset,
1350 bool tested, bool maysleep)
1351 {
1352 struct xts_pcc_param pcc_param;
1353 unsigned long cc = 1;
1354 int rc = 0;
1355
1356 while (cc) {
1357 memset(&pcc_param, 0, sizeof(pcc_param));
1358 memcpy(pcc_param.tweak, walk->iv, sizeof(pcc_param.tweak));
1359 spin_lock_bh(&ctx->pk_lock);
1360 memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen);
1361 memcpy(param->key + offset, ctx->pk[0].protkey, keylen);
1362 spin_unlock_bh(&ctx->pk_lock);
1363 cc = cpacf_pcc(ctx->fc, pcc_param.key + offset);
1364 if (cc) {
1365 if (!maysleep) {
1366 rc = -EKEYEXPIRED;
1367 break;
1368 }
1369 rc = pxts_convert_key(ctx, tested);
1370 if (rc)
1371 break;
1372 continue;
1373 }
1374 memcpy(param->init, pcc_param.xts, 16);
1375 }
1376
1377 memzero_explicit(&pcc_param, sizeof(pcc_param));
1378 return rc;
1379 }
1380
xts_paes_do_crypt_2keys(struct s390_pxts_ctx * ctx,struct s390_pxts_req_ctx * req_ctx,bool tested,bool maysleep)1381 static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx,
1382 struct s390_pxts_req_ctx *req_ctx,
1383 bool tested, bool maysleep)
1384 {
1385 struct xts_km_param *param = &req_ctx->param.km_param;
1386 struct skcipher_walk *walk = &req_ctx->walk;
1387 unsigned int keylen, offset, nbytes, n, k;
1388 int rc = 0;
1389
1390 /*
1391 * The calling function xts_paes_do_crypt() ensures the
1392 * protected key state is always PK_STATE_VALID when this
1393 * function is invoked.
1394 */
1395
1396 keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64;
1397 offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0;
1398
1399 if (!req_ctx->param_init_done) {
1400 rc = __xts_2keys_prep_param(ctx, param, walk,
1401 keylen, offset, tested, maysleep);
1402 if (rc)
1403 goto out;
1404 req_ctx->param_init_done = true;
1405 }
1406
1407 /*
1408 * Note that in case of partial processing or failure the walk
1409 * is NOT unmapped here. So a follow up task may reuse the walk
1410 * or in case of unrecoverable failure needs to unmap it.
1411 */
1412 while ((nbytes = walk->nbytes) != 0) {
1413 /* only use complete blocks */
1414 n = nbytes & ~(AES_BLOCK_SIZE - 1);
1415 k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset,
1416 walk->dst.virt.addr, walk->src.virt.addr, n);
1417 if (k) {
1418 rc = skcipher_walk_done(walk, nbytes - k);
1419 if (rc)
1420 goto out;
1421 }
1422 if (k < n) {
1423 if (!maysleep) {
1424 rc = -EKEYEXPIRED;
1425 goto out;
1426 }
1427 rc = pxts_convert_key(ctx, tested);
1428 if (rc)
1429 goto out;
1430 spin_lock_bh(&ctx->pk_lock);
1431 memcpy(param->key + offset, ctx->pk[0].protkey, keylen);
1432 spin_unlock_bh(&ctx->pk_lock);
1433 }
1434 }
1435
1436 out:
1437 pr_debug("rc=%d\n", rc);
1438 return rc;
1439 }
1440
xts_paes_do_crypt(struct s390_pxts_ctx * ctx,struct s390_pxts_req_ctx * req_ctx,bool tested,bool maysleep)1441 static int xts_paes_do_crypt(struct s390_pxts_ctx *ctx,
1442 struct s390_pxts_req_ctx *req_ctx,
1443 bool tested, bool maysleep)
1444 {
1445 int pk_state, rc = 0;
1446
1447 /* fetch and check protected key state */
1448 spin_lock_bh(&ctx->pk_lock);
1449 pk_state = ctx->pk_state;
1450 switch (pk_state) {
1451 case PK_STATE_NO_KEY:
1452 rc = -ENOKEY;
1453 break;
1454 case PK_STATE_CONVERT_IN_PROGRESS:
1455 rc = -EKEYEXPIRED;
1456 break;
1457 case PK_STATE_VALID:
1458 break;
1459 default:
1460 rc = pk_state < 0 ? pk_state : -EIO;
1461 break;
1462 }
1463 spin_unlock_bh(&ctx->pk_lock);
1464 if (rc)
1465 goto out;
1466
1467 /* Call the 'real' crypt function based on the xts prot key type. */
1468 switch (ctx->fc) {
1469 case CPACF_KM_PXTS_128:
1470 case CPACF_KM_PXTS_256:
1471 rc = xts_paes_do_crypt_2keys(ctx, req_ctx, tested, maysleep);
1472 break;
1473 case CPACF_KM_PXTS_128_FULL:
1474 case CPACF_KM_PXTS_256_FULL:
1475 rc = xts_paes_do_crypt_fullkey(ctx, req_ctx, tested, maysleep);
1476 break;
1477 default:
1478 rc = -EINVAL;
1479 }
1480
1481 out:
1482 pr_debug("rc=%d\n", rc);
1483 return rc;
1484 }
1485
xts_paes_crypt(struct skcipher_request * req,unsigned long modifier)1486 static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long modifier)
1487 {
1488 struct s390_pxts_req_ctx *req_ctx = skcipher_request_ctx(req);
1489 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1490 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
1491 struct skcipher_walk *walk = &req_ctx->walk;
1492 bool tested = crypto_skcipher_tested(tfm);
1493 bool cleanup = true;
1494 int rc;
1495
1496 /*
1497 * Attempt synchronous encryption first. If it fails, schedule the request
1498 * asynchronously via the crypto engine. To preserve execution order,
1499 * once a request is queued to the engine, further requests using the same
1500 * tfm will also be routed through the engine.
1501 */
1502
1503 rc = skcipher_walk_virt(walk, req, false);
1504 if (rc)
1505 goto out;
1506
1507 req_ctx->modifier = modifier;
1508 req_ctx->param_init_done = false;
1509
1510 /* Try synchronous operation if no active engine usage */
1511 if (!atomic_read(&ctx->via_engine_ctr)) {
1512 rc = xts_paes_do_crypt(ctx, req_ctx, tested, false);
1513 if (rc == 0)
1514 goto out;
1515 }
1516
1517 /*
1518 * If sync operation failed or key expired or there are already
1519 * requests enqueued via engine, fallback to async. Mark tfm as
1520 * using engine to serialize requests.
1521 */
1522 if (rc == 0 || rc == -EKEYEXPIRED) {
1523 atomic_inc(&ctx->via_engine_ctr);
1524 rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
1525 if (rc == -EINPROGRESS || rc == -EBUSY)
1526 cleanup = false;
1527 else
1528 atomic_dec(&ctx->via_engine_ctr);
1529 }
1530
1531 if (cleanup && walk->nbytes)
1532 skcipher_walk_done(walk, rc);
1533
1534 out:
1535 if (cleanup)
1536 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
1537 pr_debug("rc=%d\n", rc);
1538 return rc;
1539 }
1540
xts_paes_encrypt(struct skcipher_request * req)1541 static int xts_paes_encrypt(struct skcipher_request *req)
1542 {
1543 return xts_paes_crypt(req, 0);
1544 }
1545
xts_paes_decrypt(struct skcipher_request * req)1546 static int xts_paes_decrypt(struct skcipher_request *req)
1547 {
1548 return xts_paes_crypt(req, CPACF_DECRYPT);
1549 }
1550
xts_paes_init(struct crypto_skcipher * tfm)1551 static int xts_paes_init(struct crypto_skcipher *tfm)
1552 {
1553 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
1554
1555 memset(ctx, 0, sizeof(*ctx));
1556 spin_lock_init(&ctx->pk_lock);
1557
1558 crypto_skcipher_set_reqsize(tfm, sizeof(struct s390_pxts_req_ctx));
1559
1560 return 0;
1561 }
1562
xts_paes_exit(struct crypto_skcipher * tfm)1563 static void xts_paes_exit(struct crypto_skcipher *tfm)
1564 {
1565 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
1566
1567 memzero_explicit(ctx, sizeof(*ctx));
1568 }
1569
xts_paes_do_one_request(struct crypto_engine * engine,void * areq)1570 static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq)
1571 {
1572 struct skcipher_request *req = skcipher_request_cast(areq);
1573 struct s390_pxts_req_ctx *req_ctx = skcipher_request_ctx(req);
1574 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1575 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
1576 struct skcipher_walk *walk = &req_ctx->walk;
1577 bool tested = crypto_skcipher_tested(tfm);
1578 int rc;
1579
1580 /* walk has already been prepared */
1581
1582 rc = xts_paes_do_crypt(ctx, req_ctx, tested, true);
1583 if (rc == -EKEYEXPIRED) {
1584 return pkey_handle_expired();
1585 } else if (rc && walk->nbytes) {
1586 skcipher_walk_done(walk, rc);
1587 }
1588
1589 memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
1590 pr_debug("request complete with rc=%d\n", rc);
1591 local_bh_disable();
1592 atomic_dec(&ctx->via_engine_ctr);
1593 crypto_finalize_skcipher_request(engine, req, rc);
1594 local_bh_enable();
1595 return 0;
1596 }
1597
1598 static struct skcipher_engine_alg xts_paes_alg = {
1599 .base = {
1600 .base.cra_name = "xts(paes)",
1601 .base.cra_driver_name = "xts-paes-s390",
1602 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */
1603 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
1604 .base.cra_blocksize = AES_BLOCK_SIZE,
1605 .base.cra_ctxsize = sizeof(struct s390_pxts_ctx),
1606 .base.cra_module = THIS_MODULE,
1607 .base.cra_list = LIST_HEAD_INIT(xts_paes_alg.base.base.cra_list),
1608 .init = xts_paes_init,
1609 .exit = xts_paes_exit,
1610 .min_keysize = 2 * PAES_MIN_KEYSIZE,
1611 .max_keysize = 2 * PAES_MAX_KEYSIZE,
1612 .ivsize = AES_BLOCK_SIZE,
1613 .setkey = xts_paes_setkey,
1614 .encrypt = xts_paes_encrypt,
1615 .decrypt = xts_paes_decrypt,
1616 },
1617 .op = {
1618 .do_one_request = xts_paes_do_one_request,
1619 },
1620 };
1621
1622 /*
1623 * alg register, unregister, module init, exit
1624 */
1625
1626 static struct miscdevice paes_dev = {
1627 .name = "paes",
1628 .minor = MISC_DYNAMIC_MINOR,
1629 };
1630
__crypto_unregister_skcipher(struct skcipher_engine_alg * alg)1631 static inline void __crypto_unregister_skcipher(struct skcipher_engine_alg *alg)
1632 {
1633 if (!list_empty(&alg->base.base.cra_list))
1634 crypto_engine_unregister_skcipher(alg);
1635 }
1636
paes_s390_fini(void)1637 static void paes_s390_fini(void)
1638 {
1639 if (paes_crypto_engine) {
1640 crypto_engine_stop(paes_crypto_engine);
1641 crypto_engine_exit(paes_crypto_engine);
1642 }
1643 __crypto_unregister_skcipher(&ctr_paes_alg);
1644 __crypto_unregister_skcipher(&xts_paes_alg);
1645 __crypto_unregister_skcipher(&cbc_paes_alg);
1646 __crypto_unregister_skcipher(&ecb_paes_alg);
1647 if (ctrblk)
1648 free_page((unsigned long)ctrblk);
1649 misc_deregister(&paes_dev);
1650 }
1651
paes_s390_init(void)1652 static int __init paes_s390_init(void)
1653 {
1654 int rc;
1655
1656 /* register a simple paes pseudo misc device */
1657 rc = misc_register(&paes_dev);
1658 if (rc)
1659 return rc;
1660
1661 /* with this pseudo devie alloc and start a crypto engine */
1662 paes_crypto_engine =
1663 crypto_engine_alloc_init_and_set(paes_dev.this_device,
1664 true, false, MAX_QLEN);
1665 if (!paes_crypto_engine) {
1666 rc = -ENOMEM;
1667 goto out_err;
1668 }
1669 rc = crypto_engine_start(paes_crypto_engine);
1670 if (rc) {
1671 crypto_engine_exit(paes_crypto_engine);
1672 paes_crypto_engine = NULL;
1673 goto out_err;
1674 }
1675
1676 /* Query available functions for KM, KMC and KMCTR */
1677 cpacf_query(CPACF_KM, &km_functions);
1678 cpacf_query(CPACF_KMC, &kmc_functions);
1679 cpacf_query(CPACF_KMCTR, &kmctr_functions);
1680
1681 if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) ||
1682 cpacf_test_func(&km_functions, CPACF_KM_PAES_192) ||
1683 cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) {
1684 rc = crypto_engine_register_skcipher(&ecb_paes_alg);
1685 if (rc)
1686 goto out_err;
1687 pr_debug("%s registered\n", ecb_paes_alg.base.base.cra_driver_name);
1688 }
1689
1690 if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) ||
1691 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) ||
1692 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) {
1693 rc = crypto_engine_register_skcipher(&cbc_paes_alg);
1694 if (rc)
1695 goto out_err;
1696 pr_debug("%s registered\n", cbc_paes_alg.base.base.cra_driver_name);
1697 }
1698
1699 if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) ||
1700 cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) {
1701 rc = crypto_engine_register_skcipher(&xts_paes_alg);
1702 if (rc)
1703 goto out_err;
1704 pr_debug("%s registered\n", xts_paes_alg.base.base.cra_driver_name);
1705 }
1706
1707 if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) ||
1708 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) ||
1709 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) {
1710 ctrblk = (u8 *)__get_free_page(GFP_KERNEL);
1711 if (!ctrblk) {
1712 rc = -ENOMEM;
1713 goto out_err;
1714 }
1715 rc = crypto_engine_register_skcipher(&ctr_paes_alg);
1716 if (rc)
1717 goto out_err;
1718 pr_debug("%s registered\n", ctr_paes_alg.base.base.cra_driver_name);
1719 }
1720
1721 return 0;
1722
1723 out_err:
1724 paes_s390_fini();
1725 return rc;
1726 }
1727
1728 module_init(paes_s390_init);
1729 module_exit(paes_s390_fini);
1730
1731 MODULE_ALIAS_CRYPTO("ecb(paes)");
1732 MODULE_ALIAS_CRYPTO("cbc(paes)");
1733 MODULE_ALIAS_CRYPTO("ctr(paes)");
1734 MODULE_ALIAS_CRYPTO("xts(paes)");
1735
1736 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys");
1737 MODULE_LICENSE("GPL");
1738