1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Copyright IBM Corp. 2025
4 *
5 * s390 specific HMAC support for protected keys.
6 */
7
8 #define pr_fmt(fmt) "phmac_s390: " fmt
9
10 #include <asm/cpacf.h>
11 #include <asm/pkey.h>
12 #include <crypto/engine.h>
13 #include <crypto/hash.h>
14 #include <crypto/internal/hash.h>
15 #include <crypto/sha2.h>
16 #include <linux/atomic.h>
17 #include <linux/cpufeature.h>
18 #include <linux/delay.h>
19 #include <linux/miscdevice.h>
20 #include <linux/module.h>
21 #include <linux/spinlock.h>
22
23 static struct crypto_engine *phmac_crypto_engine;
24 #define MAX_QLEN 10
25
26 static bool pkey_clrkey_allowed;
27 module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444);
28 MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)");
29
30 /*
31 * A simple hash walk helper
32 */
33
34 struct hash_walk_helper {
35 struct crypto_hash_walk walk;
36 const u8 *walkaddr;
37 int walkbytes;
38 };
39
40 /*
41 * Prepare hash walk helper.
42 * Set up the base hash walk, fill walkaddr and walkbytes.
43 * Returns 0 on success or negative value on error.
44 */
hwh_prepare(struct ahash_request * req,struct hash_walk_helper * hwh)45 static inline int hwh_prepare(struct ahash_request *req,
46 struct hash_walk_helper *hwh)
47 {
48 hwh->walkbytes = crypto_hash_walk_first(req, &hwh->walk);
49 if (hwh->walkbytes < 0)
50 return hwh->walkbytes;
51 hwh->walkaddr = hwh->walk.data;
52 return 0;
53 }
54
55 /*
56 * Advance hash walk helper by n bytes.
57 * Progress the walkbytes and walkaddr fields by n bytes.
58 * If walkbytes is then 0, pull next hunk from hash walk
59 * and update walkbytes and walkaddr.
60 * If n is negative, unmap hash walk and return error.
61 * Returns 0 on success or negative value on error.
62 */
hwh_advance(struct hash_walk_helper * hwh,int n)63 static inline int hwh_advance(struct hash_walk_helper *hwh, int n)
64 {
65 if (n < 0) {
66 hwh->walkbytes = n;
67 return crypto_hash_walk_done(&hwh->walk, n);
68 }
69
70 hwh->walkbytes -= n;
71 hwh->walkaddr += n;
72 if (hwh->walkbytes > 0)
73 return 0;
74
75 hwh->walkbytes = crypto_hash_walk_done(&hwh->walk, 0);
76 if (hwh->walkbytes < 0)
77 return hwh->walkbytes;
78
79 hwh->walkaddr = hwh->walk.data;
80 return 0;
81 }
82
83 /*
84 * KMAC param block layout for sha2 function codes:
85 * The layout of the param block for the KMAC instruction depends on the
86 * blocksize of the used hashing sha2-algorithm function codes. The param block
87 * contains the hash chaining value (cv), the input message bit-length (imbl)
88 * and the hmac-secret (key). To prevent code duplication, the sizes of all
89 * these are calculated based on the blocksize.
90 *
91 * param-block:
92 * +-------+
93 * | cv |
94 * +-------+
95 * | imbl |
96 * +-------+
97 * | key |
98 * +-------+
99 *
100 * sizes:
101 * part | sh2-alg | calculation | size | type
102 * -----+---------+-------------+------+--------
103 * cv | 224/256 | blocksize/2 | 32 | u64[8]
104 * | 384/512 | | 64 | u128[8]
105 * imbl | 224/256 | blocksize/8 | 8 | u64
106 * | 384/512 | | 16 | u128
107 * key | 224/256 | blocksize | 96 | u8[96]
108 * | 384/512 | | 160 | u8[160]
109 */
110
111 #define MAX_DIGEST_SIZE SHA512_DIGEST_SIZE
112 #define MAX_IMBL_SIZE sizeof(u128)
113 #define MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
114
115 #define SHA2_CV_SIZE(bs) ((bs) >> 1)
116 #define SHA2_IMBL_SIZE(bs) ((bs) >> 3)
117
118 #define SHA2_IMBL_OFFSET(bs) (SHA2_CV_SIZE(bs))
119 #define SHA2_KEY_OFFSET(bs) (SHA2_CV_SIZE(bs) + SHA2_IMBL_SIZE(bs))
120
121 #define PHMAC_MAX_KEYSIZE 256
122 #define PHMAC_SHA256_PK_SIZE (SHA256_BLOCK_SIZE + 32)
123 #define PHMAC_SHA512_PK_SIZE (SHA512_BLOCK_SIZE + 32)
124 #define PHMAC_MAX_PK_SIZE PHMAC_SHA512_PK_SIZE
125
126 /* phmac protected key struct */
127 struct phmac_protkey {
128 u32 type;
129 u32 len;
130 u8 protkey[PHMAC_MAX_PK_SIZE];
131 };
132
133 #define PK_STATE_NO_KEY 0
134 #define PK_STATE_CONVERT_IN_PROGRESS 1
135 #define PK_STATE_VALID 2
136
137 /* phmac tfm context */
138 struct phmac_tfm_ctx {
139 /* source key material used to derive a protected key from */
140 u8 keybuf[PHMAC_MAX_KEYSIZE];
141 unsigned int keylen;
142
143 /* cpacf function code to use with this protected key type */
144 long fc;
145
146 /* nr of requests enqueued via crypto engine which use this tfm ctx */
147 atomic_t via_engine_ctr;
148
149 /* spinlock to atomic read/update all the following fields */
150 spinlock_t pk_lock;
151
152 /* see PK_STATE* defines above, < 0 holds convert failure rc */
153 int pk_state;
154 /* if state is valid, pk holds the protected key */
155 struct phmac_protkey pk;
156 };
157
158 union kmac_gr0 {
159 unsigned long reg;
160 struct {
161 unsigned long : 48;
162 unsigned long ikp : 1;
163 unsigned long iimp : 1;
164 unsigned long ccup : 1;
165 unsigned long : 6;
166 unsigned long fc : 7;
167 };
168 };
169
170 struct kmac_sha2_ctx {
171 u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + PHMAC_MAX_PK_SIZE];
172 union kmac_gr0 gr0;
173 u8 buf[MAX_BLOCK_SIZE];
174 u64 buflen[2];
175 };
176
177 enum async_op {
178 OP_NOP = 0,
179 OP_UPDATE,
180 OP_FINAL,
181 OP_FINUP,
182 };
183
184 /* phmac request context */
185 struct phmac_req_ctx {
186 struct hash_walk_helper hwh;
187 struct kmac_sha2_ctx kmac_ctx;
188 enum async_op async_op;
189 };
190
191 /*
192 * Pkey 'token' struct used to derive a protected key value from a clear key.
193 */
194 struct hmac_clrkey_token {
195 u8 type;
196 u8 res0[3];
197 u8 version;
198 u8 res1[3];
199 u32 keytype;
200 u32 len;
201 u8 key[];
202 } __packed;
203
hash_key(const u8 * in,unsigned int inlen,u8 * digest,unsigned int digestsize)204 static int hash_key(const u8 *in, unsigned int inlen,
205 u8 *digest, unsigned int digestsize)
206 {
207 unsigned long func;
208 union {
209 struct sha256_paramblock {
210 u32 h[8];
211 u64 mbl;
212 } sha256;
213 struct sha512_paramblock {
214 u64 h[8];
215 u128 mbl;
216 } sha512;
217 } __packed param;
218
219 #define PARAM_INIT(x, y, z) \
220 param.sha##x.h[0] = SHA##y ## _H0; \
221 param.sha##x.h[1] = SHA##y ## _H1; \
222 param.sha##x.h[2] = SHA##y ## _H2; \
223 param.sha##x.h[3] = SHA##y ## _H3; \
224 param.sha##x.h[4] = SHA##y ## _H4; \
225 param.sha##x.h[5] = SHA##y ## _H5; \
226 param.sha##x.h[6] = SHA##y ## _H6; \
227 param.sha##x.h[7] = SHA##y ## _H7; \
228 param.sha##x.mbl = (z)
229
230 switch (digestsize) {
231 case SHA224_DIGEST_SIZE:
232 func = CPACF_KLMD_SHA_256;
233 PARAM_INIT(256, 224, inlen * 8);
234 break;
235 case SHA256_DIGEST_SIZE:
236 func = CPACF_KLMD_SHA_256;
237 PARAM_INIT(256, 256, inlen * 8);
238 break;
239 case SHA384_DIGEST_SIZE:
240 func = CPACF_KLMD_SHA_512;
241 PARAM_INIT(512, 384, inlen * 8);
242 break;
243 case SHA512_DIGEST_SIZE:
244 func = CPACF_KLMD_SHA_512;
245 PARAM_INIT(512, 512, inlen * 8);
246 break;
247 default:
248 return -EINVAL;
249 }
250
251 #undef PARAM_INIT
252
253 cpacf_klmd(func, ¶m, in, inlen);
254
255 memcpy(digest, ¶m, digestsize);
256
257 return 0;
258 }
259
260 /*
261 * make_clrkey_token() - wrap the clear key into a pkey clearkey token.
262 */
make_clrkey_token(const u8 * clrkey,size_t clrkeylen,unsigned int digestsize,u8 * dest)263 static inline int make_clrkey_token(const u8 *clrkey, size_t clrkeylen,
264 unsigned int digestsize, u8 *dest)
265 {
266 struct hmac_clrkey_token *token = (struct hmac_clrkey_token *)dest;
267 unsigned int blocksize;
268 int rc;
269
270 token->type = 0x00;
271 token->version = 0x02;
272 switch (digestsize) {
273 case SHA224_DIGEST_SIZE:
274 case SHA256_DIGEST_SIZE:
275 token->keytype = PKEY_KEYTYPE_HMAC_512;
276 blocksize = 64;
277 break;
278 case SHA384_DIGEST_SIZE:
279 case SHA512_DIGEST_SIZE:
280 token->keytype = PKEY_KEYTYPE_HMAC_1024;
281 blocksize = 128;
282 break;
283 default:
284 return -EINVAL;
285 }
286 token->len = blocksize;
287
288 if (clrkeylen > blocksize) {
289 rc = hash_key(clrkey, clrkeylen, token->key, digestsize);
290 if (rc)
291 return rc;
292 } else {
293 memcpy(token->key, clrkey, clrkeylen);
294 }
295
296 return 0;
297 }
298
299 /*
300 * phmac_tfm_ctx_setkey() - Set key value into tfm context, maybe construct
301 * a clear key token digestible by pkey from a clear key value.
302 */
phmac_tfm_ctx_setkey(struct phmac_tfm_ctx * tfm_ctx,const u8 * key,unsigned int keylen)303 static inline int phmac_tfm_ctx_setkey(struct phmac_tfm_ctx *tfm_ctx,
304 const u8 *key, unsigned int keylen)
305 {
306 if (keylen > sizeof(tfm_ctx->keybuf))
307 return -EINVAL;
308
309 memcpy(tfm_ctx->keybuf, key, keylen);
310 tfm_ctx->keylen = keylen;
311
312 return 0;
313 }
314
315 /*
316 * Convert the raw key material into a protected key via PKEY api.
317 * This function may sleep - don't call in non-sleeping context.
318 */
convert_key(const u8 * key,unsigned int keylen,struct phmac_protkey * pk,bool tested)319 static inline int convert_key(const u8 *key, unsigned int keylen,
320 struct phmac_protkey *pk, bool tested)
321 {
322 u32 xflags = PKEY_XFLAG_NOMEMALLOC;
323 int rc, i;
324
325 if (tested && !pkey_clrkey_allowed)
326 xflags |= PKEY_XFLAG_NOCLEARKEY;
327
328 pk->len = sizeof(pk->protkey);
329
330 /*
331 * In case of a busy card retry with increasing delay
332 * of 200, 400, 800 and 1600 ms - in total 3 s.
333 */
334 for (rc = -EIO, i = 0; rc && i < 5; i++) {
335 if (rc == -EBUSY && msleep_interruptible((1 << i) * 100)) {
336 rc = -EINTR;
337 goto out;
338 }
339 rc = pkey_key2protkey(key, keylen,
340 pk->protkey, &pk->len, &pk->type,
341 xflags);
342 }
343
344 /* But finally map -EBUSY to -EIO to indicate an IO failure */
345 if (rc == -EBUSY)
346 rc = -EIO;
347
348 out:
349 pr_debug("rc=%d\n", rc);
350 return rc;
351 }
352
353 /*
354 * (Re-)Convert the raw key material from the tfm ctx into a protected
355 * key via convert_key() function. Update the pk_state, pk_type, pk_len
356 * and the protected key in the tfm context.
357 * Please note this function may be invoked concurrently with the very
358 * same tfm context. The pk_lock spinlock in the context ensures an
359 * atomic update of the pk and the pk state but does not guarantee any
360 * order of update. So a fresh converted valid protected key may get
361 * updated with an 'old' expired key value. As the cpacf instructions
362 * detect this, refuse to operate with an invalid key and the calling
363 * code triggers a (re-)conversion this does no harm. This may lead to
364 * unnecessary additional conversion but never to invalid data on the
365 * hash operation.
366 */
phmac_convert_key(struct phmac_tfm_ctx * tfm_ctx,bool tested)367 static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx, bool tested)
368 {
369 struct phmac_protkey pk;
370 int rc;
371
372 spin_lock_bh(&tfm_ctx->pk_lock);
373 tfm_ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS;
374 spin_unlock_bh(&tfm_ctx->pk_lock);
375
376 rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk, tested);
377
378 /* update context */
379 spin_lock_bh(&tfm_ctx->pk_lock);
380 if (rc) {
381 tfm_ctx->pk_state = rc;
382 } else {
383 tfm_ctx->pk_state = PK_STATE_VALID;
384 tfm_ctx->pk = pk;
385 }
386 spin_unlock_bh(&tfm_ctx->pk_lock);
387
388 memzero_explicit(&pk, sizeof(pk));
389 pr_debug("rc=%d\n", rc);
390 return rc;
391 }
392
393 /*
394 * kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize
395 */
kmac_sha2_set_imbl(u8 * param,u64 buflen_lo,u64 buflen_hi,unsigned int blocksize)396 static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo,
397 u64 buflen_hi, unsigned int blocksize)
398 {
399 u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize);
400
401 switch (blocksize) {
402 case SHA256_BLOCK_SIZE:
403 *(u64 *)imbl = buflen_lo * BITS_PER_BYTE;
404 break;
405 case SHA512_BLOCK_SIZE:
406 *(u128 *)imbl = (((u128)buflen_hi << 64) + buflen_lo) << 3;
407 break;
408 default:
409 break;
410 }
411 }
412
phmac_kmac_update(struct ahash_request * req,bool maysleep)413 static int phmac_kmac_update(struct ahash_request *req, bool maysleep)
414 {
415 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
416 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
417 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
418 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
419 struct hash_walk_helper *hwh = &req_ctx->hwh;
420 unsigned int bs = crypto_ahash_blocksize(tfm);
421 bool tested = crypto_ahash_tested(tfm);
422 unsigned int offset, k, n;
423 int rc = 0;
424
425 /*
426 * The walk is always mapped when this function is called.
427 * Note that in case of partial processing or failure the walk
428 * is NOT unmapped here. So a follow up task may reuse the walk
429 * or in case of unrecoverable failure needs to unmap it.
430 */
431
432 while (hwh->walkbytes > 0) {
433 /* check sha2 context buffer */
434 offset = ctx->buflen[0] % bs;
435 if (offset + hwh->walkbytes < bs)
436 goto store;
437
438 if (offset) {
439 /* fill ctx buffer up to blocksize and process this block */
440 n = bs - offset;
441 memcpy(ctx->buf + offset, hwh->walkaddr, n);
442 ctx->gr0.iimp = 1;
443 for (;;) {
444 k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, bs);
445 if (likely(k == bs))
446 break;
447 if (unlikely(k > 0)) {
448 /*
449 * Can't deal with hunks smaller than blocksize.
450 * And kmac should always return the nr of
451 * processed bytes as 0 or a multiple of the
452 * blocksize.
453 */
454 rc = -EIO;
455 goto out;
456 }
457 /* protected key is invalid and needs re-conversion */
458 if (!maysleep) {
459 rc = -EKEYEXPIRED;
460 goto out;
461 }
462 rc = phmac_convert_key(tfm_ctx, tested);
463 if (rc)
464 goto out;
465 spin_lock_bh(&tfm_ctx->pk_lock);
466 memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
467 tfm_ctx->pk.protkey, tfm_ctx->pk.len);
468 spin_unlock_bh(&tfm_ctx->pk_lock);
469 }
470 ctx->buflen[0] += n;
471 if (ctx->buflen[0] < n)
472 ctx->buflen[1]++;
473 rc = hwh_advance(hwh, n);
474 if (unlikely(rc))
475 goto out;
476 offset = 0;
477 }
478
479 /* process as many blocks as possible from the walk */
480 while (hwh->walkbytes >= bs) {
481 n = (hwh->walkbytes / bs) * bs;
482 ctx->gr0.iimp = 1;
483 k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, hwh->walkaddr, n);
484 if (likely(k > 0)) {
485 ctx->buflen[0] += k;
486 if (ctx->buflen[0] < k)
487 ctx->buflen[1]++;
488 rc = hwh_advance(hwh, k);
489 if (unlikely(rc))
490 goto out;
491 }
492 if (unlikely(k < n)) {
493 /* protected key is invalid and needs re-conversion */
494 if (!maysleep) {
495 rc = -EKEYEXPIRED;
496 goto out;
497 }
498 rc = phmac_convert_key(tfm_ctx, tested);
499 if (rc)
500 goto out;
501 spin_lock_bh(&tfm_ctx->pk_lock);
502 memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
503 tfm_ctx->pk.protkey, tfm_ctx->pk.len);
504 spin_unlock_bh(&tfm_ctx->pk_lock);
505 }
506 }
507
508 store:
509 /* store incomplete block in context buffer */
510 if (hwh->walkbytes) {
511 memcpy(ctx->buf + offset, hwh->walkaddr, hwh->walkbytes);
512 ctx->buflen[0] += hwh->walkbytes;
513 if (ctx->buflen[0] < hwh->walkbytes)
514 ctx->buflen[1]++;
515 rc = hwh_advance(hwh, hwh->walkbytes);
516 if (unlikely(rc))
517 goto out;
518 }
519
520 } /* end of while (hwh->walkbytes > 0) */
521
522 out:
523 pr_debug("rc=%d\n", rc);
524 return rc;
525 }
526
phmac_kmac_final(struct ahash_request * req,bool maysleep)527 static int phmac_kmac_final(struct ahash_request *req, bool maysleep)
528 {
529 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
530 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
531 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
532 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
533 unsigned int ds = crypto_ahash_digestsize(tfm);
534 unsigned int bs = crypto_ahash_blocksize(tfm);
535 bool tested = crypto_ahash_tested(tfm);
536 unsigned int k, n;
537 int rc = 0;
538
539 n = ctx->buflen[0] % bs;
540 ctx->gr0.iimp = 0;
541 kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs);
542 for (;;) {
543 k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, n);
544 if (likely(k == n))
545 break;
546 if (unlikely(k > 0)) {
547 /* Can't deal with hunks smaller than blocksize. */
548 rc = -EIO;
549 goto out;
550 }
551 /* protected key is invalid and needs re-conversion */
552 if (!maysleep) {
553 rc = -EKEYEXPIRED;
554 goto out;
555 }
556 rc = phmac_convert_key(tfm_ctx, tested);
557 if (rc)
558 goto out;
559 spin_lock_bh(&tfm_ctx->pk_lock);
560 memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
561 tfm_ctx->pk.protkey, tfm_ctx->pk.len);
562 spin_unlock_bh(&tfm_ctx->pk_lock);
563 }
564
565 memcpy(req->result, ctx->param, ds);
566
567 out:
568 pr_debug("rc=%d\n", rc);
569 return rc;
570 }
571
phmac_init(struct ahash_request * req)572 static int phmac_init(struct ahash_request *req)
573 {
574 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
575 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
576 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
577 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
578 unsigned int bs = crypto_ahash_blocksize(tfm);
579 int rc = 0;
580
581 /* zero request context (includes the kmac sha2 context) */
582 memset(req_ctx, 0, sizeof(*req_ctx));
583
584 /*
585 * setkey() should have set a valid fc into the tfm context.
586 * Copy this function code into the gr0 field of the kmac context.
587 */
588 if (!tfm_ctx->fc) {
589 rc = -ENOKEY;
590 goto out;
591 }
592 kmac_ctx->gr0.fc = tfm_ctx->fc;
593
594 /*
595 * Copy the pk from tfm ctx into kmac ctx. The protected key
596 * may be outdated but update() and final() will handle this.
597 */
598 spin_lock_bh(&tfm_ctx->pk_lock);
599 memcpy(kmac_ctx->param + SHA2_KEY_OFFSET(bs),
600 tfm_ctx->pk.protkey, tfm_ctx->pk.len);
601 spin_unlock_bh(&tfm_ctx->pk_lock);
602
603 out:
604 pr_debug("rc=%d\n", rc);
605 return rc;
606 }
607
phmac_update(struct ahash_request * req)608 static int phmac_update(struct ahash_request *req)
609 {
610 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
611 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
612 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
613 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
614 struct hash_walk_helper *hwh = &req_ctx->hwh;
615 bool cleanup = true;
616 int rc;
617
618 /* prep the walk in the request context */
619 rc = hwh_prepare(req, hwh);
620 if (rc)
621 goto out;
622
623 /* Try synchronous operation if no active engine usage */
624 if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
625 rc = phmac_kmac_update(req, false);
626 if (rc == 0)
627 goto out;
628 }
629
630 /*
631 * If sync operation failed or key expired or there are already
632 * requests enqueued via engine, fallback to async. Mark tfm as
633 * using engine to serialize requests.
634 */
635 if (rc == 0 || rc == -EKEYEXPIRED) {
636 req_ctx->async_op = OP_UPDATE;
637 atomic_inc(&tfm_ctx->via_engine_ctr);
638 rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
639 if (rc == -EINPROGRESS || rc == -EBUSY)
640 cleanup = false;
641 else
642 atomic_dec(&tfm_ctx->via_engine_ctr);
643 }
644
645 if (cleanup) {
646 if (hwh->walkbytes > 0)
647 hwh_advance(hwh, rc);
648 memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
649 }
650
651 out:
652 pr_debug("rc=%d\n", rc);
653 return rc;
654 }
655
phmac_final(struct ahash_request * req)656 static int phmac_final(struct ahash_request *req)
657 {
658 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
659 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
660 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
661 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
662 bool cleanup = true;
663 int rc = 0;
664
665 /* Try synchronous operation if no active engine usage */
666 if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
667 rc = phmac_kmac_final(req, false);
668 if (rc == 0)
669 goto out;
670 }
671
672 /*
673 * If sync operation failed or key expired or there are already
674 * requests enqueued via engine, fallback to async. Mark tfm as
675 * using engine to serialize requests.
676 */
677 if (rc == 0 || rc == -EKEYEXPIRED) {
678 req_ctx->async_op = OP_FINAL;
679 atomic_inc(&tfm_ctx->via_engine_ctr);
680 rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
681 if (rc == -EINPROGRESS || rc == -EBUSY)
682 cleanup = false;
683 else
684 atomic_dec(&tfm_ctx->via_engine_ctr);
685 }
686
687 out:
688 if (cleanup)
689 memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
690 pr_debug("rc=%d\n", rc);
691 return rc;
692 }
693
phmac_finup(struct ahash_request * req)694 static int phmac_finup(struct ahash_request *req)
695 {
696 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
697 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
698 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
699 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
700 struct hash_walk_helper *hwh = &req_ctx->hwh;
701 bool cleanup = true;
702 int rc;
703
704 /* prep the walk in the request context */
705 rc = hwh_prepare(req, hwh);
706 if (rc)
707 goto out;
708
709 req_ctx->async_op = OP_FINUP;
710
711 /* Try synchronous operations if no active engine usage */
712 if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
713 rc = phmac_kmac_update(req, false);
714 if (rc == 0)
715 req_ctx->async_op = OP_FINAL;
716 }
717 if (!rc && req_ctx->async_op == OP_FINAL &&
718 !atomic_read(&tfm_ctx->via_engine_ctr)) {
719 rc = phmac_kmac_final(req, false);
720 if (rc == 0)
721 goto out;
722 }
723
724 /*
725 * If sync operation failed or key expired or there are already
726 * requests enqueued via engine, fallback to async. Mark tfm as
727 * using engine to serialize requests.
728 */
729 if (rc == 0 || rc == -EKEYEXPIRED) {
730 /* req->async_op has been set to either OP_FINUP or OP_FINAL */
731 atomic_inc(&tfm_ctx->via_engine_ctr);
732 rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
733 if (rc == -EINPROGRESS || rc == -EBUSY)
734 cleanup = false;
735 else
736 atomic_dec(&tfm_ctx->via_engine_ctr);
737 }
738
739 if (cleanup && hwh->walkbytes > 0)
740 hwh_advance(hwh, rc);
741
742 out:
743 if (cleanup)
744 memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
745 pr_debug("rc=%d\n", rc);
746 return rc;
747 }
748
phmac_digest(struct ahash_request * req)749 static int phmac_digest(struct ahash_request *req)
750 {
751 int rc;
752
753 rc = phmac_init(req);
754 if (rc)
755 goto out;
756
757 rc = phmac_finup(req);
758
759 out:
760 pr_debug("rc=%d\n", rc);
761 return rc;
762 }
763
phmac_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)764 static int phmac_setkey(struct crypto_ahash *tfm,
765 const u8 *key, unsigned int keylen)
766 {
767 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
768 unsigned int ds = crypto_ahash_digestsize(tfm);
769 unsigned int bs = crypto_ahash_blocksize(tfm);
770 bool tested = crypto_ahash_tested(tfm);
771 unsigned int tmpkeylen;
772 u8 *tmpkey = NULL;
773 int rc = 0;
774
775 if (!tested) {
776 /*
777 * selftest running: key is a raw hmac clear key and needs
778 * to get embedded into a 'clear key token' in order to have
779 * it correctly processed by the pkey module.
780 */
781 tmpkeylen = sizeof(struct hmac_clrkey_token) + bs;
782 tmpkey = kzalloc(tmpkeylen, GFP_KERNEL);
783 if (!tmpkey) {
784 rc = -ENOMEM;
785 goto out;
786 }
787 rc = make_clrkey_token(key, keylen, ds, tmpkey);
788 if (rc)
789 goto out;
790 keylen = tmpkeylen;
791 key = tmpkey;
792 }
793
794 /* copy raw key into tfm context */
795 rc = phmac_tfm_ctx_setkey(tfm_ctx, key, keylen);
796 if (rc)
797 goto out;
798
799 /* convert raw key into protected key */
800 rc = phmac_convert_key(tfm_ctx, tested);
801 if (rc)
802 goto out;
803
804 /* set function code in tfm context, check for valid pk type */
805 switch (ds) {
806 case SHA224_DIGEST_SIZE:
807 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
808 rc = -EINVAL;
809 else
810 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_224;
811 break;
812 case SHA256_DIGEST_SIZE:
813 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
814 rc = -EINVAL;
815 else
816 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_256;
817 break;
818 case SHA384_DIGEST_SIZE:
819 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
820 rc = -EINVAL;
821 else
822 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_384;
823 break;
824 case SHA512_DIGEST_SIZE:
825 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
826 rc = -EINVAL;
827 else
828 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_512;
829 break;
830 default:
831 tfm_ctx->fc = 0;
832 rc = -EINVAL;
833 }
834
835 out:
836 kfree(tmpkey);
837 pr_debug("rc=%d\n", rc);
838 return rc;
839 }
840
phmac_export(struct ahash_request * req,void * out)841 static int phmac_export(struct ahash_request *req, void *out)
842 {
843 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
844 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
845
846 memcpy(out, ctx, sizeof(*ctx));
847
848 return 0;
849 }
850
phmac_import(struct ahash_request * req,const void * in)851 static int phmac_import(struct ahash_request *req, const void *in)
852 {
853 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
854 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
855
856 memset(req_ctx, 0, sizeof(*req_ctx));
857 memcpy(ctx, in, sizeof(*ctx));
858
859 return 0;
860 }
861
phmac_init_tfm(struct crypto_ahash * tfm)862 static int phmac_init_tfm(struct crypto_ahash *tfm)
863 {
864 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
865
866 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
867 spin_lock_init(&tfm_ctx->pk_lock);
868
869 crypto_ahash_set_reqsize(tfm, sizeof(struct phmac_req_ctx));
870
871 return 0;
872 }
873
phmac_exit_tfm(struct crypto_ahash * tfm)874 static void phmac_exit_tfm(struct crypto_ahash *tfm)
875 {
876 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
877
878 memzero_explicit(tfm_ctx->keybuf, sizeof(tfm_ctx->keybuf));
879 memzero_explicit(&tfm_ctx->pk, sizeof(tfm_ctx->pk));
880 }
881
phmac_do_one_request(struct crypto_engine * engine,void * areq)882 static int phmac_do_one_request(struct crypto_engine *engine, void *areq)
883 {
884 struct ahash_request *req = ahash_request_cast(areq);
885 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
886 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
887 struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
888 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
889 struct hash_walk_helper *hwh = &req_ctx->hwh;
890 int rc = -EINVAL;
891
892 /*
893 * Three kinds of requests come in here:
894 * 1. req->async_op == OP_UPDATE with req->nbytes > 0
895 * 2. req->async_op == OP_FINUP with req->nbytes > 0
896 * 3. req->async_op == OP_FINAL
897 * For update and finup the hwh walk has already been prepared
898 * by the caller. For final there is no hwh walk needed.
899 */
900
901 switch (req_ctx->async_op) {
902 case OP_UPDATE:
903 case OP_FINUP:
904 rc = phmac_kmac_update(req, true);
905 if (rc == -EKEYEXPIRED) {
906 return pkey_handle_expired();
907 } else if (rc) {
908 hwh_advance(hwh, rc);
909 goto out;
910 }
911 if (req_ctx->async_op == OP_UPDATE)
912 break;
913 req_ctx->async_op = OP_FINAL;
914 fallthrough;
915 case OP_FINAL:
916 rc = phmac_kmac_final(req, true);
917 if (rc == -EKEYEXPIRED)
918 return pkey_handle_expired();
919 break;
920 default:
921 /* unknown/unsupported/unimplemented asynch op */
922 return -EOPNOTSUPP;
923 }
924
925 out:
926 if (rc || req_ctx->async_op == OP_FINAL)
927 memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
928 pr_debug("request complete with rc=%d\n", rc);
929 local_bh_disable();
930 atomic_dec(&tfm_ctx->via_engine_ctr);
931 crypto_finalize_hash_request(engine, req, rc);
932 local_bh_enable();
933 return 0;
934 }
935
936 #define S390_ASYNC_PHMAC_ALG(x) \
937 { \
938 .base = { \
939 .init = phmac_init, \
940 .update = phmac_update, \
941 .final = phmac_final, \
942 .finup = phmac_finup, \
943 .digest = phmac_digest, \
944 .setkey = phmac_setkey, \
945 .import = phmac_import, \
946 .export = phmac_export, \
947 .init_tfm = phmac_init_tfm, \
948 .exit_tfm = phmac_exit_tfm, \
949 .halg = { \
950 .digestsize = SHA##x##_DIGEST_SIZE, \
951 .statesize = sizeof(struct kmac_sha2_ctx), \
952 .base = { \
953 .cra_name = "phmac(sha" #x ")", \
954 .cra_driver_name = "phmac_s390_sha" #x, \
955 .cra_blocksize = SHA##x##_BLOCK_SIZE, \
956 .cra_priority = 400, \
957 .cra_flags = CRYPTO_ALG_ASYNC | \
958 CRYPTO_ALG_NO_FALLBACK, \
959 .cra_ctxsize = sizeof(struct phmac_tfm_ctx), \
960 .cra_module = THIS_MODULE, \
961 }, \
962 }, \
963 }, \
964 .op = { \
965 .do_one_request = phmac_do_one_request, \
966 }, \
967 }
968
969 static struct phmac_alg {
970 unsigned int fc;
971 struct ahash_engine_alg alg;
972 bool registered;
973 } phmac_algs[] = {
974 {
975 .fc = CPACF_KMAC_PHMAC_SHA_224,
976 .alg = S390_ASYNC_PHMAC_ALG(224),
977 }, {
978 .fc = CPACF_KMAC_PHMAC_SHA_256,
979 .alg = S390_ASYNC_PHMAC_ALG(256),
980 }, {
981 .fc = CPACF_KMAC_PHMAC_SHA_384,
982 .alg = S390_ASYNC_PHMAC_ALG(384),
983 }, {
984 .fc = CPACF_KMAC_PHMAC_SHA_512,
985 .alg = S390_ASYNC_PHMAC_ALG(512),
986 }
987 };
988
989 static struct miscdevice phmac_dev = {
990 .name = "phmac",
991 .minor = MISC_DYNAMIC_MINOR,
992 };
993
s390_phmac_exit(void)994 static void s390_phmac_exit(void)
995 {
996 struct phmac_alg *phmac;
997 int i;
998
999 if (phmac_crypto_engine) {
1000 crypto_engine_stop(phmac_crypto_engine);
1001 crypto_engine_exit(phmac_crypto_engine);
1002 }
1003
1004 for (i = ARRAY_SIZE(phmac_algs) - 1; i >= 0; i--) {
1005 phmac = &phmac_algs[i];
1006 if (phmac->registered)
1007 crypto_engine_unregister_ahash(&phmac->alg);
1008 }
1009
1010 misc_deregister(&phmac_dev);
1011 }
1012
s390_phmac_init(void)1013 static int __init s390_phmac_init(void)
1014 {
1015 struct phmac_alg *phmac;
1016 int i, rc;
1017
1018 /* for selftest cpacf klmd subfunction is needed */
1019 if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_256))
1020 return -ENODEV;
1021 if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_512))
1022 return -ENODEV;
1023
1024 /* register a simple phmac pseudo misc device */
1025 rc = misc_register(&phmac_dev);
1026 if (rc)
1027 return rc;
1028
1029 /* with this pseudo device alloc and start a crypto engine */
1030 phmac_crypto_engine =
1031 crypto_engine_alloc_init_and_set(phmac_dev.this_device,
1032 true, false, MAX_QLEN);
1033 if (!phmac_crypto_engine) {
1034 rc = -ENOMEM;
1035 goto out_err;
1036 }
1037 rc = crypto_engine_start(phmac_crypto_engine);
1038 if (rc) {
1039 crypto_engine_exit(phmac_crypto_engine);
1040 phmac_crypto_engine = NULL;
1041 goto out_err;
1042 }
1043
1044 for (i = 0; i < ARRAY_SIZE(phmac_algs); i++) {
1045 phmac = &phmac_algs[i];
1046 if (!cpacf_query_func(CPACF_KMAC, phmac->fc))
1047 continue;
1048 rc = crypto_engine_register_ahash(&phmac->alg);
1049 if (rc)
1050 goto out_err;
1051 phmac->registered = true;
1052 pr_debug("%s registered\n", phmac->alg.base.halg.base.cra_name);
1053 }
1054
1055 return 0;
1056
1057 out_err:
1058 s390_phmac_exit();
1059 return rc;
1060 }
1061
1062 module_init(s390_phmac_init);
1063 module_exit(s390_phmac_exit);
1064
1065 MODULE_ALIAS_CRYPTO("phmac(sha224)");
1066 MODULE_ALIAS_CRYPTO("phmac(sha256)");
1067 MODULE_ALIAS_CRYPTO("phmac(sha384)");
1068 MODULE_ALIAS_CRYPTO("phmac(sha512)");
1069
1070 MODULE_DESCRIPTION("S390 HMAC driver for protected keys");
1071 MODULE_LICENSE("GPL");
1072