1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * s390 implementation of the AES Cipher Algorithm with protected keys. 6 * 7 * s390 Version: 8 * Copyright IBM Corp. 2017, 2023 9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 10 * Harald Freudenberger <freude@de.ibm.com> 11 */ 12 13 #define KMSG_COMPONENT "paes_s390" 14 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 15 16 #include <crypto/aes.h> 17 #include <crypto/algapi.h> 18 #include <linux/bug.h> 19 #include <linux/err.h> 20 #include <linux/module.h> 21 #include <linux/cpufeature.h> 22 #include <linux/init.h> 23 #include <linux/mutex.h> 24 #include <linux/spinlock.h> 25 #include <linux/delay.h> 26 #include <crypto/internal/skcipher.h> 27 #include <crypto/xts.h> 28 #include <asm/cpacf.h> 29 #include <asm/pkey.h> 30 31 /* 32 * Key blobs smaller/bigger than these defines are rejected 33 * by the common code even before the individual setkey function 34 * is called. As paes can handle different kinds of key blobs 35 * and padding is also possible, the limits need to be generous. 36 */ 37 #define PAES_MIN_KEYSIZE 16 38 #define PAES_MAX_KEYSIZE MAXEP11AESKEYBLOBSIZE 39 40 static u8 *ctrblk; 41 static DEFINE_MUTEX(ctrblk_lock); 42 43 static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; 44 45 struct key_blob { 46 /* 47 * Small keys will be stored in the keybuf. Larger keys are 48 * stored in extra allocated memory. In both cases does 49 * key point to the memory where the key is stored. 50 * The code distinguishes by checking keylen against 51 * sizeof(keybuf). See the two following helper functions. 52 */ 53 u8 *key; 54 u8 keybuf[128]; 55 unsigned int keylen; 56 }; 57 58 static inline int _key_to_kb(struct key_blob *kb, 59 const u8 *key, 60 unsigned int keylen) 61 { 62 struct clearkey_header { 63 u8 type; 64 u8 res0[3]; 65 u8 version; 66 u8 res1[3]; 67 u32 keytype; 68 u32 len; 69 } __packed * h; 70 71 switch (keylen) { 72 case 16: 73 case 24: 74 case 32: 75 /* clear key value, prepare pkey clear key token in keybuf */ 76 memset(kb->keybuf, 0, sizeof(kb->keybuf)); 77 h = (struct clearkey_header *) kb->keybuf; 78 h->version = 0x02; /* TOKVER_CLEAR_KEY */ 79 h->keytype = (keylen - 8) >> 3; 80 h->len = keylen; 81 memcpy(kb->keybuf + sizeof(*h), key, keylen); 82 kb->keylen = sizeof(*h) + keylen; 83 kb->key = kb->keybuf; 84 break; 85 default: 86 /* other key material, let pkey handle this */ 87 if (keylen <= sizeof(kb->keybuf)) 88 kb->key = kb->keybuf; 89 else { 90 kb->key = kmalloc(keylen, GFP_KERNEL); 91 if (!kb->key) 92 return -ENOMEM; 93 } 94 memcpy(kb->key, key, keylen); 95 kb->keylen = keylen; 96 break; 97 } 98 99 return 0; 100 } 101 102 static inline void _free_kb_keybuf(struct key_blob *kb) 103 { 104 if (kb->key && kb->key != kb->keybuf 105 && kb->keylen > sizeof(kb->keybuf)) { 106 kfree_sensitive(kb->key); 107 kb->key = NULL; 108 } 109 } 110 111 struct s390_paes_ctx { 112 struct key_blob kb; 113 struct pkey_protkey pk; 114 spinlock_t pk_lock; 115 unsigned long fc; 116 }; 117 118 struct s390_pxts_ctx { 119 struct key_blob kb[2]; 120 struct pkey_protkey pk[2]; 121 spinlock_t pk_lock; 122 unsigned long fc; 123 }; 124 125 static inline int __paes_keyblob2pkey(struct key_blob *kb, 126 struct pkey_protkey *pk) 127 { 128 return pkey_keyblob2pkey(kb->key, kb->keylen, 129 pk->protkey, &pk->len, &pk->type); 130 } 131 132 static inline int __paes_convert_key(struct s390_paes_ctx *ctx) 133 { 134 int ret; 135 struct pkey_protkey pkey; 136 137 pkey.len = sizeof(pkey.protkey); 138 ret = __paes_keyblob2pkey(&ctx->kb, &pkey); 139 if (ret) 140 return ret; 141 142 spin_lock_bh(&ctx->pk_lock); 143 memcpy(&ctx->pk, &pkey, sizeof(pkey)); 144 spin_unlock_bh(&ctx->pk_lock); 145 146 return 0; 147 } 148 149 static int ecb_paes_init(struct crypto_skcipher *tfm) 150 { 151 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 152 153 ctx->kb.key = NULL; 154 spin_lock_init(&ctx->pk_lock); 155 156 return 0; 157 } 158 159 static void ecb_paes_exit(struct crypto_skcipher *tfm) 160 { 161 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 162 163 _free_kb_keybuf(&ctx->kb); 164 } 165 166 static inline int __ecb_paes_set_key(struct s390_paes_ctx *ctx) 167 { 168 int rc; 169 unsigned long fc; 170 171 rc = __paes_convert_key(ctx); 172 if (rc) 173 return rc; 174 175 /* Pick the correct function code based on the protected key type */ 176 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 : 177 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 : 178 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0; 179 180 /* Check if the function code is available */ 181 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 182 183 return ctx->fc ? 0 : -EINVAL; 184 } 185 186 static int ecb_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 187 unsigned int key_len) 188 { 189 int rc; 190 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 191 192 _free_kb_keybuf(&ctx->kb); 193 rc = _key_to_kb(&ctx->kb, in_key, key_len); 194 if (rc) 195 return rc; 196 197 return __ecb_paes_set_key(ctx); 198 } 199 200 static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) 201 { 202 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 203 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 204 struct skcipher_walk walk; 205 unsigned int nbytes, n, k; 206 int ret; 207 struct { 208 u8 key[MAXPROTKEYSIZE]; 209 } param; 210 211 ret = skcipher_walk_virt(&walk, req, false); 212 if (ret) 213 return ret; 214 215 spin_lock_bh(&ctx->pk_lock); 216 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 217 spin_unlock_bh(&ctx->pk_lock); 218 219 while ((nbytes = walk.nbytes) != 0) { 220 /* only use complete blocks */ 221 n = nbytes & ~(AES_BLOCK_SIZE - 1); 222 k = cpacf_km(ctx->fc | modifier, ¶m, 223 walk.dst.virt.addr, walk.src.virt.addr, n); 224 if (k) 225 ret = skcipher_walk_done(&walk, nbytes - k); 226 if (k < n) { 227 if (__paes_convert_key(ctx)) 228 return skcipher_walk_done(&walk, -EIO); 229 spin_lock_bh(&ctx->pk_lock); 230 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 231 spin_unlock_bh(&ctx->pk_lock); 232 } 233 } 234 return ret; 235 } 236 237 static int ecb_paes_encrypt(struct skcipher_request *req) 238 { 239 return ecb_paes_crypt(req, 0); 240 } 241 242 static int ecb_paes_decrypt(struct skcipher_request *req) 243 { 244 return ecb_paes_crypt(req, CPACF_DECRYPT); 245 } 246 247 static struct skcipher_alg ecb_paes_alg = { 248 .base.cra_name = "ecb(paes)", 249 .base.cra_driver_name = "ecb-paes-s390", 250 .base.cra_priority = 401, /* combo: aes + ecb + 1 */ 251 .base.cra_blocksize = AES_BLOCK_SIZE, 252 .base.cra_ctxsize = sizeof(struct s390_paes_ctx), 253 .base.cra_module = THIS_MODULE, 254 .base.cra_list = LIST_HEAD_INIT(ecb_paes_alg.base.cra_list), 255 .init = ecb_paes_init, 256 .exit = ecb_paes_exit, 257 .min_keysize = PAES_MIN_KEYSIZE, 258 .max_keysize = PAES_MAX_KEYSIZE, 259 .setkey = ecb_paes_set_key, 260 .encrypt = ecb_paes_encrypt, 261 .decrypt = ecb_paes_decrypt, 262 }; 263 264 static int cbc_paes_init(struct crypto_skcipher *tfm) 265 { 266 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 267 268 ctx->kb.key = NULL; 269 spin_lock_init(&ctx->pk_lock); 270 271 return 0; 272 } 273 274 static void cbc_paes_exit(struct crypto_skcipher *tfm) 275 { 276 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 277 278 _free_kb_keybuf(&ctx->kb); 279 } 280 281 static inline int __cbc_paes_set_key(struct s390_paes_ctx *ctx) 282 { 283 int rc; 284 unsigned long fc; 285 286 rc = __paes_convert_key(ctx); 287 if (rc) 288 return rc; 289 290 /* Pick the correct function code based on the protected key type */ 291 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMC_PAES_128 : 292 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMC_PAES_192 : 293 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KMC_PAES_256 : 0; 294 295 /* Check if the function code is available */ 296 ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0; 297 298 return ctx->fc ? 0 : -EINVAL; 299 } 300 301 static int cbc_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 302 unsigned int key_len) 303 { 304 int rc; 305 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 306 307 _free_kb_keybuf(&ctx->kb); 308 rc = _key_to_kb(&ctx->kb, in_key, key_len); 309 if (rc) 310 return rc; 311 312 return __cbc_paes_set_key(ctx); 313 } 314 315 static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) 316 { 317 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 318 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 319 struct skcipher_walk walk; 320 unsigned int nbytes, n, k; 321 int ret; 322 struct { 323 u8 iv[AES_BLOCK_SIZE]; 324 u8 key[MAXPROTKEYSIZE]; 325 } param; 326 327 ret = skcipher_walk_virt(&walk, req, false); 328 if (ret) 329 return ret; 330 331 memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); 332 spin_lock_bh(&ctx->pk_lock); 333 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 334 spin_unlock_bh(&ctx->pk_lock); 335 336 while ((nbytes = walk.nbytes) != 0) { 337 /* only use complete blocks */ 338 n = nbytes & ~(AES_BLOCK_SIZE - 1); 339 k = cpacf_kmc(ctx->fc | modifier, ¶m, 340 walk.dst.virt.addr, walk.src.virt.addr, n); 341 if (k) { 342 memcpy(walk.iv, param.iv, AES_BLOCK_SIZE); 343 ret = skcipher_walk_done(&walk, nbytes - k); 344 } 345 if (k < n) { 346 if (__paes_convert_key(ctx)) 347 return skcipher_walk_done(&walk, -EIO); 348 spin_lock_bh(&ctx->pk_lock); 349 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 350 spin_unlock_bh(&ctx->pk_lock); 351 } 352 } 353 return ret; 354 } 355 356 static int cbc_paes_encrypt(struct skcipher_request *req) 357 { 358 return cbc_paes_crypt(req, 0); 359 } 360 361 static int cbc_paes_decrypt(struct skcipher_request *req) 362 { 363 return cbc_paes_crypt(req, CPACF_DECRYPT); 364 } 365 366 static struct skcipher_alg cbc_paes_alg = { 367 .base.cra_name = "cbc(paes)", 368 .base.cra_driver_name = "cbc-paes-s390", 369 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ 370 .base.cra_blocksize = AES_BLOCK_SIZE, 371 .base.cra_ctxsize = sizeof(struct s390_paes_ctx), 372 .base.cra_module = THIS_MODULE, 373 .base.cra_list = LIST_HEAD_INIT(cbc_paes_alg.base.cra_list), 374 .init = cbc_paes_init, 375 .exit = cbc_paes_exit, 376 .min_keysize = PAES_MIN_KEYSIZE, 377 .max_keysize = PAES_MAX_KEYSIZE, 378 .ivsize = AES_BLOCK_SIZE, 379 .setkey = cbc_paes_set_key, 380 .encrypt = cbc_paes_encrypt, 381 .decrypt = cbc_paes_decrypt, 382 }; 383 384 static int xts_paes_init(struct crypto_skcipher *tfm) 385 { 386 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 387 388 ctx->kb[0].key = NULL; 389 ctx->kb[1].key = NULL; 390 spin_lock_init(&ctx->pk_lock); 391 392 return 0; 393 } 394 395 static void xts_paes_exit(struct crypto_skcipher *tfm) 396 { 397 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 398 399 _free_kb_keybuf(&ctx->kb[0]); 400 _free_kb_keybuf(&ctx->kb[1]); 401 } 402 403 static inline int __xts_paes_convert_key(struct s390_pxts_ctx *ctx) 404 { 405 struct pkey_protkey pkey0, pkey1; 406 407 pkey0.len = sizeof(pkey0.protkey); 408 pkey1.len = sizeof(pkey1.protkey); 409 410 if (__paes_keyblob2pkey(&ctx->kb[0], &pkey0) || 411 __paes_keyblob2pkey(&ctx->kb[1], &pkey1)) 412 return -EINVAL; 413 414 spin_lock_bh(&ctx->pk_lock); 415 memcpy(&ctx->pk[0], &pkey0, sizeof(pkey0)); 416 memcpy(&ctx->pk[1], &pkey1, sizeof(pkey1)); 417 spin_unlock_bh(&ctx->pk_lock); 418 419 return 0; 420 } 421 422 static inline int __xts_paes_set_key(struct s390_pxts_ctx *ctx) 423 { 424 unsigned long fc; 425 426 if (__xts_paes_convert_key(ctx)) 427 return -EINVAL; 428 429 if (ctx->pk[0].type != ctx->pk[1].type) 430 return -EINVAL; 431 432 /* Pick the correct function code based on the protected key type */ 433 fc = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PXTS_128 : 434 (ctx->pk[0].type == PKEY_KEYTYPE_AES_256) ? 435 CPACF_KM_PXTS_256 : 0; 436 437 /* Check if the function code is available */ 438 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 439 440 return ctx->fc ? 0 : -EINVAL; 441 } 442 443 static int xts_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 444 unsigned int xts_key_len) 445 { 446 int rc; 447 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 448 u8 ckey[2 * AES_MAX_KEY_SIZE]; 449 unsigned int ckey_len, key_len; 450 451 if (xts_key_len % 2) 452 return -EINVAL; 453 454 key_len = xts_key_len / 2; 455 456 _free_kb_keybuf(&ctx->kb[0]); 457 _free_kb_keybuf(&ctx->kb[1]); 458 rc = _key_to_kb(&ctx->kb[0], in_key, key_len); 459 if (rc) 460 return rc; 461 rc = _key_to_kb(&ctx->kb[1], in_key + key_len, key_len); 462 if (rc) 463 return rc; 464 465 rc = __xts_paes_set_key(ctx); 466 if (rc) 467 return rc; 468 469 /* 470 * xts_verify_key verifies the key length is not odd and makes 471 * sure that the two keys are not the same. This can be done 472 * on the two protected keys as well 473 */ 474 ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 475 AES_KEYSIZE_128 : AES_KEYSIZE_256; 476 memcpy(ckey, ctx->pk[0].protkey, ckey_len); 477 memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len); 478 return xts_verify_key(tfm, ckey, 2*ckey_len); 479 } 480 481 static int xts_paes_crypt(struct skcipher_request *req, unsigned long modifier) 482 { 483 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 484 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 485 struct skcipher_walk walk; 486 unsigned int keylen, offset, nbytes, n, k; 487 int ret; 488 struct { 489 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 490 u8 tweak[16]; 491 u8 block[16]; 492 u8 bit[16]; 493 u8 xts[16]; 494 } pcc_param; 495 struct { 496 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 497 u8 init[16]; 498 } xts_param; 499 500 ret = skcipher_walk_virt(&walk, req, false); 501 if (ret) 502 return ret; 503 504 keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64; 505 offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0; 506 507 memset(&pcc_param, 0, sizeof(pcc_param)); 508 memcpy(pcc_param.tweak, walk.iv, sizeof(pcc_param.tweak)); 509 spin_lock_bh(&ctx->pk_lock); 510 memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen); 511 memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen); 512 spin_unlock_bh(&ctx->pk_lock); 513 cpacf_pcc(ctx->fc, pcc_param.key + offset); 514 memcpy(xts_param.init, pcc_param.xts, 16); 515 516 while ((nbytes = walk.nbytes) != 0) { 517 /* only use complete blocks */ 518 n = nbytes & ~(AES_BLOCK_SIZE - 1); 519 k = cpacf_km(ctx->fc | modifier, xts_param.key + offset, 520 walk.dst.virt.addr, walk.src.virt.addr, n); 521 if (k) 522 ret = skcipher_walk_done(&walk, nbytes - k); 523 if (k < n) { 524 if (__xts_paes_convert_key(ctx)) 525 return skcipher_walk_done(&walk, -EIO); 526 spin_lock_bh(&ctx->pk_lock); 527 memcpy(xts_param.key + offset, 528 ctx->pk[0].protkey, keylen); 529 spin_unlock_bh(&ctx->pk_lock); 530 } 531 } 532 533 return ret; 534 } 535 536 static int xts_paes_encrypt(struct skcipher_request *req) 537 { 538 return xts_paes_crypt(req, 0); 539 } 540 541 static int xts_paes_decrypt(struct skcipher_request *req) 542 { 543 return xts_paes_crypt(req, CPACF_DECRYPT); 544 } 545 546 static struct skcipher_alg xts_paes_alg = { 547 .base.cra_name = "xts(paes)", 548 .base.cra_driver_name = "xts-paes-s390", 549 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ 550 .base.cra_blocksize = AES_BLOCK_SIZE, 551 .base.cra_ctxsize = sizeof(struct s390_pxts_ctx), 552 .base.cra_module = THIS_MODULE, 553 .base.cra_list = LIST_HEAD_INIT(xts_paes_alg.base.cra_list), 554 .init = xts_paes_init, 555 .exit = xts_paes_exit, 556 .min_keysize = 2 * PAES_MIN_KEYSIZE, 557 .max_keysize = 2 * PAES_MAX_KEYSIZE, 558 .ivsize = AES_BLOCK_SIZE, 559 .setkey = xts_paes_set_key, 560 .encrypt = xts_paes_encrypt, 561 .decrypt = xts_paes_decrypt, 562 }; 563 564 static int ctr_paes_init(struct crypto_skcipher *tfm) 565 { 566 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 567 568 ctx->kb.key = NULL; 569 spin_lock_init(&ctx->pk_lock); 570 571 return 0; 572 } 573 574 static void ctr_paes_exit(struct crypto_skcipher *tfm) 575 { 576 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 577 578 _free_kb_keybuf(&ctx->kb); 579 } 580 581 static inline int __ctr_paes_set_key(struct s390_paes_ctx *ctx) 582 { 583 int rc; 584 unsigned long fc; 585 586 rc = __paes_convert_key(ctx); 587 if (rc) 588 return rc; 589 590 /* Pick the correct function code based on the protected key type */ 591 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMCTR_PAES_128 : 592 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMCTR_PAES_192 : 593 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? 594 CPACF_KMCTR_PAES_256 : 0; 595 596 /* Check if the function code is available */ 597 ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0; 598 599 return ctx->fc ? 0 : -EINVAL; 600 } 601 602 static int ctr_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 603 unsigned int key_len) 604 { 605 int rc; 606 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 607 608 _free_kb_keybuf(&ctx->kb); 609 rc = _key_to_kb(&ctx->kb, in_key, key_len); 610 if (rc) 611 return rc; 612 613 return __ctr_paes_set_key(ctx); 614 } 615 616 static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) 617 { 618 unsigned int i, n; 619 620 /* only use complete blocks, max. PAGE_SIZE */ 621 memcpy(ctrptr, iv, AES_BLOCK_SIZE); 622 n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1); 623 for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) { 624 memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE); 625 crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE); 626 ctrptr += AES_BLOCK_SIZE; 627 } 628 return n; 629 } 630 631 static int ctr_paes_crypt(struct skcipher_request *req) 632 { 633 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 634 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 635 u8 buf[AES_BLOCK_SIZE], *ctrptr; 636 struct skcipher_walk walk; 637 unsigned int nbytes, n, k; 638 int ret, locked; 639 struct { 640 u8 key[MAXPROTKEYSIZE]; 641 } param; 642 643 ret = skcipher_walk_virt(&walk, req, false); 644 if (ret) 645 return ret; 646 647 spin_lock_bh(&ctx->pk_lock); 648 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 649 spin_unlock_bh(&ctx->pk_lock); 650 651 locked = mutex_trylock(&ctrblk_lock); 652 653 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 654 n = AES_BLOCK_SIZE; 655 if (nbytes >= 2*AES_BLOCK_SIZE && locked) 656 n = __ctrblk_init(ctrblk, walk.iv, nbytes); 657 ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; 658 k = cpacf_kmctr(ctx->fc, ¶m, walk.dst.virt.addr, 659 walk.src.virt.addr, n, ctrptr); 660 if (k) { 661 if (ctrptr == ctrblk) 662 memcpy(walk.iv, ctrptr + k - AES_BLOCK_SIZE, 663 AES_BLOCK_SIZE); 664 crypto_inc(walk.iv, AES_BLOCK_SIZE); 665 ret = skcipher_walk_done(&walk, nbytes - k); 666 } 667 if (k < n) { 668 if (__paes_convert_key(ctx)) { 669 if (locked) 670 mutex_unlock(&ctrblk_lock); 671 return skcipher_walk_done(&walk, -EIO); 672 } 673 spin_lock_bh(&ctx->pk_lock); 674 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 675 spin_unlock_bh(&ctx->pk_lock); 676 } 677 } 678 if (locked) 679 mutex_unlock(&ctrblk_lock); 680 /* 681 * final block may be < AES_BLOCK_SIZE, copy only nbytes 682 */ 683 if (nbytes) { 684 memset(buf, 0, AES_BLOCK_SIZE); 685 memcpy(buf, walk.src.virt.addr, nbytes); 686 while (1) { 687 if (cpacf_kmctr(ctx->fc, ¶m, buf, 688 buf, AES_BLOCK_SIZE, 689 walk.iv) == AES_BLOCK_SIZE) 690 break; 691 if (__paes_convert_key(ctx)) 692 return skcipher_walk_done(&walk, -EIO); 693 spin_lock_bh(&ctx->pk_lock); 694 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 695 spin_unlock_bh(&ctx->pk_lock); 696 } 697 memcpy(walk.dst.virt.addr, buf, nbytes); 698 crypto_inc(walk.iv, AES_BLOCK_SIZE); 699 ret = skcipher_walk_done(&walk, nbytes); 700 } 701 702 return ret; 703 } 704 705 static struct skcipher_alg ctr_paes_alg = { 706 .base.cra_name = "ctr(paes)", 707 .base.cra_driver_name = "ctr-paes-s390", 708 .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ 709 .base.cra_blocksize = 1, 710 .base.cra_ctxsize = sizeof(struct s390_paes_ctx), 711 .base.cra_module = THIS_MODULE, 712 .base.cra_list = LIST_HEAD_INIT(ctr_paes_alg.base.cra_list), 713 .init = ctr_paes_init, 714 .exit = ctr_paes_exit, 715 .min_keysize = PAES_MIN_KEYSIZE, 716 .max_keysize = PAES_MAX_KEYSIZE, 717 .ivsize = AES_BLOCK_SIZE, 718 .setkey = ctr_paes_set_key, 719 .encrypt = ctr_paes_crypt, 720 .decrypt = ctr_paes_crypt, 721 .chunksize = AES_BLOCK_SIZE, 722 }; 723 724 static inline void __crypto_unregister_skcipher(struct skcipher_alg *alg) 725 { 726 if (!list_empty(&alg->base.cra_list)) 727 crypto_unregister_skcipher(alg); 728 } 729 730 static void paes_s390_fini(void) 731 { 732 __crypto_unregister_skcipher(&ctr_paes_alg); 733 __crypto_unregister_skcipher(&xts_paes_alg); 734 __crypto_unregister_skcipher(&cbc_paes_alg); 735 __crypto_unregister_skcipher(&ecb_paes_alg); 736 if (ctrblk) 737 free_page((unsigned long) ctrblk); 738 } 739 740 static int __init paes_s390_init(void) 741 { 742 int ret; 743 744 /* Query available functions for KM, KMC and KMCTR */ 745 cpacf_query(CPACF_KM, &km_functions); 746 cpacf_query(CPACF_KMC, &kmc_functions); 747 cpacf_query(CPACF_KMCTR, &kmctr_functions); 748 749 if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) || 750 cpacf_test_func(&km_functions, CPACF_KM_PAES_192) || 751 cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) { 752 ret = crypto_register_skcipher(&ecb_paes_alg); 753 if (ret) 754 goto out_err; 755 } 756 757 if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) || 758 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) || 759 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) { 760 ret = crypto_register_skcipher(&cbc_paes_alg); 761 if (ret) 762 goto out_err; 763 } 764 765 if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) || 766 cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) { 767 ret = crypto_register_skcipher(&xts_paes_alg); 768 if (ret) 769 goto out_err; 770 } 771 772 if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) || 773 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) || 774 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) { 775 ctrblk = (u8 *) __get_free_page(GFP_KERNEL); 776 if (!ctrblk) { 777 ret = -ENOMEM; 778 goto out_err; 779 } 780 ret = crypto_register_skcipher(&ctr_paes_alg); 781 if (ret) 782 goto out_err; 783 } 784 785 return 0; 786 out_err: 787 paes_s390_fini(); 788 return ret; 789 } 790 791 module_init(paes_s390_init); 792 module_exit(paes_s390_fini); 793 794 MODULE_ALIAS_CRYPTO("paes"); 795 796 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys"); 797 MODULE_LICENSE("GPL"); 798