1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AES CCM routines supporting the Power 7+ Nest Accelerators driver 4 * 5 * Copyright (C) 2012 International Business Machines Inc. 6 * 7 * Author: Kent Yoder <yoder1@us.ibm.com> 8 */ 9 10 #include <crypto/internal/aead.h> 11 #include <crypto/aes.h> 12 #include <crypto/algapi.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/module.h> 15 #include <linux/types.h> 16 #include <linux/crypto.h> 17 #include <asm/vio.h> 18 19 #include "nx_csbcpb.h" 20 #include "nx.h" 21 22 23 static int ccm_aes_nx_set_key(struct crypto_aead *tfm, 24 const u8 *in_key, 25 unsigned int key_len) 26 { 27 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); 28 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 29 struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; 30 31 nx_ctx_init(nx_ctx, HCOP_FC_AES); 32 33 switch (key_len) { 34 case AES_KEYSIZE_128: 35 NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); 36 NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128); 37 nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; 38 break; 39 default: 40 return -EINVAL; 41 } 42 43 csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM; 44 memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len); 45 46 csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA; 47 memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len); 48 49 return 0; 50 51 } 52 53 static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm, 54 const u8 *in_key, 55 unsigned int key_len) 56 { 57 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); 58 59 if (key_len < 3) 60 return -EINVAL; 61 62 key_len -= 3; 63 64 memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3); 65 66 return ccm_aes_nx_set_key(tfm, in_key, key_len); 67 } 68 69 static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm, 70 unsigned int authsize) 71 { 72 switch (authsize) { 73 case 4: 74 case 6: 75 case 8: 76 case 10: 77 case 12: 78 case 14: 79 case 16: 80 break; 81 default: 82 return -EINVAL; 83 } 84 85 return 0; 86 } 87 88 static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm, 89 unsigned int authsize) 90 { 91 switch (authsize) { 92 case 8: 93 case 12: 94 case 16: 95 break; 96 default: 97 return -EINVAL; 98 } 99 100 return 0; 101 } 102 103 /* taken from crypto/ccm.c */ 104 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 105 { 106 __be32 data; 107 108 memset(block, 0, csize); 109 block += csize; 110 111 if (csize >= 4) 112 csize = 4; 113 else if (msglen > (unsigned int)(1 << (8 * csize))) 114 return -EOVERFLOW; 115 116 data = cpu_to_be32(msglen); 117 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 118 119 return 0; 120 } 121 122 /* taken from crypto/ccm.c */ 123 static inline int crypto_ccm_check_iv(const u8 *iv) 124 { 125 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 126 if (1 > iv[0] || iv[0] > 7) 127 return -EINVAL; 128 129 return 0; 130 } 131 132 /* based on code from crypto/ccm.c */ 133 static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize, 134 unsigned int cryptlen, u8 *b0) 135 { 136 unsigned int l, lp, m = authsize; 137 138 memcpy(b0, iv, 16); 139 140 lp = b0[0]; 141 l = lp + 1; 142 143 /* set m, bits 3-5 */ 144 *b0 |= (8 * ((m - 2) / 2)); 145 146 /* set adata, bit 6, if associated data is used */ 147 if (assoclen) 148 *b0 |= 64; 149 150 return set_msg_len(b0 + 16 - l, cryptlen, l); 151 } 152 153 static int generate_pat(u8 *iv, 154 struct aead_request *req, 155 struct nx_crypto_ctx *nx_ctx, 156 unsigned int authsize, 157 unsigned int nbytes, 158 unsigned int assoclen, 159 u8 *out) 160 { 161 struct nx_sg *nx_insg = nx_ctx->in_sg; 162 struct nx_sg *nx_outsg = nx_ctx->out_sg; 163 unsigned int iauth_len = 0; 164 u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL; 165 int rc; 166 unsigned int max_sg_len; 167 168 /* zero the ctr value */ 169 memset(iv + 15 - iv[0], 0, iv[0] + 1); 170 171 /* page 78 of nx_wb.pdf has, 172 * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes 173 * in length. If a full message is used, the AES CCA implementation 174 * restricts the maximum AAD length to 2^32 -1 bytes. 175 * If partial messages are used, the implementation supports 176 * 2^64 -1 bytes maximum AAD length. 177 * 178 * However, in the cryptoapi's aead_request structure, 179 * assoclen is an unsigned int, thus it cannot hold a length 180 * value greater than 2^32 - 1. 181 * Thus the AAD is further constrained by this and is never 182 * greater than 2^32. 183 */ 184 185 if (!assoclen) { 186 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; 187 } else if (assoclen <= 14) { 188 /* if associated data is 14 bytes or less, we do 1 GCM 189 * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1, 190 * which is fed in through the source buffers here */ 191 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; 192 b1 = nx_ctx->priv.ccm.iauth_tag; 193 iauth_len = assoclen; 194 } else if (assoclen <= 65280) { 195 /* if associated data is less than (2^16 - 2^8), we construct 196 * B1 differently and feed in the associated data to a CCA 197 * operation */ 198 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0; 199 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1; 200 iauth_len = 14; 201 } else { 202 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0; 203 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1; 204 iauth_len = 10; 205 } 206 207 /* generate B0 */ 208 rc = generate_b0(iv, assoclen, authsize, nbytes, b0); 209 if (rc) 210 return rc; 211 212 /* generate B1: 213 * add control info for associated data 214 * RFC 3610 and NIST Special Publication 800-38C 215 */ 216 if (b1) { 217 memset(b1, 0, 16); 218 if (assoclen <= 65280) { 219 *(u16 *)b1 = assoclen; 220 scatterwalk_map_and_copy(b1 + 2, req->src, 0, 221 iauth_len, SCATTERWALK_FROM_SG); 222 } else { 223 *(u16 *)b1 = (u16)(0xfffe); 224 *(u32 *)&b1[2] = assoclen; 225 scatterwalk_map_and_copy(b1 + 6, req->src, 0, 226 iauth_len, SCATTERWALK_FROM_SG); 227 } 228 } 229 230 /* now copy any remaining AAD to scatterlist and call nx... */ 231 if (!assoclen) { 232 return rc; 233 } else if (assoclen <= 14) { 234 unsigned int len = 16; 235 236 nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen); 237 238 if (len != 16) 239 return -EINVAL; 240 241 nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len, 242 nx_ctx->ap->sglen); 243 244 if (len != 16) 245 return -EINVAL; 246 247 /* inlen should be negative, indicating to phyp that its a 248 * pointer to an sg list */ 249 nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) * 250 sizeof(struct nx_sg); 251 nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) * 252 sizeof(struct nx_sg); 253 254 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT; 255 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE; 256 257 result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac; 258 259 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 260 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 261 if (rc) 262 return rc; 263 264 atomic_inc(&(nx_ctx->stats->aes_ops)); 265 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); 266 267 } else { 268 unsigned int processed = 0, to_process; 269 270 processed += iauth_len; 271 272 /* page_limit: number of sg entries that fit on one page */ 273 max_sg_len = min_t(u64, nx_ctx->ap->sglen, 274 nx_driver.of.max_sg_len/sizeof(struct nx_sg)); 275 max_sg_len = min_t(u64, max_sg_len, 276 nx_ctx->ap->databytelen/NX_PAGE_SIZE); 277 278 do { 279 to_process = min_t(u32, assoclen - processed, 280 nx_ctx->ap->databytelen); 281 282 nx_insg = nx_walk_and_build(nx_ctx->in_sg, 283 nx_ctx->ap->sglen, 284 req->src, processed, 285 &to_process); 286 287 if ((to_process + processed) < assoclen) { 288 NX_CPB_FDM(nx_ctx->csbcpb_aead) |= 289 NX_FDM_INTERMEDIATE; 290 } else { 291 NX_CPB_FDM(nx_ctx->csbcpb_aead) &= 292 ~NX_FDM_INTERMEDIATE; 293 } 294 295 296 nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) * 297 sizeof(struct nx_sg); 298 299 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; 300 301 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead, 302 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 303 if (rc) 304 return rc; 305 306 memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0, 307 nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0, 308 AES_BLOCK_SIZE); 309 310 NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION; 311 312 atomic_inc(&(nx_ctx->stats->aes_ops)); 313 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); 314 315 processed += to_process; 316 } while (processed < assoclen); 317 318 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; 319 } 320 321 memcpy(out, result, AES_BLOCK_SIZE); 322 323 return rc; 324 } 325 326 static int ccm_nx_decrypt(struct aead_request *req, 327 u8 *iv, 328 unsigned int assoclen) 329 { 330 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 331 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 332 unsigned int nbytes = req->cryptlen; 333 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 334 struct nx_ccm_priv *priv = &nx_ctx->priv.ccm; 335 unsigned long irq_flags; 336 unsigned int processed = 0, to_process; 337 int rc = -1; 338 339 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 340 341 nbytes -= authsize; 342 343 /* copy out the auth tag to compare with later */ 344 scatterwalk_map_and_copy(priv->oauth_tag, 345 req->src, nbytes + req->assoclen, authsize, 346 SCATTERWALK_FROM_SG); 347 348 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen, 349 csbcpb->cpb.aes_ccm.in_pat_or_b0); 350 if (rc) 351 goto out; 352 353 do { 354 355 /* to_process: the AES_BLOCK_SIZE data chunk to process in this 356 * update. This value is bound by sg list limits. 357 */ 358 to_process = nbytes - processed; 359 360 if ((to_process + processed) < nbytes) 361 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; 362 else 363 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 364 365 NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; 366 367 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src, 368 &to_process, processed + req->assoclen, 369 csbcpb->cpb.aes_ccm.iv_or_ctr); 370 if (rc) 371 goto out; 372 373 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 374 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 375 if (rc) 376 goto out; 377 378 /* for partial completion, copy following for next 379 * entry into loop... 380 */ 381 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE); 382 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0, 383 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE); 384 memcpy(csbcpb->cpb.aes_ccm.in_s0, 385 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE); 386 387 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 388 389 /* update stats */ 390 atomic_inc(&(nx_ctx->stats->aes_ops)); 391 atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), 392 &(nx_ctx->stats->aes_bytes)); 393 394 processed += to_process; 395 } while (processed < nbytes); 396 397 rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag, 398 authsize) ? -EBADMSG : 0; 399 out: 400 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); 401 return rc; 402 } 403 404 static int ccm_nx_encrypt(struct aead_request *req, 405 u8 *iv, 406 unsigned int assoclen) 407 { 408 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 409 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 410 unsigned int nbytes = req->cryptlen; 411 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 412 unsigned long irq_flags; 413 unsigned int processed = 0, to_process; 414 int rc = -1; 415 416 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 417 418 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen, 419 csbcpb->cpb.aes_ccm.in_pat_or_b0); 420 if (rc) 421 goto out; 422 423 do { 424 /* to process: the AES_BLOCK_SIZE data chunk to process in this 425 * update. This value is bound by sg list limits. 426 */ 427 to_process = nbytes - processed; 428 429 if ((to_process + processed) < nbytes) 430 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; 431 else 432 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 433 434 NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; 435 436 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src, 437 &to_process, processed + req->assoclen, 438 csbcpb->cpb.aes_ccm.iv_or_ctr); 439 if (rc) 440 goto out; 441 442 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 443 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 444 if (rc) 445 goto out; 446 447 /* for partial completion, copy following for next 448 * entry into loop... 449 */ 450 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE); 451 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0, 452 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE); 453 memcpy(csbcpb->cpb.aes_ccm.in_s0, 454 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE); 455 456 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 457 458 /* update stats */ 459 atomic_inc(&(nx_ctx->stats->aes_ops)); 460 atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), 461 &(nx_ctx->stats->aes_bytes)); 462 463 processed += to_process; 464 465 } while (processed < nbytes); 466 467 /* copy out the auth tag */ 468 scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac, 469 req->dst, nbytes + req->assoclen, authsize, 470 SCATTERWALK_TO_SG); 471 472 out: 473 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); 474 return rc; 475 } 476 477 static int ccm4309_aes_nx_encrypt(struct aead_request *req) 478 { 479 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 480 struct nx_gcm_rctx *rctx = aead_request_ctx(req); 481 u8 *iv = rctx->iv; 482 483 iv[0] = 3; 484 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3); 485 memcpy(iv + 4, req->iv, 8); 486 487 return ccm_nx_encrypt(req, iv, req->assoclen - 8); 488 } 489 490 static int ccm_aes_nx_encrypt(struct aead_request *req) 491 { 492 int rc; 493 494 rc = crypto_ccm_check_iv(req->iv); 495 if (rc) 496 return rc; 497 498 return ccm_nx_encrypt(req, req->iv, req->assoclen); 499 } 500 501 static int ccm4309_aes_nx_decrypt(struct aead_request *req) 502 { 503 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 504 struct nx_gcm_rctx *rctx = aead_request_ctx(req); 505 u8 *iv = rctx->iv; 506 507 iv[0] = 3; 508 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3); 509 memcpy(iv + 4, req->iv, 8); 510 511 return ccm_nx_decrypt(req, iv, req->assoclen - 8); 512 } 513 514 static int ccm_aes_nx_decrypt(struct aead_request *req) 515 { 516 int rc; 517 518 rc = crypto_ccm_check_iv(req->iv); 519 if (rc) 520 return rc; 521 522 return ccm_nx_decrypt(req, req->iv, req->assoclen); 523 } 524 525 struct aead_alg nx_ccm_aes_alg = { 526 .base = { 527 .cra_name = "ccm(aes)", 528 .cra_driver_name = "ccm-aes-nx", 529 .cra_priority = 300, 530 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 531 .cra_blocksize = 1, 532 .cra_ctxsize = sizeof(struct nx_crypto_ctx), 533 .cra_module = THIS_MODULE, 534 }, 535 .init = nx_crypto_ctx_aes_ccm_init, 536 .exit = nx_crypto_ctx_aead_exit, 537 .ivsize = AES_BLOCK_SIZE, 538 .maxauthsize = AES_BLOCK_SIZE, 539 .setkey = ccm_aes_nx_set_key, 540 .setauthsize = ccm_aes_nx_setauthsize, 541 .encrypt = ccm_aes_nx_encrypt, 542 .decrypt = ccm_aes_nx_decrypt, 543 }; 544 545 struct aead_alg nx_ccm4309_aes_alg = { 546 .base = { 547 .cra_name = "rfc4309(ccm(aes))", 548 .cra_driver_name = "rfc4309-ccm-aes-nx", 549 .cra_priority = 300, 550 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 551 .cra_blocksize = 1, 552 .cra_ctxsize = sizeof(struct nx_crypto_ctx), 553 .cra_module = THIS_MODULE, 554 }, 555 .init = nx_crypto_ctx_aes_ccm_init, 556 .exit = nx_crypto_ctx_aead_exit, 557 .ivsize = 8, 558 .maxauthsize = AES_BLOCK_SIZE, 559 .setkey = ccm4309_aes_nx_set_key, 560 .setauthsize = ccm4309_aes_nx_setauthsize, 561 .encrypt = ccm4309_aes_nx_encrypt, 562 .decrypt = ccm4309_aes_nx_decrypt, 563 }; 564