1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include "dpseci-debugfs.h" 19 #include <linux/fsl/mc.h> 20 #include <soc/fsl/dpaa2-io.h> 21 #include <soc/fsl/dpaa2-fd.h> 22 #include <crypto/xts.h> 23 #include <asm/unaligned.h> 24 25 #define CAAM_CRA_PRIORITY 2000 26 27 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 28 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 29 SHA512_DIGEST_SIZE * 2) 30 31 /* 32 * This is a a cache of buffers, from which the users of CAAM QI driver 33 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 34 * NOTE: A more elegant solution would be to have some headroom in the frames 35 * being processed. This can be added by the dpaa2-eth driver. This would 36 * pose a problem for userspace application processing which cannot 37 * know of this limitation. So for now, this will work. 38 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 39 */ 40 static struct kmem_cache *qi_cache; 41 42 struct caam_alg_entry { 43 struct device *dev; 44 int class1_alg_type; 45 int class2_alg_type; 46 bool rfc3686; 47 bool geniv; 48 bool nodkp; 49 }; 50 51 struct caam_aead_alg { 52 struct aead_alg aead; 53 struct caam_alg_entry caam; 54 bool registered; 55 }; 56 57 struct caam_skcipher_alg { 58 struct skcipher_alg skcipher; 59 struct caam_alg_entry caam; 60 bool registered; 61 }; 62 63 /** 64 * struct caam_ctx - per-session context 65 * @flc: Flow Contexts array 66 * @key: [authentication key], encryption key 67 * @flc_dma: I/O virtual addresses of the Flow Contexts 68 * @key_dma: I/O virtual address of the key 69 * @dir: DMA direction for mapping key and Flow Contexts 70 * @dev: dpseci device 71 * @adata: authentication algorithm details 72 * @cdata: encryption algorithm details 73 * @authsize: authentication tag (a.k.a. ICV / MAC) size 74 * @xts_key_fallback: true if fallback tfm needs to be used due 75 * to unsupported xts key lengths 76 * @fallback: xts fallback tfm 77 */ 78 struct caam_ctx { 79 struct caam_flc flc[NUM_OP]; 80 u8 key[CAAM_MAX_KEY_SIZE]; 81 dma_addr_t flc_dma[NUM_OP]; 82 dma_addr_t key_dma; 83 enum dma_data_direction dir; 84 struct device *dev; 85 struct alginfo adata; 86 struct alginfo cdata; 87 unsigned int authsize; 88 bool xts_key_fallback; 89 struct crypto_skcipher *fallback; 90 }; 91 92 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 93 dma_addr_t iova_addr) 94 { 95 phys_addr_t phys_addr; 96 97 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 98 iova_addr; 99 100 return phys_to_virt(phys_addr); 101 } 102 103 /* 104 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 105 * 106 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 107 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 108 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 109 * hosting 16 SG entries. 110 * 111 * @flags - flags that would be used for the equivalent kmalloc(..) call 112 * 113 * Returns a pointer to a retrieved buffer on success or NULL on failure. 114 */ 115 static inline void *qi_cache_zalloc(gfp_t flags) 116 { 117 return kmem_cache_zalloc(qi_cache, flags); 118 } 119 120 /* 121 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 122 * 123 * @obj - buffer previously allocated by qi_cache_zalloc 124 * 125 * No checking is being done, the call is a passthrough call to 126 * kmem_cache_free(...) 127 */ 128 static inline void qi_cache_free(void *obj) 129 { 130 kmem_cache_free(qi_cache, obj); 131 } 132 133 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 134 { 135 switch (crypto_tfm_alg_type(areq->tfm)) { 136 case CRYPTO_ALG_TYPE_SKCIPHER: 137 return skcipher_request_ctx(skcipher_request_cast(areq)); 138 case CRYPTO_ALG_TYPE_AEAD: 139 return aead_request_ctx(container_of(areq, struct aead_request, 140 base)); 141 case CRYPTO_ALG_TYPE_AHASH: 142 return ahash_request_ctx(ahash_request_cast(areq)); 143 default: 144 return ERR_PTR(-EINVAL); 145 } 146 } 147 148 static void caam_unmap(struct device *dev, struct scatterlist *src, 149 struct scatterlist *dst, int src_nents, 150 int dst_nents, dma_addr_t iv_dma, int ivsize, 151 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 152 int qm_sg_bytes) 153 { 154 if (dst != src) { 155 if (src_nents) 156 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 157 if (dst_nents) 158 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 159 } else { 160 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 161 } 162 163 if (iv_dma) 164 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 165 166 if (qm_sg_bytes) 167 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 168 } 169 170 static int aead_set_sh_desc(struct crypto_aead *aead) 171 { 172 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 173 typeof(*alg), aead); 174 struct caam_ctx *ctx = crypto_aead_ctx(aead); 175 unsigned int ivsize = crypto_aead_ivsize(aead); 176 struct device *dev = ctx->dev; 177 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 178 struct caam_flc *flc; 179 u32 *desc; 180 u32 ctx1_iv_off = 0; 181 u32 *nonce = NULL; 182 unsigned int data_len[2]; 183 u32 inl_mask; 184 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 185 OP_ALG_AAI_CTR_MOD128); 186 const bool is_rfc3686 = alg->caam.rfc3686; 187 188 if (!ctx->cdata.keylen || !ctx->authsize) 189 return 0; 190 191 /* 192 * AES-CTR needs to load IV in CONTEXT1 reg 193 * at an offset of 128bits (16bytes) 194 * CONTEXT1[255:128] = IV 195 */ 196 if (ctr_mode) 197 ctx1_iv_off = 16; 198 199 /* 200 * RFC3686 specific: 201 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 202 */ 203 if (is_rfc3686) { 204 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 205 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 206 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 207 } 208 209 /* 210 * In case |user key| > |derived key|, using DKP<imm,imm> would result 211 * in invalid opcodes (last bytes of user key) in the resulting 212 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 213 * addresses are needed. 214 */ 215 ctx->adata.key_virt = ctx->key; 216 ctx->adata.key_dma = ctx->key_dma; 217 218 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 219 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 220 221 data_len[0] = ctx->adata.keylen_pad; 222 data_len[1] = ctx->cdata.keylen; 223 224 /* aead_encrypt shared descriptor */ 225 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 226 DESC_QI_AEAD_ENC_LEN) + 227 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 228 DESC_JOB_IO_LEN, data_len, &inl_mask, 229 ARRAY_SIZE(data_len)) < 0) 230 return -EINVAL; 231 232 ctx->adata.key_inline = !!(inl_mask & 1); 233 ctx->cdata.key_inline = !!(inl_mask & 2); 234 235 flc = &ctx->flc[ENCRYPT]; 236 desc = flc->sh_desc; 237 238 if (alg->caam.geniv) 239 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 240 ivsize, ctx->authsize, is_rfc3686, 241 nonce, ctx1_iv_off, true, 242 priv->sec_attr.era); 243 else 244 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 245 ivsize, ctx->authsize, is_rfc3686, nonce, 246 ctx1_iv_off, true, priv->sec_attr.era); 247 248 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 249 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 250 sizeof(flc->flc) + desc_bytes(desc), 251 ctx->dir); 252 253 /* aead_decrypt shared descriptor */ 254 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 255 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 256 DESC_JOB_IO_LEN, data_len, &inl_mask, 257 ARRAY_SIZE(data_len)) < 0) 258 return -EINVAL; 259 260 ctx->adata.key_inline = !!(inl_mask & 1); 261 ctx->cdata.key_inline = !!(inl_mask & 2); 262 263 flc = &ctx->flc[DECRYPT]; 264 desc = flc->sh_desc; 265 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 266 ivsize, ctx->authsize, alg->caam.geniv, 267 is_rfc3686, nonce, ctx1_iv_off, true, 268 priv->sec_attr.era); 269 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 270 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 271 sizeof(flc->flc) + desc_bytes(desc), 272 ctx->dir); 273 274 return 0; 275 } 276 277 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 278 { 279 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 280 281 ctx->authsize = authsize; 282 aead_set_sh_desc(authenc); 283 284 return 0; 285 } 286 287 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 288 unsigned int keylen) 289 { 290 struct caam_ctx *ctx = crypto_aead_ctx(aead); 291 struct device *dev = ctx->dev; 292 struct crypto_authenc_keys keys; 293 294 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 295 goto badkey; 296 297 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 298 keys.authkeylen + keys.enckeylen, keys.enckeylen, 299 keys.authkeylen); 300 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 301 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 302 303 ctx->adata.keylen = keys.authkeylen; 304 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 305 OP_ALG_ALGSEL_MASK); 306 307 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 308 goto badkey; 309 310 memcpy(ctx->key, keys.authkey, keys.authkeylen); 311 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 312 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 313 keys.enckeylen, ctx->dir); 314 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 315 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 316 ctx->adata.keylen_pad + keys.enckeylen, 1); 317 318 ctx->cdata.keylen = keys.enckeylen; 319 320 memzero_explicit(&keys, sizeof(keys)); 321 return aead_set_sh_desc(aead); 322 badkey: 323 memzero_explicit(&keys, sizeof(keys)); 324 return -EINVAL; 325 } 326 327 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 328 unsigned int keylen) 329 { 330 struct crypto_authenc_keys keys; 331 int err; 332 333 err = crypto_authenc_extractkeys(&keys, key, keylen); 334 if (unlikely(err)) 335 goto out; 336 337 err = -EINVAL; 338 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 339 goto out; 340 341 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 342 aead_setkey(aead, key, keylen); 343 344 out: 345 memzero_explicit(&keys, sizeof(keys)); 346 return err; 347 } 348 349 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 350 bool encrypt) 351 { 352 struct crypto_aead *aead = crypto_aead_reqtfm(req); 353 struct caam_request *req_ctx = aead_request_ctx(req); 354 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 355 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 356 struct caam_ctx *ctx = crypto_aead_ctx(aead); 357 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 358 typeof(*alg), aead); 359 struct device *dev = ctx->dev; 360 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 361 GFP_KERNEL : GFP_ATOMIC; 362 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 363 int src_len, dst_len = 0; 364 struct aead_edesc *edesc; 365 dma_addr_t qm_sg_dma, iv_dma = 0; 366 int ivsize = 0; 367 unsigned int authsize = ctx->authsize; 368 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 369 int in_len, out_len; 370 struct dpaa2_sg_entry *sg_table; 371 372 /* allocate space for base edesc, link tables and IV */ 373 edesc = qi_cache_zalloc(GFP_DMA | flags); 374 if (unlikely(!edesc)) { 375 dev_err(dev, "could not allocate extended descriptor\n"); 376 return ERR_PTR(-ENOMEM); 377 } 378 379 if (unlikely(req->dst != req->src)) { 380 src_len = req->assoclen + req->cryptlen; 381 dst_len = src_len + (encrypt ? authsize : (-authsize)); 382 383 src_nents = sg_nents_for_len(req->src, src_len); 384 if (unlikely(src_nents < 0)) { 385 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 386 src_len); 387 qi_cache_free(edesc); 388 return ERR_PTR(src_nents); 389 } 390 391 dst_nents = sg_nents_for_len(req->dst, dst_len); 392 if (unlikely(dst_nents < 0)) { 393 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 394 dst_len); 395 qi_cache_free(edesc); 396 return ERR_PTR(dst_nents); 397 } 398 399 if (src_nents) { 400 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 401 DMA_TO_DEVICE); 402 if (unlikely(!mapped_src_nents)) { 403 dev_err(dev, "unable to map source\n"); 404 qi_cache_free(edesc); 405 return ERR_PTR(-ENOMEM); 406 } 407 } else { 408 mapped_src_nents = 0; 409 } 410 411 if (dst_nents) { 412 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 413 DMA_FROM_DEVICE); 414 if (unlikely(!mapped_dst_nents)) { 415 dev_err(dev, "unable to map destination\n"); 416 dma_unmap_sg(dev, req->src, src_nents, 417 DMA_TO_DEVICE); 418 qi_cache_free(edesc); 419 return ERR_PTR(-ENOMEM); 420 } 421 } else { 422 mapped_dst_nents = 0; 423 } 424 } else { 425 src_len = req->assoclen + req->cryptlen + 426 (encrypt ? authsize : 0); 427 428 src_nents = sg_nents_for_len(req->src, src_len); 429 if (unlikely(src_nents < 0)) { 430 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 431 src_len); 432 qi_cache_free(edesc); 433 return ERR_PTR(src_nents); 434 } 435 436 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 437 DMA_BIDIRECTIONAL); 438 if (unlikely(!mapped_src_nents)) { 439 dev_err(dev, "unable to map source\n"); 440 qi_cache_free(edesc); 441 return ERR_PTR(-ENOMEM); 442 } 443 } 444 445 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 446 ivsize = crypto_aead_ivsize(aead); 447 448 /* 449 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 450 * Input is not contiguous. 451 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 452 * the end of the table by allocating more S/G entries. Logic: 453 * if (src != dst && output S/G) 454 * pad output S/G, if needed 455 * else if (src == dst && S/G) 456 * overlapping S/Gs; pad one of them 457 * else if (input S/G) ... 458 * pad input S/G, if needed 459 */ 460 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 461 if (mapped_dst_nents > 1) 462 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 463 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 464 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 465 1 + !!ivsize + 466 pad_sg_nents(mapped_src_nents)); 467 else 468 qm_sg_nents = pad_sg_nents(qm_sg_nents); 469 470 sg_table = &edesc->sgt[0]; 471 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 472 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 473 CAAM_QI_MEMCACHE_SIZE)) { 474 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 475 qm_sg_nents, ivsize); 476 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 477 0, DMA_NONE, 0, 0); 478 qi_cache_free(edesc); 479 return ERR_PTR(-ENOMEM); 480 } 481 482 if (ivsize) { 483 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 484 485 /* Make sure IV is located in a DMAable area */ 486 memcpy(iv, req->iv, ivsize); 487 488 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 489 if (dma_mapping_error(dev, iv_dma)) { 490 dev_err(dev, "unable to map IV\n"); 491 caam_unmap(dev, req->src, req->dst, src_nents, 492 dst_nents, 0, 0, DMA_NONE, 0, 0); 493 qi_cache_free(edesc); 494 return ERR_PTR(-ENOMEM); 495 } 496 } 497 498 edesc->src_nents = src_nents; 499 edesc->dst_nents = dst_nents; 500 edesc->iv_dma = iv_dma; 501 502 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 503 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 504 /* 505 * The associated data comes already with the IV but we need 506 * to skip it when we authenticate or encrypt... 507 */ 508 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 509 else 510 edesc->assoclen = cpu_to_caam32(req->assoclen); 511 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 512 DMA_TO_DEVICE); 513 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 514 dev_err(dev, "unable to map assoclen\n"); 515 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 516 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 517 qi_cache_free(edesc); 518 return ERR_PTR(-ENOMEM); 519 } 520 521 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 522 qm_sg_index++; 523 if (ivsize) { 524 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 525 qm_sg_index++; 526 } 527 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 528 qm_sg_index += mapped_src_nents; 529 530 if (mapped_dst_nents > 1) 531 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 532 533 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 534 if (dma_mapping_error(dev, qm_sg_dma)) { 535 dev_err(dev, "unable to map S/G table\n"); 536 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 537 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 538 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 539 qi_cache_free(edesc); 540 return ERR_PTR(-ENOMEM); 541 } 542 543 edesc->qm_sg_dma = qm_sg_dma; 544 edesc->qm_sg_bytes = qm_sg_bytes; 545 546 out_len = req->assoclen + req->cryptlen + 547 (encrypt ? ctx->authsize : (-ctx->authsize)); 548 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 549 550 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 551 dpaa2_fl_set_final(in_fle, true); 552 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 553 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 554 dpaa2_fl_set_len(in_fle, in_len); 555 556 if (req->dst == req->src) { 557 if (mapped_src_nents == 1) { 558 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 559 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 560 } else { 561 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 562 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 563 (1 + !!ivsize) * sizeof(*sg_table)); 564 } 565 } else if (!mapped_dst_nents) { 566 /* 567 * crypto engine requires the output entry to be present when 568 * "frame list" FD is used. 569 * Since engine does not support FMT=2'b11 (unused entry type), 570 * leaving out_fle zeroized is the best option. 571 */ 572 goto skip_out_fle; 573 } else if (mapped_dst_nents == 1) { 574 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 575 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 576 } else { 577 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 578 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 579 sizeof(*sg_table)); 580 } 581 582 dpaa2_fl_set_len(out_fle, out_len); 583 584 skip_out_fle: 585 return edesc; 586 } 587 588 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 589 { 590 struct caam_ctx *ctx = crypto_aead_ctx(aead); 591 unsigned int ivsize = crypto_aead_ivsize(aead); 592 struct device *dev = ctx->dev; 593 struct caam_flc *flc; 594 u32 *desc; 595 596 if (!ctx->cdata.keylen || !ctx->authsize) 597 return 0; 598 599 flc = &ctx->flc[ENCRYPT]; 600 desc = flc->sh_desc; 601 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 602 ctx->authsize, true, true); 603 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 604 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 605 sizeof(flc->flc) + desc_bytes(desc), 606 ctx->dir); 607 608 flc = &ctx->flc[DECRYPT]; 609 desc = flc->sh_desc; 610 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 611 ctx->authsize, false, true); 612 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 613 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 614 sizeof(flc->flc) + desc_bytes(desc), 615 ctx->dir); 616 617 return 0; 618 } 619 620 static int chachapoly_setauthsize(struct crypto_aead *aead, 621 unsigned int authsize) 622 { 623 struct caam_ctx *ctx = crypto_aead_ctx(aead); 624 625 if (authsize != POLY1305_DIGEST_SIZE) 626 return -EINVAL; 627 628 ctx->authsize = authsize; 629 return chachapoly_set_sh_desc(aead); 630 } 631 632 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 633 unsigned int keylen) 634 { 635 struct caam_ctx *ctx = crypto_aead_ctx(aead); 636 unsigned int ivsize = crypto_aead_ivsize(aead); 637 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 638 639 if (keylen != CHACHA_KEY_SIZE + saltlen) 640 return -EINVAL; 641 642 ctx->cdata.key_virt = key; 643 ctx->cdata.keylen = keylen - saltlen; 644 645 return chachapoly_set_sh_desc(aead); 646 } 647 648 static int gcm_set_sh_desc(struct crypto_aead *aead) 649 { 650 struct caam_ctx *ctx = crypto_aead_ctx(aead); 651 struct device *dev = ctx->dev; 652 unsigned int ivsize = crypto_aead_ivsize(aead); 653 struct caam_flc *flc; 654 u32 *desc; 655 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 656 ctx->cdata.keylen; 657 658 if (!ctx->cdata.keylen || !ctx->authsize) 659 return 0; 660 661 /* 662 * AES GCM encrypt shared descriptor 663 * Job Descriptor and Shared Descriptor 664 * must fit into the 64-word Descriptor h/w Buffer 665 */ 666 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 667 ctx->cdata.key_inline = true; 668 ctx->cdata.key_virt = ctx->key; 669 } else { 670 ctx->cdata.key_inline = false; 671 ctx->cdata.key_dma = ctx->key_dma; 672 } 673 674 flc = &ctx->flc[ENCRYPT]; 675 desc = flc->sh_desc; 676 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 677 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 678 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 679 sizeof(flc->flc) + desc_bytes(desc), 680 ctx->dir); 681 682 /* 683 * Job Descriptor and Shared Descriptors 684 * must all fit into the 64-word Descriptor h/w Buffer 685 */ 686 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 687 ctx->cdata.key_inline = true; 688 ctx->cdata.key_virt = ctx->key; 689 } else { 690 ctx->cdata.key_inline = false; 691 ctx->cdata.key_dma = ctx->key_dma; 692 } 693 694 flc = &ctx->flc[DECRYPT]; 695 desc = flc->sh_desc; 696 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 697 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 698 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 699 sizeof(flc->flc) + desc_bytes(desc), 700 ctx->dir); 701 702 return 0; 703 } 704 705 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 706 { 707 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 708 int err; 709 710 err = crypto_gcm_check_authsize(authsize); 711 if (err) 712 return err; 713 714 ctx->authsize = authsize; 715 gcm_set_sh_desc(authenc); 716 717 return 0; 718 } 719 720 static int gcm_setkey(struct crypto_aead *aead, 721 const u8 *key, unsigned int keylen) 722 { 723 struct caam_ctx *ctx = crypto_aead_ctx(aead); 724 struct device *dev = ctx->dev; 725 int ret; 726 727 ret = aes_check_keylen(keylen); 728 if (ret) 729 return ret; 730 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 731 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 732 733 memcpy(ctx->key, key, keylen); 734 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 735 ctx->cdata.keylen = keylen; 736 737 return gcm_set_sh_desc(aead); 738 } 739 740 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 741 { 742 struct caam_ctx *ctx = crypto_aead_ctx(aead); 743 struct device *dev = ctx->dev; 744 unsigned int ivsize = crypto_aead_ivsize(aead); 745 struct caam_flc *flc; 746 u32 *desc; 747 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 748 ctx->cdata.keylen; 749 750 if (!ctx->cdata.keylen || !ctx->authsize) 751 return 0; 752 753 ctx->cdata.key_virt = ctx->key; 754 755 /* 756 * RFC4106 encrypt shared descriptor 757 * Job Descriptor and Shared Descriptor 758 * must fit into the 64-word Descriptor h/w Buffer 759 */ 760 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 761 ctx->cdata.key_inline = true; 762 } else { 763 ctx->cdata.key_inline = false; 764 ctx->cdata.key_dma = ctx->key_dma; 765 } 766 767 flc = &ctx->flc[ENCRYPT]; 768 desc = flc->sh_desc; 769 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 770 true); 771 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 772 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 773 sizeof(flc->flc) + desc_bytes(desc), 774 ctx->dir); 775 776 /* 777 * Job Descriptor and Shared Descriptors 778 * must all fit into the 64-word Descriptor h/w Buffer 779 */ 780 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 781 ctx->cdata.key_inline = true; 782 } else { 783 ctx->cdata.key_inline = false; 784 ctx->cdata.key_dma = ctx->key_dma; 785 } 786 787 flc = &ctx->flc[DECRYPT]; 788 desc = flc->sh_desc; 789 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 790 true); 791 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 792 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 793 sizeof(flc->flc) + desc_bytes(desc), 794 ctx->dir); 795 796 return 0; 797 } 798 799 static int rfc4106_setauthsize(struct crypto_aead *authenc, 800 unsigned int authsize) 801 { 802 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 803 int err; 804 805 err = crypto_rfc4106_check_authsize(authsize); 806 if (err) 807 return err; 808 809 ctx->authsize = authsize; 810 rfc4106_set_sh_desc(authenc); 811 812 return 0; 813 } 814 815 static int rfc4106_setkey(struct crypto_aead *aead, 816 const u8 *key, unsigned int keylen) 817 { 818 struct caam_ctx *ctx = crypto_aead_ctx(aead); 819 struct device *dev = ctx->dev; 820 int ret; 821 822 ret = aes_check_keylen(keylen - 4); 823 if (ret) 824 return ret; 825 826 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 827 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 828 829 memcpy(ctx->key, key, keylen); 830 /* 831 * The last four bytes of the key material are used as the salt value 832 * in the nonce. Update the AES key length. 833 */ 834 ctx->cdata.keylen = keylen - 4; 835 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 836 ctx->dir); 837 838 return rfc4106_set_sh_desc(aead); 839 } 840 841 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 842 { 843 struct caam_ctx *ctx = crypto_aead_ctx(aead); 844 struct device *dev = ctx->dev; 845 unsigned int ivsize = crypto_aead_ivsize(aead); 846 struct caam_flc *flc; 847 u32 *desc; 848 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 849 ctx->cdata.keylen; 850 851 if (!ctx->cdata.keylen || !ctx->authsize) 852 return 0; 853 854 ctx->cdata.key_virt = ctx->key; 855 856 /* 857 * RFC4543 encrypt shared descriptor 858 * Job Descriptor and Shared Descriptor 859 * must fit into the 64-word Descriptor h/w Buffer 860 */ 861 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 862 ctx->cdata.key_inline = true; 863 } else { 864 ctx->cdata.key_inline = false; 865 ctx->cdata.key_dma = ctx->key_dma; 866 } 867 868 flc = &ctx->flc[ENCRYPT]; 869 desc = flc->sh_desc; 870 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 871 true); 872 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 873 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 874 sizeof(flc->flc) + desc_bytes(desc), 875 ctx->dir); 876 877 /* 878 * Job Descriptor and Shared Descriptors 879 * must all fit into the 64-word Descriptor h/w Buffer 880 */ 881 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 882 ctx->cdata.key_inline = true; 883 } else { 884 ctx->cdata.key_inline = false; 885 ctx->cdata.key_dma = ctx->key_dma; 886 } 887 888 flc = &ctx->flc[DECRYPT]; 889 desc = flc->sh_desc; 890 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 891 true); 892 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 893 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 894 sizeof(flc->flc) + desc_bytes(desc), 895 ctx->dir); 896 897 return 0; 898 } 899 900 static int rfc4543_setauthsize(struct crypto_aead *authenc, 901 unsigned int authsize) 902 { 903 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 904 905 if (authsize != 16) 906 return -EINVAL; 907 908 ctx->authsize = authsize; 909 rfc4543_set_sh_desc(authenc); 910 911 return 0; 912 } 913 914 static int rfc4543_setkey(struct crypto_aead *aead, 915 const u8 *key, unsigned int keylen) 916 { 917 struct caam_ctx *ctx = crypto_aead_ctx(aead); 918 struct device *dev = ctx->dev; 919 int ret; 920 921 ret = aes_check_keylen(keylen - 4); 922 if (ret) 923 return ret; 924 925 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 926 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 927 928 memcpy(ctx->key, key, keylen); 929 /* 930 * The last four bytes of the key material are used as the salt value 931 * in the nonce. Update the AES key length. 932 */ 933 ctx->cdata.keylen = keylen - 4; 934 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 935 ctx->dir); 936 937 return rfc4543_set_sh_desc(aead); 938 } 939 940 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 941 unsigned int keylen, const u32 ctx1_iv_off) 942 { 943 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 944 struct caam_skcipher_alg *alg = 945 container_of(crypto_skcipher_alg(skcipher), 946 struct caam_skcipher_alg, skcipher); 947 struct device *dev = ctx->dev; 948 struct caam_flc *flc; 949 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 950 u32 *desc; 951 const bool is_rfc3686 = alg->caam.rfc3686; 952 953 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 954 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 955 956 ctx->cdata.keylen = keylen; 957 ctx->cdata.key_virt = key; 958 ctx->cdata.key_inline = true; 959 960 /* skcipher_encrypt shared descriptor */ 961 flc = &ctx->flc[ENCRYPT]; 962 desc = flc->sh_desc; 963 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 964 ctx1_iv_off); 965 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 966 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 967 sizeof(flc->flc) + desc_bytes(desc), 968 ctx->dir); 969 970 /* skcipher_decrypt shared descriptor */ 971 flc = &ctx->flc[DECRYPT]; 972 desc = flc->sh_desc; 973 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 974 ctx1_iv_off); 975 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 976 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 977 sizeof(flc->flc) + desc_bytes(desc), 978 ctx->dir); 979 980 return 0; 981 } 982 983 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 984 const u8 *key, unsigned int keylen) 985 { 986 int err; 987 988 err = aes_check_keylen(keylen); 989 if (err) 990 return err; 991 992 return skcipher_setkey(skcipher, key, keylen, 0); 993 } 994 995 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 996 const u8 *key, unsigned int keylen) 997 { 998 u32 ctx1_iv_off; 999 int err; 1000 1001 /* 1002 * RFC3686 specific: 1003 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1004 * | *key = {KEY, NONCE} 1005 */ 1006 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1007 keylen -= CTR_RFC3686_NONCE_SIZE; 1008 1009 err = aes_check_keylen(keylen); 1010 if (err) 1011 return err; 1012 1013 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1014 } 1015 1016 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1017 const u8 *key, unsigned int keylen) 1018 { 1019 u32 ctx1_iv_off; 1020 int err; 1021 1022 /* 1023 * AES-CTR needs to load IV in CONTEXT1 reg 1024 * at an offset of 128bits (16bytes) 1025 * CONTEXT1[255:128] = IV 1026 */ 1027 ctx1_iv_off = 16; 1028 1029 err = aes_check_keylen(keylen); 1030 if (err) 1031 return err; 1032 1033 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1034 } 1035 1036 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1037 const u8 *key, unsigned int keylen) 1038 { 1039 if (keylen != CHACHA_KEY_SIZE) 1040 return -EINVAL; 1041 1042 return skcipher_setkey(skcipher, key, keylen, 0); 1043 } 1044 1045 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1046 const u8 *key, unsigned int keylen) 1047 { 1048 return verify_skcipher_des_key(skcipher, key) ?: 1049 skcipher_setkey(skcipher, key, keylen, 0); 1050 } 1051 1052 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1053 const u8 *key, unsigned int keylen) 1054 { 1055 return verify_skcipher_des3_key(skcipher, key) ?: 1056 skcipher_setkey(skcipher, key, keylen, 0); 1057 } 1058 1059 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1060 unsigned int keylen) 1061 { 1062 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1063 struct device *dev = ctx->dev; 1064 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 1065 struct caam_flc *flc; 1066 u32 *desc; 1067 int err; 1068 1069 err = xts_verify_key(skcipher, key, keylen); 1070 if (err) { 1071 dev_dbg(dev, "key size mismatch\n"); 1072 return err; 1073 } 1074 1075 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 1076 ctx->xts_key_fallback = true; 1077 1078 if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { 1079 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 1080 if (err) 1081 return err; 1082 } 1083 1084 ctx->cdata.keylen = keylen; 1085 ctx->cdata.key_virt = key; 1086 ctx->cdata.key_inline = true; 1087 1088 /* xts_skcipher_encrypt shared descriptor */ 1089 flc = &ctx->flc[ENCRYPT]; 1090 desc = flc->sh_desc; 1091 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1092 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1093 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1094 sizeof(flc->flc) + desc_bytes(desc), 1095 ctx->dir); 1096 1097 /* xts_skcipher_decrypt shared descriptor */ 1098 flc = &ctx->flc[DECRYPT]; 1099 desc = flc->sh_desc; 1100 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1101 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1102 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1103 sizeof(flc->flc) + desc_bytes(desc), 1104 ctx->dir); 1105 1106 return 0; 1107 } 1108 1109 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1110 { 1111 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1112 struct caam_request *req_ctx = skcipher_request_ctx(req); 1113 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1114 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1115 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1116 struct device *dev = ctx->dev; 1117 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1118 GFP_KERNEL : GFP_ATOMIC; 1119 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1120 struct skcipher_edesc *edesc; 1121 dma_addr_t iv_dma; 1122 u8 *iv; 1123 int ivsize = crypto_skcipher_ivsize(skcipher); 1124 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1125 struct dpaa2_sg_entry *sg_table; 1126 1127 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1128 if (unlikely(src_nents < 0)) { 1129 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1130 req->cryptlen); 1131 return ERR_PTR(src_nents); 1132 } 1133 1134 if (unlikely(req->dst != req->src)) { 1135 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1136 if (unlikely(dst_nents < 0)) { 1137 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1138 req->cryptlen); 1139 return ERR_PTR(dst_nents); 1140 } 1141 1142 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1143 DMA_TO_DEVICE); 1144 if (unlikely(!mapped_src_nents)) { 1145 dev_err(dev, "unable to map source\n"); 1146 return ERR_PTR(-ENOMEM); 1147 } 1148 1149 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1150 DMA_FROM_DEVICE); 1151 if (unlikely(!mapped_dst_nents)) { 1152 dev_err(dev, "unable to map destination\n"); 1153 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1154 return ERR_PTR(-ENOMEM); 1155 } 1156 } else { 1157 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1158 DMA_BIDIRECTIONAL); 1159 if (unlikely(!mapped_src_nents)) { 1160 dev_err(dev, "unable to map source\n"); 1161 return ERR_PTR(-ENOMEM); 1162 } 1163 } 1164 1165 qm_sg_ents = 1 + mapped_src_nents; 1166 dst_sg_idx = qm_sg_ents; 1167 1168 /* 1169 * Input, output HW S/G tables: [IV, src][dst, IV] 1170 * IV entries point to the same buffer 1171 * If src == dst, S/G entries are reused (S/G tables overlap) 1172 * 1173 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1174 * the end of the table by allocating more S/G entries. 1175 */ 1176 if (req->src != req->dst) 1177 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1178 else 1179 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1180 1181 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1182 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1183 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1184 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1185 qm_sg_ents, ivsize); 1186 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1187 0, DMA_NONE, 0, 0); 1188 return ERR_PTR(-ENOMEM); 1189 } 1190 1191 /* allocate space for base edesc, link tables and IV */ 1192 edesc = qi_cache_zalloc(GFP_DMA | flags); 1193 if (unlikely(!edesc)) { 1194 dev_err(dev, "could not allocate extended descriptor\n"); 1195 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1196 0, DMA_NONE, 0, 0); 1197 return ERR_PTR(-ENOMEM); 1198 } 1199 1200 /* Make sure IV is located in a DMAable area */ 1201 sg_table = &edesc->sgt[0]; 1202 iv = (u8 *)(sg_table + qm_sg_ents); 1203 memcpy(iv, req->iv, ivsize); 1204 1205 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1206 if (dma_mapping_error(dev, iv_dma)) { 1207 dev_err(dev, "unable to map IV\n"); 1208 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1209 0, DMA_NONE, 0, 0); 1210 qi_cache_free(edesc); 1211 return ERR_PTR(-ENOMEM); 1212 } 1213 1214 edesc->src_nents = src_nents; 1215 edesc->dst_nents = dst_nents; 1216 edesc->iv_dma = iv_dma; 1217 edesc->qm_sg_bytes = qm_sg_bytes; 1218 1219 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1220 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1221 1222 if (req->src != req->dst) 1223 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1224 1225 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1226 ivsize, 0); 1227 1228 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1229 DMA_TO_DEVICE); 1230 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1231 dev_err(dev, "unable to map S/G table\n"); 1232 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1233 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1234 qi_cache_free(edesc); 1235 return ERR_PTR(-ENOMEM); 1236 } 1237 1238 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1239 dpaa2_fl_set_final(in_fle, true); 1240 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1241 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1242 1243 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1244 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1245 1246 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1247 1248 if (req->src == req->dst) 1249 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1250 sizeof(*sg_table)); 1251 else 1252 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1253 sizeof(*sg_table)); 1254 1255 return edesc; 1256 } 1257 1258 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1259 struct aead_request *req) 1260 { 1261 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1262 int ivsize = crypto_aead_ivsize(aead); 1263 1264 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1265 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1266 edesc->qm_sg_bytes); 1267 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1268 } 1269 1270 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1271 struct skcipher_request *req) 1272 { 1273 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1274 int ivsize = crypto_skcipher_ivsize(skcipher); 1275 1276 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1277 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1278 edesc->qm_sg_bytes); 1279 } 1280 1281 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1282 { 1283 struct crypto_async_request *areq = cbk_ctx; 1284 struct aead_request *req = container_of(areq, struct aead_request, 1285 base); 1286 struct caam_request *req_ctx = to_caam_req(areq); 1287 struct aead_edesc *edesc = req_ctx->edesc; 1288 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1289 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1290 int ecode = 0; 1291 1292 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1293 1294 if (unlikely(status)) 1295 ecode = caam_qi2_strstatus(ctx->dev, status); 1296 1297 aead_unmap(ctx->dev, edesc, req); 1298 qi_cache_free(edesc); 1299 aead_request_complete(req, ecode); 1300 } 1301 1302 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1303 { 1304 struct crypto_async_request *areq = cbk_ctx; 1305 struct aead_request *req = container_of(areq, struct aead_request, 1306 base); 1307 struct caam_request *req_ctx = to_caam_req(areq); 1308 struct aead_edesc *edesc = req_ctx->edesc; 1309 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1310 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1311 int ecode = 0; 1312 1313 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1314 1315 if (unlikely(status)) 1316 ecode = caam_qi2_strstatus(ctx->dev, status); 1317 1318 aead_unmap(ctx->dev, edesc, req); 1319 qi_cache_free(edesc); 1320 aead_request_complete(req, ecode); 1321 } 1322 1323 static int aead_encrypt(struct aead_request *req) 1324 { 1325 struct aead_edesc *edesc; 1326 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1327 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1328 struct caam_request *caam_req = aead_request_ctx(req); 1329 int ret; 1330 1331 /* allocate extended descriptor */ 1332 edesc = aead_edesc_alloc(req, true); 1333 if (IS_ERR(edesc)) 1334 return PTR_ERR(edesc); 1335 1336 caam_req->flc = &ctx->flc[ENCRYPT]; 1337 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1338 caam_req->cbk = aead_encrypt_done; 1339 caam_req->ctx = &req->base; 1340 caam_req->edesc = edesc; 1341 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1342 if (ret != -EINPROGRESS && 1343 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1344 aead_unmap(ctx->dev, edesc, req); 1345 qi_cache_free(edesc); 1346 } 1347 1348 return ret; 1349 } 1350 1351 static int aead_decrypt(struct aead_request *req) 1352 { 1353 struct aead_edesc *edesc; 1354 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1355 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1356 struct caam_request *caam_req = aead_request_ctx(req); 1357 int ret; 1358 1359 /* allocate extended descriptor */ 1360 edesc = aead_edesc_alloc(req, false); 1361 if (IS_ERR(edesc)) 1362 return PTR_ERR(edesc); 1363 1364 caam_req->flc = &ctx->flc[DECRYPT]; 1365 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1366 caam_req->cbk = aead_decrypt_done; 1367 caam_req->ctx = &req->base; 1368 caam_req->edesc = edesc; 1369 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1370 if (ret != -EINPROGRESS && 1371 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1372 aead_unmap(ctx->dev, edesc, req); 1373 qi_cache_free(edesc); 1374 } 1375 1376 return ret; 1377 } 1378 1379 static int ipsec_gcm_encrypt(struct aead_request *req) 1380 { 1381 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1382 } 1383 1384 static int ipsec_gcm_decrypt(struct aead_request *req) 1385 { 1386 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1387 } 1388 1389 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1390 { 1391 struct crypto_async_request *areq = cbk_ctx; 1392 struct skcipher_request *req = skcipher_request_cast(areq); 1393 struct caam_request *req_ctx = to_caam_req(areq); 1394 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1395 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1396 struct skcipher_edesc *edesc = req_ctx->edesc; 1397 int ecode = 0; 1398 int ivsize = crypto_skcipher_ivsize(skcipher); 1399 1400 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1401 1402 if (unlikely(status)) 1403 ecode = caam_qi2_strstatus(ctx->dev, status); 1404 1405 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1406 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1407 edesc->src_nents > 1 ? 100 : ivsize, 1); 1408 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1409 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1410 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1411 1412 skcipher_unmap(ctx->dev, edesc, req); 1413 1414 /* 1415 * The crypto API expects us to set the IV (req->iv) to the last 1416 * ciphertext block (CBC mode) or last counter (CTR mode). 1417 * This is used e.g. by the CTS mode. 1418 */ 1419 if (!ecode) 1420 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1421 ivsize); 1422 1423 qi_cache_free(edesc); 1424 skcipher_request_complete(req, ecode); 1425 } 1426 1427 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1428 { 1429 struct crypto_async_request *areq = cbk_ctx; 1430 struct skcipher_request *req = skcipher_request_cast(areq); 1431 struct caam_request *req_ctx = to_caam_req(areq); 1432 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1433 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1434 struct skcipher_edesc *edesc = req_ctx->edesc; 1435 int ecode = 0; 1436 int ivsize = crypto_skcipher_ivsize(skcipher); 1437 1438 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1439 1440 if (unlikely(status)) 1441 ecode = caam_qi2_strstatus(ctx->dev, status); 1442 1443 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1444 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1445 edesc->src_nents > 1 ? 100 : ivsize, 1); 1446 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1447 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1448 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1449 1450 skcipher_unmap(ctx->dev, edesc, req); 1451 1452 /* 1453 * The crypto API expects us to set the IV (req->iv) to the last 1454 * ciphertext block (CBC mode) or last counter (CTR mode). 1455 * This is used e.g. by the CTS mode. 1456 */ 1457 if (!ecode) 1458 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1459 ivsize); 1460 1461 qi_cache_free(edesc); 1462 skcipher_request_complete(req, ecode); 1463 } 1464 1465 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1466 { 1467 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1468 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1469 1470 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1471 } 1472 1473 static int skcipher_encrypt(struct skcipher_request *req) 1474 { 1475 struct skcipher_edesc *edesc; 1476 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1477 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1478 struct caam_request *caam_req = skcipher_request_ctx(req); 1479 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1480 int ret; 1481 1482 /* 1483 * XTS is expected to return an error even for input length = 0 1484 * Note that the case input length < block size will be caught during 1485 * HW offloading and return an error. 1486 */ 1487 if (!req->cryptlen && !ctx->fallback) 1488 return 0; 1489 1490 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1491 ctx->xts_key_fallback)) { 1492 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1493 skcipher_request_set_callback(&caam_req->fallback_req, 1494 req->base.flags, 1495 req->base.complete, 1496 req->base.data); 1497 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1498 req->dst, req->cryptlen, req->iv); 1499 1500 return crypto_skcipher_encrypt(&caam_req->fallback_req); 1501 } 1502 1503 /* allocate extended descriptor */ 1504 edesc = skcipher_edesc_alloc(req); 1505 if (IS_ERR(edesc)) 1506 return PTR_ERR(edesc); 1507 1508 caam_req->flc = &ctx->flc[ENCRYPT]; 1509 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1510 caam_req->cbk = skcipher_encrypt_done; 1511 caam_req->ctx = &req->base; 1512 caam_req->edesc = edesc; 1513 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1514 if (ret != -EINPROGRESS && 1515 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1516 skcipher_unmap(ctx->dev, edesc, req); 1517 qi_cache_free(edesc); 1518 } 1519 1520 return ret; 1521 } 1522 1523 static int skcipher_decrypt(struct skcipher_request *req) 1524 { 1525 struct skcipher_edesc *edesc; 1526 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1527 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1528 struct caam_request *caam_req = skcipher_request_ctx(req); 1529 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1530 int ret; 1531 1532 /* 1533 * XTS is expected to return an error even for input length = 0 1534 * Note that the case input length < block size will be caught during 1535 * HW offloading and return an error. 1536 */ 1537 if (!req->cryptlen && !ctx->fallback) 1538 return 0; 1539 1540 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1541 ctx->xts_key_fallback)) { 1542 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1543 skcipher_request_set_callback(&caam_req->fallback_req, 1544 req->base.flags, 1545 req->base.complete, 1546 req->base.data); 1547 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1548 req->dst, req->cryptlen, req->iv); 1549 1550 return crypto_skcipher_decrypt(&caam_req->fallback_req); 1551 } 1552 1553 /* allocate extended descriptor */ 1554 edesc = skcipher_edesc_alloc(req); 1555 if (IS_ERR(edesc)) 1556 return PTR_ERR(edesc); 1557 1558 caam_req->flc = &ctx->flc[DECRYPT]; 1559 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1560 caam_req->cbk = skcipher_decrypt_done; 1561 caam_req->ctx = &req->base; 1562 caam_req->edesc = edesc; 1563 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1564 if (ret != -EINPROGRESS && 1565 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1566 skcipher_unmap(ctx->dev, edesc, req); 1567 qi_cache_free(edesc); 1568 } 1569 1570 return ret; 1571 } 1572 1573 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1574 bool uses_dkp) 1575 { 1576 dma_addr_t dma_addr; 1577 int i; 1578 1579 /* copy descriptor header template value */ 1580 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1581 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1582 1583 ctx->dev = caam->dev; 1584 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1585 1586 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1587 offsetof(struct caam_ctx, flc_dma), 1588 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1589 if (dma_mapping_error(ctx->dev, dma_addr)) { 1590 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1591 return -ENOMEM; 1592 } 1593 1594 for (i = 0; i < NUM_OP; i++) 1595 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1596 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1597 1598 return 0; 1599 } 1600 1601 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1602 { 1603 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1604 struct caam_skcipher_alg *caam_alg = 1605 container_of(alg, typeof(*caam_alg), skcipher); 1606 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 1607 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 1608 int ret = 0; 1609 1610 if (alg_aai == OP_ALG_AAI_XTS) { 1611 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1612 struct crypto_skcipher *fallback; 1613 1614 fallback = crypto_alloc_skcipher(tfm_name, 0, 1615 CRYPTO_ALG_NEED_FALLBACK); 1616 if (IS_ERR(fallback)) { 1617 dev_err(caam_alg->caam.dev, 1618 "Failed to allocate %s fallback: %ld\n", 1619 tfm_name, PTR_ERR(fallback)); 1620 return PTR_ERR(fallback); 1621 } 1622 1623 ctx->fallback = fallback; 1624 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request) + 1625 crypto_skcipher_reqsize(fallback)); 1626 } else { 1627 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); 1628 } 1629 1630 ret = caam_cra_init(ctx, &caam_alg->caam, false); 1631 if (ret && ctx->fallback) 1632 crypto_free_skcipher(ctx->fallback); 1633 1634 return ret; 1635 } 1636 1637 static int caam_cra_init_aead(struct crypto_aead *tfm) 1638 { 1639 struct aead_alg *alg = crypto_aead_alg(tfm); 1640 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1641 aead); 1642 1643 crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); 1644 return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam, 1645 !caam_alg->caam.nodkp); 1646 } 1647 1648 static void caam_exit_common(struct caam_ctx *ctx) 1649 { 1650 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1651 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1652 DMA_ATTR_SKIP_CPU_SYNC); 1653 } 1654 1655 static void caam_cra_exit(struct crypto_skcipher *tfm) 1656 { 1657 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 1658 1659 if (ctx->fallback) 1660 crypto_free_skcipher(ctx->fallback); 1661 caam_exit_common(ctx); 1662 } 1663 1664 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1665 { 1666 caam_exit_common(crypto_aead_ctx(tfm)); 1667 } 1668 1669 static struct caam_skcipher_alg driver_algs[] = { 1670 { 1671 .skcipher = { 1672 .base = { 1673 .cra_name = "cbc(aes)", 1674 .cra_driver_name = "cbc-aes-caam-qi2", 1675 .cra_blocksize = AES_BLOCK_SIZE, 1676 }, 1677 .setkey = aes_skcipher_setkey, 1678 .encrypt = skcipher_encrypt, 1679 .decrypt = skcipher_decrypt, 1680 .min_keysize = AES_MIN_KEY_SIZE, 1681 .max_keysize = AES_MAX_KEY_SIZE, 1682 .ivsize = AES_BLOCK_SIZE, 1683 }, 1684 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1685 }, 1686 { 1687 .skcipher = { 1688 .base = { 1689 .cra_name = "cbc(des3_ede)", 1690 .cra_driver_name = "cbc-3des-caam-qi2", 1691 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1692 }, 1693 .setkey = des3_skcipher_setkey, 1694 .encrypt = skcipher_encrypt, 1695 .decrypt = skcipher_decrypt, 1696 .min_keysize = DES3_EDE_KEY_SIZE, 1697 .max_keysize = DES3_EDE_KEY_SIZE, 1698 .ivsize = DES3_EDE_BLOCK_SIZE, 1699 }, 1700 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1701 }, 1702 { 1703 .skcipher = { 1704 .base = { 1705 .cra_name = "cbc(des)", 1706 .cra_driver_name = "cbc-des-caam-qi2", 1707 .cra_blocksize = DES_BLOCK_SIZE, 1708 }, 1709 .setkey = des_skcipher_setkey, 1710 .encrypt = skcipher_encrypt, 1711 .decrypt = skcipher_decrypt, 1712 .min_keysize = DES_KEY_SIZE, 1713 .max_keysize = DES_KEY_SIZE, 1714 .ivsize = DES_BLOCK_SIZE, 1715 }, 1716 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1717 }, 1718 { 1719 .skcipher = { 1720 .base = { 1721 .cra_name = "ctr(aes)", 1722 .cra_driver_name = "ctr-aes-caam-qi2", 1723 .cra_blocksize = 1, 1724 }, 1725 .setkey = ctr_skcipher_setkey, 1726 .encrypt = skcipher_encrypt, 1727 .decrypt = skcipher_decrypt, 1728 .min_keysize = AES_MIN_KEY_SIZE, 1729 .max_keysize = AES_MAX_KEY_SIZE, 1730 .ivsize = AES_BLOCK_SIZE, 1731 .chunksize = AES_BLOCK_SIZE, 1732 }, 1733 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1734 OP_ALG_AAI_CTR_MOD128, 1735 }, 1736 { 1737 .skcipher = { 1738 .base = { 1739 .cra_name = "rfc3686(ctr(aes))", 1740 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1741 .cra_blocksize = 1, 1742 }, 1743 .setkey = rfc3686_skcipher_setkey, 1744 .encrypt = skcipher_encrypt, 1745 .decrypt = skcipher_decrypt, 1746 .min_keysize = AES_MIN_KEY_SIZE + 1747 CTR_RFC3686_NONCE_SIZE, 1748 .max_keysize = AES_MAX_KEY_SIZE + 1749 CTR_RFC3686_NONCE_SIZE, 1750 .ivsize = CTR_RFC3686_IV_SIZE, 1751 .chunksize = AES_BLOCK_SIZE, 1752 }, 1753 .caam = { 1754 .class1_alg_type = OP_ALG_ALGSEL_AES | 1755 OP_ALG_AAI_CTR_MOD128, 1756 .rfc3686 = true, 1757 }, 1758 }, 1759 { 1760 .skcipher = { 1761 .base = { 1762 .cra_name = "xts(aes)", 1763 .cra_driver_name = "xts-aes-caam-qi2", 1764 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1765 .cra_blocksize = AES_BLOCK_SIZE, 1766 }, 1767 .setkey = xts_skcipher_setkey, 1768 .encrypt = skcipher_encrypt, 1769 .decrypt = skcipher_decrypt, 1770 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1771 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1772 .ivsize = AES_BLOCK_SIZE, 1773 }, 1774 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1775 }, 1776 { 1777 .skcipher = { 1778 .base = { 1779 .cra_name = "chacha20", 1780 .cra_driver_name = "chacha20-caam-qi2", 1781 .cra_blocksize = 1, 1782 }, 1783 .setkey = chacha20_skcipher_setkey, 1784 .encrypt = skcipher_encrypt, 1785 .decrypt = skcipher_decrypt, 1786 .min_keysize = CHACHA_KEY_SIZE, 1787 .max_keysize = CHACHA_KEY_SIZE, 1788 .ivsize = CHACHA_IV_SIZE, 1789 }, 1790 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1791 }, 1792 }; 1793 1794 static struct caam_aead_alg driver_aeads[] = { 1795 { 1796 .aead = { 1797 .base = { 1798 .cra_name = "rfc4106(gcm(aes))", 1799 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1800 .cra_blocksize = 1, 1801 }, 1802 .setkey = rfc4106_setkey, 1803 .setauthsize = rfc4106_setauthsize, 1804 .encrypt = ipsec_gcm_encrypt, 1805 .decrypt = ipsec_gcm_decrypt, 1806 .ivsize = 8, 1807 .maxauthsize = AES_BLOCK_SIZE, 1808 }, 1809 .caam = { 1810 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1811 .nodkp = true, 1812 }, 1813 }, 1814 { 1815 .aead = { 1816 .base = { 1817 .cra_name = "rfc4543(gcm(aes))", 1818 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1819 .cra_blocksize = 1, 1820 }, 1821 .setkey = rfc4543_setkey, 1822 .setauthsize = rfc4543_setauthsize, 1823 .encrypt = ipsec_gcm_encrypt, 1824 .decrypt = ipsec_gcm_decrypt, 1825 .ivsize = 8, 1826 .maxauthsize = AES_BLOCK_SIZE, 1827 }, 1828 .caam = { 1829 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1830 .nodkp = true, 1831 }, 1832 }, 1833 /* Galois Counter Mode */ 1834 { 1835 .aead = { 1836 .base = { 1837 .cra_name = "gcm(aes)", 1838 .cra_driver_name = "gcm-aes-caam-qi2", 1839 .cra_blocksize = 1, 1840 }, 1841 .setkey = gcm_setkey, 1842 .setauthsize = gcm_setauthsize, 1843 .encrypt = aead_encrypt, 1844 .decrypt = aead_decrypt, 1845 .ivsize = 12, 1846 .maxauthsize = AES_BLOCK_SIZE, 1847 }, 1848 .caam = { 1849 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1850 .nodkp = true, 1851 } 1852 }, 1853 /* single-pass ipsec_esp descriptor */ 1854 { 1855 .aead = { 1856 .base = { 1857 .cra_name = "authenc(hmac(md5),cbc(aes))", 1858 .cra_driver_name = "authenc-hmac-md5-" 1859 "cbc-aes-caam-qi2", 1860 .cra_blocksize = AES_BLOCK_SIZE, 1861 }, 1862 .setkey = aead_setkey, 1863 .setauthsize = aead_setauthsize, 1864 .encrypt = aead_encrypt, 1865 .decrypt = aead_decrypt, 1866 .ivsize = AES_BLOCK_SIZE, 1867 .maxauthsize = MD5_DIGEST_SIZE, 1868 }, 1869 .caam = { 1870 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1871 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1872 OP_ALG_AAI_HMAC_PRECOMP, 1873 } 1874 }, 1875 { 1876 .aead = { 1877 .base = { 1878 .cra_name = "echainiv(authenc(hmac(md5)," 1879 "cbc(aes)))", 1880 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1881 "cbc-aes-caam-qi2", 1882 .cra_blocksize = AES_BLOCK_SIZE, 1883 }, 1884 .setkey = aead_setkey, 1885 .setauthsize = aead_setauthsize, 1886 .encrypt = aead_encrypt, 1887 .decrypt = aead_decrypt, 1888 .ivsize = AES_BLOCK_SIZE, 1889 .maxauthsize = MD5_DIGEST_SIZE, 1890 }, 1891 .caam = { 1892 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1893 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1894 OP_ALG_AAI_HMAC_PRECOMP, 1895 .geniv = true, 1896 } 1897 }, 1898 { 1899 .aead = { 1900 .base = { 1901 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1902 .cra_driver_name = "authenc-hmac-sha1-" 1903 "cbc-aes-caam-qi2", 1904 .cra_blocksize = AES_BLOCK_SIZE, 1905 }, 1906 .setkey = aead_setkey, 1907 .setauthsize = aead_setauthsize, 1908 .encrypt = aead_encrypt, 1909 .decrypt = aead_decrypt, 1910 .ivsize = AES_BLOCK_SIZE, 1911 .maxauthsize = SHA1_DIGEST_SIZE, 1912 }, 1913 .caam = { 1914 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1915 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1916 OP_ALG_AAI_HMAC_PRECOMP, 1917 } 1918 }, 1919 { 1920 .aead = { 1921 .base = { 1922 .cra_name = "echainiv(authenc(hmac(sha1)," 1923 "cbc(aes)))", 1924 .cra_driver_name = "echainiv-authenc-" 1925 "hmac-sha1-cbc-aes-caam-qi2", 1926 .cra_blocksize = AES_BLOCK_SIZE, 1927 }, 1928 .setkey = aead_setkey, 1929 .setauthsize = aead_setauthsize, 1930 .encrypt = aead_encrypt, 1931 .decrypt = aead_decrypt, 1932 .ivsize = AES_BLOCK_SIZE, 1933 .maxauthsize = SHA1_DIGEST_SIZE, 1934 }, 1935 .caam = { 1936 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1937 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1938 OP_ALG_AAI_HMAC_PRECOMP, 1939 .geniv = true, 1940 }, 1941 }, 1942 { 1943 .aead = { 1944 .base = { 1945 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1946 .cra_driver_name = "authenc-hmac-sha224-" 1947 "cbc-aes-caam-qi2", 1948 .cra_blocksize = AES_BLOCK_SIZE, 1949 }, 1950 .setkey = aead_setkey, 1951 .setauthsize = aead_setauthsize, 1952 .encrypt = aead_encrypt, 1953 .decrypt = aead_decrypt, 1954 .ivsize = AES_BLOCK_SIZE, 1955 .maxauthsize = SHA224_DIGEST_SIZE, 1956 }, 1957 .caam = { 1958 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1959 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1960 OP_ALG_AAI_HMAC_PRECOMP, 1961 } 1962 }, 1963 { 1964 .aead = { 1965 .base = { 1966 .cra_name = "echainiv(authenc(hmac(sha224)," 1967 "cbc(aes)))", 1968 .cra_driver_name = "echainiv-authenc-" 1969 "hmac-sha224-cbc-aes-caam-qi2", 1970 .cra_blocksize = AES_BLOCK_SIZE, 1971 }, 1972 .setkey = aead_setkey, 1973 .setauthsize = aead_setauthsize, 1974 .encrypt = aead_encrypt, 1975 .decrypt = aead_decrypt, 1976 .ivsize = AES_BLOCK_SIZE, 1977 .maxauthsize = SHA224_DIGEST_SIZE, 1978 }, 1979 .caam = { 1980 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1981 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1982 OP_ALG_AAI_HMAC_PRECOMP, 1983 .geniv = true, 1984 } 1985 }, 1986 { 1987 .aead = { 1988 .base = { 1989 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1990 .cra_driver_name = "authenc-hmac-sha256-" 1991 "cbc-aes-caam-qi2", 1992 .cra_blocksize = AES_BLOCK_SIZE, 1993 }, 1994 .setkey = aead_setkey, 1995 .setauthsize = aead_setauthsize, 1996 .encrypt = aead_encrypt, 1997 .decrypt = aead_decrypt, 1998 .ivsize = AES_BLOCK_SIZE, 1999 .maxauthsize = SHA256_DIGEST_SIZE, 2000 }, 2001 .caam = { 2002 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2003 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2004 OP_ALG_AAI_HMAC_PRECOMP, 2005 } 2006 }, 2007 { 2008 .aead = { 2009 .base = { 2010 .cra_name = "echainiv(authenc(hmac(sha256)," 2011 "cbc(aes)))", 2012 .cra_driver_name = "echainiv-authenc-" 2013 "hmac-sha256-cbc-aes-" 2014 "caam-qi2", 2015 .cra_blocksize = AES_BLOCK_SIZE, 2016 }, 2017 .setkey = aead_setkey, 2018 .setauthsize = aead_setauthsize, 2019 .encrypt = aead_encrypt, 2020 .decrypt = aead_decrypt, 2021 .ivsize = AES_BLOCK_SIZE, 2022 .maxauthsize = SHA256_DIGEST_SIZE, 2023 }, 2024 .caam = { 2025 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2026 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2027 OP_ALG_AAI_HMAC_PRECOMP, 2028 .geniv = true, 2029 } 2030 }, 2031 { 2032 .aead = { 2033 .base = { 2034 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2035 .cra_driver_name = "authenc-hmac-sha384-" 2036 "cbc-aes-caam-qi2", 2037 .cra_blocksize = AES_BLOCK_SIZE, 2038 }, 2039 .setkey = aead_setkey, 2040 .setauthsize = aead_setauthsize, 2041 .encrypt = aead_encrypt, 2042 .decrypt = aead_decrypt, 2043 .ivsize = AES_BLOCK_SIZE, 2044 .maxauthsize = SHA384_DIGEST_SIZE, 2045 }, 2046 .caam = { 2047 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2048 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2049 OP_ALG_AAI_HMAC_PRECOMP, 2050 } 2051 }, 2052 { 2053 .aead = { 2054 .base = { 2055 .cra_name = "echainiv(authenc(hmac(sha384)," 2056 "cbc(aes)))", 2057 .cra_driver_name = "echainiv-authenc-" 2058 "hmac-sha384-cbc-aes-" 2059 "caam-qi2", 2060 .cra_blocksize = AES_BLOCK_SIZE, 2061 }, 2062 .setkey = aead_setkey, 2063 .setauthsize = aead_setauthsize, 2064 .encrypt = aead_encrypt, 2065 .decrypt = aead_decrypt, 2066 .ivsize = AES_BLOCK_SIZE, 2067 .maxauthsize = SHA384_DIGEST_SIZE, 2068 }, 2069 .caam = { 2070 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2071 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2072 OP_ALG_AAI_HMAC_PRECOMP, 2073 .geniv = true, 2074 } 2075 }, 2076 { 2077 .aead = { 2078 .base = { 2079 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2080 .cra_driver_name = "authenc-hmac-sha512-" 2081 "cbc-aes-caam-qi2", 2082 .cra_blocksize = AES_BLOCK_SIZE, 2083 }, 2084 .setkey = aead_setkey, 2085 .setauthsize = aead_setauthsize, 2086 .encrypt = aead_encrypt, 2087 .decrypt = aead_decrypt, 2088 .ivsize = AES_BLOCK_SIZE, 2089 .maxauthsize = SHA512_DIGEST_SIZE, 2090 }, 2091 .caam = { 2092 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2093 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2094 OP_ALG_AAI_HMAC_PRECOMP, 2095 } 2096 }, 2097 { 2098 .aead = { 2099 .base = { 2100 .cra_name = "echainiv(authenc(hmac(sha512)," 2101 "cbc(aes)))", 2102 .cra_driver_name = "echainiv-authenc-" 2103 "hmac-sha512-cbc-aes-" 2104 "caam-qi2", 2105 .cra_blocksize = AES_BLOCK_SIZE, 2106 }, 2107 .setkey = aead_setkey, 2108 .setauthsize = aead_setauthsize, 2109 .encrypt = aead_encrypt, 2110 .decrypt = aead_decrypt, 2111 .ivsize = AES_BLOCK_SIZE, 2112 .maxauthsize = SHA512_DIGEST_SIZE, 2113 }, 2114 .caam = { 2115 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2116 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2117 OP_ALG_AAI_HMAC_PRECOMP, 2118 .geniv = true, 2119 } 2120 }, 2121 { 2122 .aead = { 2123 .base = { 2124 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2125 .cra_driver_name = "authenc-hmac-md5-" 2126 "cbc-des3_ede-caam-qi2", 2127 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2128 }, 2129 .setkey = des3_aead_setkey, 2130 .setauthsize = aead_setauthsize, 2131 .encrypt = aead_encrypt, 2132 .decrypt = aead_decrypt, 2133 .ivsize = DES3_EDE_BLOCK_SIZE, 2134 .maxauthsize = MD5_DIGEST_SIZE, 2135 }, 2136 .caam = { 2137 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2138 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2139 OP_ALG_AAI_HMAC_PRECOMP, 2140 } 2141 }, 2142 { 2143 .aead = { 2144 .base = { 2145 .cra_name = "echainiv(authenc(hmac(md5)," 2146 "cbc(des3_ede)))", 2147 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2148 "cbc-des3_ede-caam-qi2", 2149 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2150 }, 2151 .setkey = des3_aead_setkey, 2152 .setauthsize = aead_setauthsize, 2153 .encrypt = aead_encrypt, 2154 .decrypt = aead_decrypt, 2155 .ivsize = DES3_EDE_BLOCK_SIZE, 2156 .maxauthsize = MD5_DIGEST_SIZE, 2157 }, 2158 .caam = { 2159 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2160 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2161 OP_ALG_AAI_HMAC_PRECOMP, 2162 .geniv = true, 2163 } 2164 }, 2165 { 2166 .aead = { 2167 .base = { 2168 .cra_name = "authenc(hmac(sha1)," 2169 "cbc(des3_ede))", 2170 .cra_driver_name = "authenc-hmac-sha1-" 2171 "cbc-des3_ede-caam-qi2", 2172 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2173 }, 2174 .setkey = des3_aead_setkey, 2175 .setauthsize = aead_setauthsize, 2176 .encrypt = aead_encrypt, 2177 .decrypt = aead_decrypt, 2178 .ivsize = DES3_EDE_BLOCK_SIZE, 2179 .maxauthsize = SHA1_DIGEST_SIZE, 2180 }, 2181 .caam = { 2182 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2183 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2184 OP_ALG_AAI_HMAC_PRECOMP, 2185 }, 2186 }, 2187 { 2188 .aead = { 2189 .base = { 2190 .cra_name = "echainiv(authenc(hmac(sha1)," 2191 "cbc(des3_ede)))", 2192 .cra_driver_name = "echainiv-authenc-" 2193 "hmac-sha1-" 2194 "cbc-des3_ede-caam-qi2", 2195 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2196 }, 2197 .setkey = des3_aead_setkey, 2198 .setauthsize = aead_setauthsize, 2199 .encrypt = aead_encrypt, 2200 .decrypt = aead_decrypt, 2201 .ivsize = DES3_EDE_BLOCK_SIZE, 2202 .maxauthsize = SHA1_DIGEST_SIZE, 2203 }, 2204 .caam = { 2205 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2206 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2207 OP_ALG_AAI_HMAC_PRECOMP, 2208 .geniv = true, 2209 } 2210 }, 2211 { 2212 .aead = { 2213 .base = { 2214 .cra_name = "authenc(hmac(sha224)," 2215 "cbc(des3_ede))", 2216 .cra_driver_name = "authenc-hmac-sha224-" 2217 "cbc-des3_ede-caam-qi2", 2218 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2219 }, 2220 .setkey = des3_aead_setkey, 2221 .setauthsize = aead_setauthsize, 2222 .encrypt = aead_encrypt, 2223 .decrypt = aead_decrypt, 2224 .ivsize = DES3_EDE_BLOCK_SIZE, 2225 .maxauthsize = SHA224_DIGEST_SIZE, 2226 }, 2227 .caam = { 2228 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2229 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2230 OP_ALG_AAI_HMAC_PRECOMP, 2231 }, 2232 }, 2233 { 2234 .aead = { 2235 .base = { 2236 .cra_name = "echainiv(authenc(hmac(sha224)," 2237 "cbc(des3_ede)))", 2238 .cra_driver_name = "echainiv-authenc-" 2239 "hmac-sha224-" 2240 "cbc-des3_ede-caam-qi2", 2241 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2242 }, 2243 .setkey = des3_aead_setkey, 2244 .setauthsize = aead_setauthsize, 2245 .encrypt = aead_encrypt, 2246 .decrypt = aead_decrypt, 2247 .ivsize = DES3_EDE_BLOCK_SIZE, 2248 .maxauthsize = SHA224_DIGEST_SIZE, 2249 }, 2250 .caam = { 2251 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2252 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2253 OP_ALG_AAI_HMAC_PRECOMP, 2254 .geniv = true, 2255 } 2256 }, 2257 { 2258 .aead = { 2259 .base = { 2260 .cra_name = "authenc(hmac(sha256)," 2261 "cbc(des3_ede))", 2262 .cra_driver_name = "authenc-hmac-sha256-" 2263 "cbc-des3_ede-caam-qi2", 2264 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2265 }, 2266 .setkey = des3_aead_setkey, 2267 .setauthsize = aead_setauthsize, 2268 .encrypt = aead_encrypt, 2269 .decrypt = aead_decrypt, 2270 .ivsize = DES3_EDE_BLOCK_SIZE, 2271 .maxauthsize = SHA256_DIGEST_SIZE, 2272 }, 2273 .caam = { 2274 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2275 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2276 OP_ALG_AAI_HMAC_PRECOMP, 2277 }, 2278 }, 2279 { 2280 .aead = { 2281 .base = { 2282 .cra_name = "echainiv(authenc(hmac(sha256)," 2283 "cbc(des3_ede)))", 2284 .cra_driver_name = "echainiv-authenc-" 2285 "hmac-sha256-" 2286 "cbc-des3_ede-caam-qi2", 2287 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2288 }, 2289 .setkey = des3_aead_setkey, 2290 .setauthsize = aead_setauthsize, 2291 .encrypt = aead_encrypt, 2292 .decrypt = aead_decrypt, 2293 .ivsize = DES3_EDE_BLOCK_SIZE, 2294 .maxauthsize = SHA256_DIGEST_SIZE, 2295 }, 2296 .caam = { 2297 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2298 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2299 OP_ALG_AAI_HMAC_PRECOMP, 2300 .geniv = true, 2301 } 2302 }, 2303 { 2304 .aead = { 2305 .base = { 2306 .cra_name = "authenc(hmac(sha384)," 2307 "cbc(des3_ede))", 2308 .cra_driver_name = "authenc-hmac-sha384-" 2309 "cbc-des3_ede-caam-qi2", 2310 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2311 }, 2312 .setkey = des3_aead_setkey, 2313 .setauthsize = aead_setauthsize, 2314 .encrypt = aead_encrypt, 2315 .decrypt = aead_decrypt, 2316 .ivsize = DES3_EDE_BLOCK_SIZE, 2317 .maxauthsize = SHA384_DIGEST_SIZE, 2318 }, 2319 .caam = { 2320 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2321 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2322 OP_ALG_AAI_HMAC_PRECOMP, 2323 }, 2324 }, 2325 { 2326 .aead = { 2327 .base = { 2328 .cra_name = "echainiv(authenc(hmac(sha384)," 2329 "cbc(des3_ede)))", 2330 .cra_driver_name = "echainiv-authenc-" 2331 "hmac-sha384-" 2332 "cbc-des3_ede-caam-qi2", 2333 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2334 }, 2335 .setkey = des3_aead_setkey, 2336 .setauthsize = aead_setauthsize, 2337 .encrypt = aead_encrypt, 2338 .decrypt = aead_decrypt, 2339 .ivsize = DES3_EDE_BLOCK_SIZE, 2340 .maxauthsize = SHA384_DIGEST_SIZE, 2341 }, 2342 .caam = { 2343 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2344 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2345 OP_ALG_AAI_HMAC_PRECOMP, 2346 .geniv = true, 2347 } 2348 }, 2349 { 2350 .aead = { 2351 .base = { 2352 .cra_name = "authenc(hmac(sha512)," 2353 "cbc(des3_ede))", 2354 .cra_driver_name = "authenc-hmac-sha512-" 2355 "cbc-des3_ede-caam-qi2", 2356 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2357 }, 2358 .setkey = des3_aead_setkey, 2359 .setauthsize = aead_setauthsize, 2360 .encrypt = aead_encrypt, 2361 .decrypt = aead_decrypt, 2362 .ivsize = DES3_EDE_BLOCK_SIZE, 2363 .maxauthsize = SHA512_DIGEST_SIZE, 2364 }, 2365 .caam = { 2366 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2367 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2368 OP_ALG_AAI_HMAC_PRECOMP, 2369 }, 2370 }, 2371 { 2372 .aead = { 2373 .base = { 2374 .cra_name = "echainiv(authenc(hmac(sha512)," 2375 "cbc(des3_ede)))", 2376 .cra_driver_name = "echainiv-authenc-" 2377 "hmac-sha512-" 2378 "cbc-des3_ede-caam-qi2", 2379 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2380 }, 2381 .setkey = des3_aead_setkey, 2382 .setauthsize = aead_setauthsize, 2383 .encrypt = aead_encrypt, 2384 .decrypt = aead_decrypt, 2385 .ivsize = DES3_EDE_BLOCK_SIZE, 2386 .maxauthsize = SHA512_DIGEST_SIZE, 2387 }, 2388 .caam = { 2389 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2390 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2391 OP_ALG_AAI_HMAC_PRECOMP, 2392 .geniv = true, 2393 } 2394 }, 2395 { 2396 .aead = { 2397 .base = { 2398 .cra_name = "authenc(hmac(md5),cbc(des))", 2399 .cra_driver_name = "authenc-hmac-md5-" 2400 "cbc-des-caam-qi2", 2401 .cra_blocksize = DES_BLOCK_SIZE, 2402 }, 2403 .setkey = aead_setkey, 2404 .setauthsize = aead_setauthsize, 2405 .encrypt = aead_encrypt, 2406 .decrypt = aead_decrypt, 2407 .ivsize = DES_BLOCK_SIZE, 2408 .maxauthsize = MD5_DIGEST_SIZE, 2409 }, 2410 .caam = { 2411 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2412 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2413 OP_ALG_AAI_HMAC_PRECOMP, 2414 }, 2415 }, 2416 { 2417 .aead = { 2418 .base = { 2419 .cra_name = "echainiv(authenc(hmac(md5)," 2420 "cbc(des)))", 2421 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2422 "cbc-des-caam-qi2", 2423 .cra_blocksize = DES_BLOCK_SIZE, 2424 }, 2425 .setkey = aead_setkey, 2426 .setauthsize = aead_setauthsize, 2427 .encrypt = aead_encrypt, 2428 .decrypt = aead_decrypt, 2429 .ivsize = DES_BLOCK_SIZE, 2430 .maxauthsize = MD5_DIGEST_SIZE, 2431 }, 2432 .caam = { 2433 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2434 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2435 OP_ALG_AAI_HMAC_PRECOMP, 2436 .geniv = true, 2437 } 2438 }, 2439 { 2440 .aead = { 2441 .base = { 2442 .cra_name = "authenc(hmac(sha1),cbc(des))", 2443 .cra_driver_name = "authenc-hmac-sha1-" 2444 "cbc-des-caam-qi2", 2445 .cra_blocksize = DES_BLOCK_SIZE, 2446 }, 2447 .setkey = aead_setkey, 2448 .setauthsize = aead_setauthsize, 2449 .encrypt = aead_encrypt, 2450 .decrypt = aead_decrypt, 2451 .ivsize = DES_BLOCK_SIZE, 2452 .maxauthsize = SHA1_DIGEST_SIZE, 2453 }, 2454 .caam = { 2455 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2456 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2457 OP_ALG_AAI_HMAC_PRECOMP, 2458 }, 2459 }, 2460 { 2461 .aead = { 2462 .base = { 2463 .cra_name = "echainiv(authenc(hmac(sha1)," 2464 "cbc(des)))", 2465 .cra_driver_name = "echainiv-authenc-" 2466 "hmac-sha1-cbc-des-caam-qi2", 2467 .cra_blocksize = DES_BLOCK_SIZE, 2468 }, 2469 .setkey = aead_setkey, 2470 .setauthsize = aead_setauthsize, 2471 .encrypt = aead_encrypt, 2472 .decrypt = aead_decrypt, 2473 .ivsize = DES_BLOCK_SIZE, 2474 .maxauthsize = SHA1_DIGEST_SIZE, 2475 }, 2476 .caam = { 2477 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2478 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2479 OP_ALG_AAI_HMAC_PRECOMP, 2480 .geniv = true, 2481 } 2482 }, 2483 { 2484 .aead = { 2485 .base = { 2486 .cra_name = "authenc(hmac(sha224),cbc(des))", 2487 .cra_driver_name = "authenc-hmac-sha224-" 2488 "cbc-des-caam-qi2", 2489 .cra_blocksize = DES_BLOCK_SIZE, 2490 }, 2491 .setkey = aead_setkey, 2492 .setauthsize = aead_setauthsize, 2493 .encrypt = aead_encrypt, 2494 .decrypt = aead_decrypt, 2495 .ivsize = DES_BLOCK_SIZE, 2496 .maxauthsize = SHA224_DIGEST_SIZE, 2497 }, 2498 .caam = { 2499 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2500 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2501 OP_ALG_AAI_HMAC_PRECOMP, 2502 }, 2503 }, 2504 { 2505 .aead = { 2506 .base = { 2507 .cra_name = "echainiv(authenc(hmac(sha224)," 2508 "cbc(des)))", 2509 .cra_driver_name = "echainiv-authenc-" 2510 "hmac-sha224-cbc-des-" 2511 "caam-qi2", 2512 .cra_blocksize = DES_BLOCK_SIZE, 2513 }, 2514 .setkey = aead_setkey, 2515 .setauthsize = aead_setauthsize, 2516 .encrypt = aead_encrypt, 2517 .decrypt = aead_decrypt, 2518 .ivsize = DES_BLOCK_SIZE, 2519 .maxauthsize = SHA224_DIGEST_SIZE, 2520 }, 2521 .caam = { 2522 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2523 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2524 OP_ALG_AAI_HMAC_PRECOMP, 2525 .geniv = true, 2526 } 2527 }, 2528 { 2529 .aead = { 2530 .base = { 2531 .cra_name = "authenc(hmac(sha256),cbc(des))", 2532 .cra_driver_name = "authenc-hmac-sha256-" 2533 "cbc-des-caam-qi2", 2534 .cra_blocksize = DES_BLOCK_SIZE, 2535 }, 2536 .setkey = aead_setkey, 2537 .setauthsize = aead_setauthsize, 2538 .encrypt = aead_encrypt, 2539 .decrypt = aead_decrypt, 2540 .ivsize = DES_BLOCK_SIZE, 2541 .maxauthsize = SHA256_DIGEST_SIZE, 2542 }, 2543 .caam = { 2544 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2545 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2546 OP_ALG_AAI_HMAC_PRECOMP, 2547 }, 2548 }, 2549 { 2550 .aead = { 2551 .base = { 2552 .cra_name = "echainiv(authenc(hmac(sha256)," 2553 "cbc(des)))", 2554 .cra_driver_name = "echainiv-authenc-" 2555 "hmac-sha256-cbc-des-" 2556 "caam-qi2", 2557 .cra_blocksize = DES_BLOCK_SIZE, 2558 }, 2559 .setkey = aead_setkey, 2560 .setauthsize = aead_setauthsize, 2561 .encrypt = aead_encrypt, 2562 .decrypt = aead_decrypt, 2563 .ivsize = DES_BLOCK_SIZE, 2564 .maxauthsize = SHA256_DIGEST_SIZE, 2565 }, 2566 .caam = { 2567 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2568 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2569 OP_ALG_AAI_HMAC_PRECOMP, 2570 .geniv = true, 2571 }, 2572 }, 2573 { 2574 .aead = { 2575 .base = { 2576 .cra_name = "authenc(hmac(sha384),cbc(des))", 2577 .cra_driver_name = "authenc-hmac-sha384-" 2578 "cbc-des-caam-qi2", 2579 .cra_blocksize = DES_BLOCK_SIZE, 2580 }, 2581 .setkey = aead_setkey, 2582 .setauthsize = aead_setauthsize, 2583 .encrypt = aead_encrypt, 2584 .decrypt = aead_decrypt, 2585 .ivsize = DES_BLOCK_SIZE, 2586 .maxauthsize = SHA384_DIGEST_SIZE, 2587 }, 2588 .caam = { 2589 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2590 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2591 OP_ALG_AAI_HMAC_PRECOMP, 2592 }, 2593 }, 2594 { 2595 .aead = { 2596 .base = { 2597 .cra_name = "echainiv(authenc(hmac(sha384)," 2598 "cbc(des)))", 2599 .cra_driver_name = "echainiv-authenc-" 2600 "hmac-sha384-cbc-des-" 2601 "caam-qi2", 2602 .cra_blocksize = DES_BLOCK_SIZE, 2603 }, 2604 .setkey = aead_setkey, 2605 .setauthsize = aead_setauthsize, 2606 .encrypt = aead_encrypt, 2607 .decrypt = aead_decrypt, 2608 .ivsize = DES_BLOCK_SIZE, 2609 .maxauthsize = SHA384_DIGEST_SIZE, 2610 }, 2611 .caam = { 2612 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2613 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2614 OP_ALG_AAI_HMAC_PRECOMP, 2615 .geniv = true, 2616 } 2617 }, 2618 { 2619 .aead = { 2620 .base = { 2621 .cra_name = "authenc(hmac(sha512),cbc(des))", 2622 .cra_driver_name = "authenc-hmac-sha512-" 2623 "cbc-des-caam-qi2", 2624 .cra_blocksize = DES_BLOCK_SIZE, 2625 }, 2626 .setkey = aead_setkey, 2627 .setauthsize = aead_setauthsize, 2628 .encrypt = aead_encrypt, 2629 .decrypt = aead_decrypt, 2630 .ivsize = DES_BLOCK_SIZE, 2631 .maxauthsize = SHA512_DIGEST_SIZE, 2632 }, 2633 .caam = { 2634 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2635 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2636 OP_ALG_AAI_HMAC_PRECOMP, 2637 } 2638 }, 2639 { 2640 .aead = { 2641 .base = { 2642 .cra_name = "echainiv(authenc(hmac(sha512)," 2643 "cbc(des)))", 2644 .cra_driver_name = "echainiv-authenc-" 2645 "hmac-sha512-cbc-des-" 2646 "caam-qi2", 2647 .cra_blocksize = DES_BLOCK_SIZE, 2648 }, 2649 .setkey = aead_setkey, 2650 .setauthsize = aead_setauthsize, 2651 .encrypt = aead_encrypt, 2652 .decrypt = aead_decrypt, 2653 .ivsize = DES_BLOCK_SIZE, 2654 .maxauthsize = SHA512_DIGEST_SIZE, 2655 }, 2656 .caam = { 2657 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2658 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2659 OP_ALG_AAI_HMAC_PRECOMP, 2660 .geniv = true, 2661 } 2662 }, 2663 { 2664 .aead = { 2665 .base = { 2666 .cra_name = "authenc(hmac(md5)," 2667 "rfc3686(ctr(aes)))", 2668 .cra_driver_name = "authenc-hmac-md5-" 2669 "rfc3686-ctr-aes-caam-qi2", 2670 .cra_blocksize = 1, 2671 }, 2672 .setkey = aead_setkey, 2673 .setauthsize = aead_setauthsize, 2674 .encrypt = aead_encrypt, 2675 .decrypt = aead_decrypt, 2676 .ivsize = CTR_RFC3686_IV_SIZE, 2677 .maxauthsize = MD5_DIGEST_SIZE, 2678 }, 2679 .caam = { 2680 .class1_alg_type = OP_ALG_ALGSEL_AES | 2681 OP_ALG_AAI_CTR_MOD128, 2682 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2683 OP_ALG_AAI_HMAC_PRECOMP, 2684 .rfc3686 = true, 2685 }, 2686 }, 2687 { 2688 .aead = { 2689 .base = { 2690 .cra_name = "seqiv(authenc(" 2691 "hmac(md5),rfc3686(ctr(aes))))", 2692 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2693 "rfc3686-ctr-aes-caam-qi2", 2694 .cra_blocksize = 1, 2695 }, 2696 .setkey = aead_setkey, 2697 .setauthsize = aead_setauthsize, 2698 .encrypt = aead_encrypt, 2699 .decrypt = aead_decrypt, 2700 .ivsize = CTR_RFC3686_IV_SIZE, 2701 .maxauthsize = MD5_DIGEST_SIZE, 2702 }, 2703 .caam = { 2704 .class1_alg_type = OP_ALG_ALGSEL_AES | 2705 OP_ALG_AAI_CTR_MOD128, 2706 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2707 OP_ALG_AAI_HMAC_PRECOMP, 2708 .rfc3686 = true, 2709 .geniv = true, 2710 }, 2711 }, 2712 { 2713 .aead = { 2714 .base = { 2715 .cra_name = "authenc(hmac(sha1)," 2716 "rfc3686(ctr(aes)))", 2717 .cra_driver_name = "authenc-hmac-sha1-" 2718 "rfc3686-ctr-aes-caam-qi2", 2719 .cra_blocksize = 1, 2720 }, 2721 .setkey = aead_setkey, 2722 .setauthsize = aead_setauthsize, 2723 .encrypt = aead_encrypt, 2724 .decrypt = aead_decrypt, 2725 .ivsize = CTR_RFC3686_IV_SIZE, 2726 .maxauthsize = SHA1_DIGEST_SIZE, 2727 }, 2728 .caam = { 2729 .class1_alg_type = OP_ALG_ALGSEL_AES | 2730 OP_ALG_AAI_CTR_MOD128, 2731 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2732 OP_ALG_AAI_HMAC_PRECOMP, 2733 .rfc3686 = true, 2734 }, 2735 }, 2736 { 2737 .aead = { 2738 .base = { 2739 .cra_name = "seqiv(authenc(" 2740 "hmac(sha1),rfc3686(ctr(aes))))", 2741 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2742 "rfc3686-ctr-aes-caam-qi2", 2743 .cra_blocksize = 1, 2744 }, 2745 .setkey = aead_setkey, 2746 .setauthsize = aead_setauthsize, 2747 .encrypt = aead_encrypt, 2748 .decrypt = aead_decrypt, 2749 .ivsize = CTR_RFC3686_IV_SIZE, 2750 .maxauthsize = SHA1_DIGEST_SIZE, 2751 }, 2752 .caam = { 2753 .class1_alg_type = OP_ALG_ALGSEL_AES | 2754 OP_ALG_AAI_CTR_MOD128, 2755 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2756 OP_ALG_AAI_HMAC_PRECOMP, 2757 .rfc3686 = true, 2758 .geniv = true, 2759 }, 2760 }, 2761 { 2762 .aead = { 2763 .base = { 2764 .cra_name = "authenc(hmac(sha224)," 2765 "rfc3686(ctr(aes)))", 2766 .cra_driver_name = "authenc-hmac-sha224-" 2767 "rfc3686-ctr-aes-caam-qi2", 2768 .cra_blocksize = 1, 2769 }, 2770 .setkey = aead_setkey, 2771 .setauthsize = aead_setauthsize, 2772 .encrypt = aead_encrypt, 2773 .decrypt = aead_decrypt, 2774 .ivsize = CTR_RFC3686_IV_SIZE, 2775 .maxauthsize = SHA224_DIGEST_SIZE, 2776 }, 2777 .caam = { 2778 .class1_alg_type = OP_ALG_ALGSEL_AES | 2779 OP_ALG_AAI_CTR_MOD128, 2780 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2781 OP_ALG_AAI_HMAC_PRECOMP, 2782 .rfc3686 = true, 2783 }, 2784 }, 2785 { 2786 .aead = { 2787 .base = { 2788 .cra_name = "seqiv(authenc(" 2789 "hmac(sha224),rfc3686(ctr(aes))))", 2790 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2791 "rfc3686-ctr-aes-caam-qi2", 2792 .cra_blocksize = 1, 2793 }, 2794 .setkey = aead_setkey, 2795 .setauthsize = aead_setauthsize, 2796 .encrypt = aead_encrypt, 2797 .decrypt = aead_decrypt, 2798 .ivsize = CTR_RFC3686_IV_SIZE, 2799 .maxauthsize = SHA224_DIGEST_SIZE, 2800 }, 2801 .caam = { 2802 .class1_alg_type = OP_ALG_ALGSEL_AES | 2803 OP_ALG_AAI_CTR_MOD128, 2804 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2805 OP_ALG_AAI_HMAC_PRECOMP, 2806 .rfc3686 = true, 2807 .geniv = true, 2808 }, 2809 }, 2810 { 2811 .aead = { 2812 .base = { 2813 .cra_name = "authenc(hmac(sha256)," 2814 "rfc3686(ctr(aes)))", 2815 .cra_driver_name = "authenc-hmac-sha256-" 2816 "rfc3686-ctr-aes-caam-qi2", 2817 .cra_blocksize = 1, 2818 }, 2819 .setkey = aead_setkey, 2820 .setauthsize = aead_setauthsize, 2821 .encrypt = aead_encrypt, 2822 .decrypt = aead_decrypt, 2823 .ivsize = CTR_RFC3686_IV_SIZE, 2824 .maxauthsize = SHA256_DIGEST_SIZE, 2825 }, 2826 .caam = { 2827 .class1_alg_type = OP_ALG_ALGSEL_AES | 2828 OP_ALG_AAI_CTR_MOD128, 2829 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2830 OP_ALG_AAI_HMAC_PRECOMP, 2831 .rfc3686 = true, 2832 }, 2833 }, 2834 { 2835 .aead = { 2836 .base = { 2837 .cra_name = "seqiv(authenc(hmac(sha256)," 2838 "rfc3686(ctr(aes))))", 2839 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2840 "rfc3686-ctr-aes-caam-qi2", 2841 .cra_blocksize = 1, 2842 }, 2843 .setkey = aead_setkey, 2844 .setauthsize = aead_setauthsize, 2845 .encrypt = aead_encrypt, 2846 .decrypt = aead_decrypt, 2847 .ivsize = CTR_RFC3686_IV_SIZE, 2848 .maxauthsize = SHA256_DIGEST_SIZE, 2849 }, 2850 .caam = { 2851 .class1_alg_type = OP_ALG_ALGSEL_AES | 2852 OP_ALG_AAI_CTR_MOD128, 2853 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2854 OP_ALG_AAI_HMAC_PRECOMP, 2855 .rfc3686 = true, 2856 .geniv = true, 2857 }, 2858 }, 2859 { 2860 .aead = { 2861 .base = { 2862 .cra_name = "authenc(hmac(sha384)," 2863 "rfc3686(ctr(aes)))", 2864 .cra_driver_name = "authenc-hmac-sha384-" 2865 "rfc3686-ctr-aes-caam-qi2", 2866 .cra_blocksize = 1, 2867 }, 2868 .setkey = aead_setkey, 2869 .setauthsize = aead_setauthsize, 2870 .encrypt = aead_encrypt, 2871 .decrypt = aead_decrypt, 2872 .ivsize = CTR_RFC3686_IV_SIZE, 2873 .maxauthsize = SHA384_DIGEST_SIZE, 2874 }, 2875 .caam = { 2876 .class1_alg_type = OP_ALG_ALGSEL_AES | 2877 OP_ALG_AAI_CTR_MOD128, 2878 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2879 OP_ALG_AAI_HMAC_PRECOMP, 2880 .rfc3686 = true, 2881 }, 2882 }, 2883 { 2884 .aead = { 2885 .base = { 2886 .cra_name = "seqiv(authenc(hmac(sha384)," 2887 "rfc3686(ctr(aes))))", 2888 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2889 "rfc3686-ctr-aes-caam-qi2", 2890 .cra_blocksize = 1, 2891 }, 2892 .setkey = aead_setkey, 2893 .setauthsize = aead_setauthsize, 2894 .encrypt = aead_encrypt, 2895 .decrypt = aead_decrypt, 2896 .ivsize = CTR_RFC3686_IV_SIZE, 2897 .maxauthsize = SHA384_DIGEST_SIZE, 2898 }, 2899 .caam = { 2900 .class1_alg_type = OP_ALG_ALGSEL_AES | 2901 OP_ALG_AAI_CTR_MOD128, 2902 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2903 OP_ALG_AAI_HMAC_PRECOMP, 2904 .rfc3686 = true, 2905 .geniv = true, 2906 }, 2907 }, 2908 { 2909 .aead = { 2910 .base = { 2911 .cra_name = "rfc7539(chacha20,poly1305)", 2912 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2913 "caam-qi2", 2914 .cra_blocksize = 1, 2915 }, 2916 .setkey = chachapoly_setkey, 2917 .setauthsize = chachapoly_setauthsize, 2918 .encrypt = aead_encrypt, 2919 .decrypt = aead_decrypt, 2920 .ivsize = CHACHAPOLY_IV_SIZE, 2921 .maxauthsize = POLY1305_DIGEST_SIZE, 2922 }, 2923 .caam = { 2924 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2925 OP_ALG_AAI_AEAD, 2926 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2927 OP_ALG_AAI_AEAD, 2928 .nodkp = true, 2929 }, 2930 }, 2931 { 2932 .aead = { 2933 .base = { 2934 .cra_name = "rfc7539esp(chacha20,poly1305)", 2935 .cra_driver_name = "rfc7539esp-chacha20-" 2936 "poly1305-caam-qi2", 2937 .cra_blocksize = 1, 2938 }, 2939 .setkey = chachapoly_setkey, 2940 .setauthsize = chachapoly_setauthsize, 2941 .encrypt = aead_encrypt, 2942 .decrypt = aead_decrypt, 2943 .ivsize = 8, 2944 .maxauthsize = POLY1305_DIGEST_SIZE, 2945 }, 2946 .caam = { 2947 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2948 OP_ALG_AAI_AEAD, 2949 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2950 OP_ALG_AAI_AEAD, 2951 .nodkp = true, 2952 }, 2953 }, 2954 { 2955 .aead = { 2956 .base = { 2957 .cra_name = "authenc(hmac(sha512)," 2958 "rfc3686(ctr(aes)))", 2959 .cra_driver_name = "authenc-hmac-sha512-" 2960 "rfc3686-ctr-aes-caam-qi2", 2961 .cra_blocksize = 1, 2962 }, 2963 .setkey = aead_setkey, 2964 .setauthsize = aead_setauthsize, 2965 .encrypt = aead_encrypt, 2966 .decrypt = aead_decrypt, 2967 .ivsize = CTR_RFC3686_IV_SIZE, 2968 .maxauthsize = SHA512_DIGEST_SIZE, 2969 }, 2970 .caam = { 2971 .class1_alg_type = OP_ALG_ALGSEL_AES | 2972 OP_ALG_AAI_CTR_MOD128, 2973 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2974 OP_ALG_AAI_HMAC_PRECOMP, 2975 .rfc3686 = true, 2976 }, 2977 }, 2978 { 2979 .aead = { 2980 .base = { 2981 .cra_name = "seqiv(authenc(hmac(sha512)," 2982 "rfc3686(ctr(aes))))", 2983 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2984 "rfc3686-ctr-aes-caam-qi2", 2985 .cra_blocksize = 1, 2986 }, 2987 .setkey = aead_setkey, 2988 .setauthsize = aead_setauthsize, 2989 .encrypt = aead_encrypt, 2990 .decrypt = aead_decrypt, 2991 .ivsize = CTR_RFC3686_IV_SIZE, 2992 .maxauthsize = SHA512_DIGEST_SIZE, 2993 }, 2994 .caam = { 2995 .class1_alg_type = OP_ALG_ALGSEL_AES | 2996 OP_ALG_AAI_CTR_MOD128, 2997 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2998 OP_ALG_AAI_HMAC_PRECOMP, 2999 .rfc3686 = true, 3000 .geniv = true, 3001 }, 3002 }, 3003 }; 3004 3005 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3006 { 3007 struct skcipher_alg *alg = &t_alg->skcipher; 3008 3009 alg->base.cra_module = THIS_MODULE; 3010 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3011 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3012 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3013 CRYPTO_ALG_KERN_DRIVER_ONLY); 3014 3015 alg->init = caam_cra_init_skcipher; 3016 alg->exit = caam_cra_exit; 3017 } 3018 3019 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3020 { 3021 struct aead_alg *alg = &t_alg->aead; 3022 3023 alg->base.cra_module = THIS_MODULE; 3024 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3025 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3026 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3027 CRYPTO_ALG_KERN_DRIVER_ONLY; 3028 3029 alg->init = caam_cra_init_aead; 3030 alg->exit = caam_cra_exit_aead; 3031 } 3032 3033 /* max hash key is max split key size */ 3034 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 3035 3036 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 3037 3038 /* caam context sizes for hashes: running digest + 8 */ 3039 #define HASH_MSG_LEN 8 3040 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 3041 3042 enum hash_optype { 3043 UPDATE = 0, 3044 UPDATE_FIRST, 3045 FINALIZE, 3046 DIGEST, 3047 HASH_NUM_OP 3048 }; 3049 3050 /** 3051 * struct caam_hash_ctx - ahash per-session context 3052 * @flc: Flow Contexts array 3053 * @key: authentication key 3054 * @flc_dma: I/O virtual addresses of the Flow Contexts 3055 * @dev: dpseci device 3056 * @ctx_len: size of Context Register 3057 * @adata: hashing algorithm details 3058 */ 3059 struct caam_hash_ctx { 3060 struct caam_flc flc[HASH_NUM_OP]; 3061 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3062 dma_addr_t flc_dma[HASH_NUM_OP]; 3063 struct device *dev; 3064 int ctx_len; 3065 struct alginfo adata; 3066 }; 3067 3068 /* ahash state */ 3069 struct caam_hash_state { 3070 struct caam_request caam_req; 3071 dma_addr_t buf_dma; 3072 dma_addr_t ctx_dma; 3073 int ctx_dma_len; 3074 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3075 int buflen; 3076 int next_buflen; 3077 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 3078 int (*update)(struct ahash_request *req); 3079 int (*final)(struct ahash_request *req); 3080 int (*finup)(struct ahash_request *req); 3081 }; 3082 3083 struct caam_export_state { 3084 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 3085 u8 caam_ctx[MAX_CTX_LEN]; 3086 int buflen; 3087 int (*update)(struct ahash_request *req); 3088 int (*final)(struct ahash_request *req); 3089 int (*finup)(struct ahash_request *req); 3090 }; 3091 3092 /* Map current buffer in state (if length > 0) and put it in link table */ 3093 static inline int buf_map_to_qm_sg(struct device *dev, 3094 struct dpaa2_sg_entry *qm_sg, 3095 struct caam_hash_state *state) 3096 { 3097 int buflen = state->buflen; 3098 3099 if (!buflen) 3100 return 0; 3101 3102 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3103 DMA_TO_DEVICE); 3104 if (dma_mapping_error(dev, state->buf_dma)) { 3105 dev_err(dev, "unable to map buf\n"); 3106 state->buf_dma = 0; 3107 return -ENOMEM; 3108 } 3109 3110 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3111 3112 return 0; 3113 } 3114 3115 /* Map state->caam_ctx, and add it to link table */ 3116 static inline int ctx_map_to_qm_sg(struct device *dev, 3117 struct caam_hash_state *state, int ctx_len, 3118 struct dpaa2_sg_entry *qm_sg, u32 flag) 3119 { 3120 state->ctx_dma_len = ctx_len; 3121 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3122 if (dma_mapping_error(dev, state->ctx_dma)) { 3123 dev_err(dev, "unable to map ctx\n"); 3124 state->ctx_dma = 0; 3125 return -ENOMEM; 3126 } 3127 3128 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3129 3130 return 0; 3131 } 3132 3133 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3134 { 3135 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3136 int digestsize = crypto_ahash_digestsize(ahash); 3137 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3138 struct caam_flc *flc; 3139 u32 *desc; 3140 3141 /* ahash_update shared descriptor */ 3142 flc = &ctx->flc[UPDATE]; 3143 desc = flc->sh_desc; 3144 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3145 ctx->ctx_len, true, priv->sec_attr.era); 3146 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3147 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3148 desc_bytes(desc), DMA_BIDIRECTIONAL); 3149 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3150 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3151 1); 3152 3153 /* ahash_update_first shared descriptor */ 3154 flc = &ctx->flc[UPDATE_FIRST]; 3155 desc = flc->sh_desc; 3156 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3157 ctx->ctx_len, false, priv->sec_attr.era); 3158 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3159 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3160 desc_bytes(desc), DMA_BIDIRECTIONAL); 3161 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3162 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3163 1); 3164 3165 /* ahash_final shared descriptor */ 3166 flc = &ctx->flc[FINALIZE]; 3167 desc = flc->sh_desc; 3168 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3169 ctx->ctx_len, true, priv->sec_attr.era); 3170 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3171 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3172 desc_bytes(desc), DMA_BIDIRECTIONAL); 3173 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3174 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3175 1); 3176 3177 /* ahash_digest shared descriptor */ 3178 flc = &ctx->flc[DIGEST]; 3179 desc = flc->sh_desc; 3180 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3181 ctx->ctx_len, false, priv->sec_attr.era); 3182 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3183 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3184 desc_bytes(desc), DMA_BIDIRECTIONAL); 3185 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3186 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3187 1); 3188 3189 return 0; 3190 } 3191 3192 struct split_key_sh_result { 3193 struct completion completion; 3194 int err; 3195 struct device *dev; 3196 }; 3197 3198 static void split_key_sh_done(void *cbk_ctx, u32 err) 3199 { 3200 struct split_key_sh_result *res = cbk_ctx; 3201 3202 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3203 3204 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3205 complete(&res->completion); 3206 } 3207 3208 /* Digest hash size if it is too large */ 3209 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3210 u32 digestsize) 3211 { 3212 struct caam_request *req_ctx; 3213 u32 *desc; 3214 struct split_key_sh_result result; 3215 dma_addr_t key_dma; 3216 struct caam_flc *flc; 3217 dma_addr_t flc_dma; 3218 int ret = -ENOMEM; 3219 struct dpaa2_fl_entry *in_fle, *out_fle; 3220 3221 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); 3222 if (!req_ctx) 3223 return -ENOMEM; 3224 3225 in_fle = &req_ctx->fd_flt[1]; 3226 out_fle = &req_ctx->fd_flt[0]; 3227 3228 flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); 3229 if (!flc) 3230 goto err_flc; 3231 3232 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3233 if (dma_mapping_error(ctx->dev, key_dma)) { 3234 dev_err(ctx->dev, "unable to map key memory\n"); 3235 goto err_key_dma; 3236 } 3237 3238 desc = flc->sh_desc; 3239 3240 init_sh_desc(desc, 0); 3241 3242 /* descriptor to perform unkeyed hash on key_in */ 3243 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3244 OP_ALG_AS_INITFINAL); 3245 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3246 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3247 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3248 LDST_SRCDST_BYTE_CONTEXT); 3249 3250 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3251 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3252 desc_bytes(desc), DMA_TO_DEVICE); 3253 if (dma_mapping_error(ctx->dev, flc_dma)) { 3254 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3255 goto err_flc_dma; 3256 } 3257 3258 dpaa2_fl_set_final(in_fle, true); 3259 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3260 dpaa2_fl_set_addr(in_fle, key_dma); 3261 dpaa2_fl_set_len(in_fle, *keylen); 3262 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3263 dpaa2_fl_set_addr(out_fle, key_dma); 3264 dpaa2_fl_set_len(out_fle, digestsize); 3265 3266 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3267 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3268 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3269 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3270 1); 3271 3272 result.err = 0; 3273 init_completion(&result.completion); 3274 result.dev = ctx->dev; 3275 3276 req_ctx->flc = flc; 3277 req_ctx->flc_dma = flc_dma; 3278 req_ctx->cbk = split_key_sh_done; 3279 req_ctx->ctx = &result; 3280 3281 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3282 if (ret == -EINPROGRESS) { 3283 /* in progress */ 3284 wait_for_completion(&result.completion); 3285 ret = result.err; 3286 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3287 DUMP_PREFIX_ADDRESS, 16, 4, key, 3288 digestsize, 1); 3289 } 3290 3291 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3292 DMA_TO_DEVICE); 3293 err_flc_dma: 3294 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3295 err_key_dma: 3296 kfree(flc); 3297 err_flc: 3298 kfree(req_ctx); 3299 3300 *keylen = digestsize; 3301 3302 return ret; 3303 } 3304 3305 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3306 unsigned int keylen) 3307 { 3308 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3309 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3310 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3311 int ret; 3312 u8 *hashed_key = NULL; 3313 3314 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3315 3316 if (keylen > blocksize) { 3317 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 3318 if (!hashed_key) 3319 return -ENOMEM; 3320 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3321 if (ret) 3322 goto bad_free_key; 3323 key = hashed_key; 3324 } 3325 3326 ctx->adata.keylen = keylen; 3327 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3328 OP_ALG_ALGSEL_MASK); 3329 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3330 goto bad_free_key; 3331 3332 ctx->adata.key_virt = key; 3333 ctx->adata.key_inline = true; 3334 3335 /* 3336 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3337 * in invalid opcodes (last bytes of user key) in the resulting 3338 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3339 * addresses are needed. 3340 */ 3341 if (keylen > ctx->adata.keylen_pad) { 3342 memcpy(ctx->key, key, keylen); 3343 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3344 ctx->adata.keylen_pad, 3345 DMA_TO_DEVICE); 3346 } 3347 3348 ret = ahash_set_sh_desc(ahash); 3349 kfree(hashed_key); 3350 return ret; 3351 bad_free_key: 3352 kfree(hashed_key); 3353 return -EINVAL; 3354 } 3355 3356 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3357 struct ahash_request *req) 3358 { 3359 struct caam_hash_state *state = ahash_request_ctx(req); 3360 3361 if (edesc->src_nents) 3362 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3363 3364 if (edesc->qm_sg_bytes) 3365 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3366 DMA_TO_DEVICE); 3367 3368 if (state->buf_dma) { 3369 dma_unmap_single(dev, state->buf_dma, state->buflen, 3370 DMA_TO_DEVICE); 3371 state->buf_dma = 0; 3372 } 3373 } 3374 3375 static inline void ahash_unmap_ctx(struct device *dev, 3376 struct ahash_edesc *edesc, 3377 struct ahash_request *req, u32 flag) 3378 { 3379 struct caam_hash_state *state = ahash_request_ctx(req); 3380 3381 if (state->ctx_dma) { 3382 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3383 state->ctx_dma = 0; 3384 } 3385 ahash_unmap(dev, edesc, req); 3386 } 3387 3388 static void ahash_done(void *cbk_ctx, u32 status) 3389 { 3390 struct crypto_async_request *areq = cbk_ctx; 3391 struct ahash_request *req = ahash_request_cast(areq); 3392 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3393 struct caam_hash_state *state = ahash_request_ctx(req); 3394 struct ahash_edesc *edesc = state->caam_req.edesc; 3395 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3396 int digestsize = crypto_ahash_digestsize(ahash); 3397 int ecode = 0; 3398 3399 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3400 3401 if (unlikely(status)) 3402 ecode = caam_qi2_strstatus(ctx->dev, status); 3403 3404 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3405 memcpy(req->result, state->caam_ctx, digestsize); 3406 qi_cache_free(edesc); 3407 3408 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3409 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3410 ctx->ctx_len, 1); 3411 3412 req->base.complete(&req->base, ecode); 3413 } 3414 3415 static void ahash_done_bi(void *cbk_ctx, u32 status) 3416 { 3417 struct crypto_async_request *areq = cbk_ctx; 3418 struct ahash_request *req = ahash_request_cast(areq); 3419 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3420 struct caam_hash_state *state = ahash_request_ctx(req); 3421 struct ahash_edesc *edesc = state->caam_req.edesc; 3422 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3423 int ecode = 0; 3424 3425 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3426 3427 if (unlikely(status)) 3428 ecode = caam_qi2_strstatus(ctx->dev, status); 3429 3430 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3431 qi_cache_free(edesc); 3432 3433 scatterwalk_map_and_copy(state->buf, req->src, 3434 req->nbytes - state->next_buflen, 3435 state->next_buflen, 0); 3436 state->buflen = state->next_buflen; 3437 3438 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3439 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3440 state->buflen, 1); 3441 3442 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3443 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3444 ctx->ctx_len, 1); 3445 if (req->result) 3446 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3447 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3448 crypto_ahash_digestsize(ahash), 1); 3449 3450 req->base.complete(&req->base, ecode); 3451 } 3452 3453 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3454 { 3455 struct crypto_async_request *areq = cbk_ctx; 3456 struct ahash_request *req = ahash_request_cast(areq); 3457 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3458 struct caam_hash_state *state = ahash_request_ctx(req); 3459 struct ahash_edesc *edesc = state->caam_req.edesc; 3460 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3461 int digestsize = crypto_ahash_digestsize(ahash); 3462 int ecode = 0; 3463 3464 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3465 3466 if (unlikely(status)) 3467 ecode = caam_qi2_strstatus(ctx->dev, status); 3468 3469 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3470 memcpy(req->result, state->caam_ctx, digestsize); 3471 qi_cache_free(edesc); 3472 3473 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3474 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3475 ctx->ctx_len, 1); 3476 3477 req->base.complete(&req->base, ecode); 3478 } 3479 3480 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3481 { 3482 struct crypto_async_request *areq = cbk_ctx; 3483 struct ahash_request *req = ahash_request_cast(areq); 3484 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3485 struct caam_hash_state *state = ahash_request_ctx(req); 3486 struct ahash_edesc *edesc = state->caam_req.edesc; 3487 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3488 int ecode = 0; 3489 3490 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3491 3492 if (unlikely(status)) 3493 ecode = caam_qi2_strstatus(ctx->dev, status); 3494 3495 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3496 qi_cache_free(edesc); 3497 3498 scatterwalk_map_and_copy(state->buf, req->src, 3499 req->nbytes - state->next_buflen, 3500 state->next_buflen, 0); 3501 state->buflen = state->next_buflen; 3502 3503 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3504 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3505 state->buflen, 1); 3506 3507 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3508 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3509 ctx->ctx_len, 1); 3510 if (req->result) 3511 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3512 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3513 crypto_ahash_digestsize(ahash), 1); 3514 3515 req->base.complete(&req->base, ecode); 3516 } 3517 3518 static int ahash_update_ctx(struct ahash_request *req) 3519 { 3520 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3521 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3522 struct caam_hash_state *state = ahash_request_ctx(req); 3523 struct caam_request *req_ctx = &state->caam_req; 3524 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3525 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3526 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3527 GFP_KERNEL : GFP_ATOMIC; 3528 u8 *buf = state->buf; 3529 int *buflen = &state->buflen; 3530 int *next_buflen = &state->next_buflen; 3531 int in_len = *buflen + req->nbytes, to_hash; 3532 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3533 struct ahash_edesc *edesc; 3534 int ret = 0; 3535 3536 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3537 to_hash = in_len - *next_buflen; 3538 3539 if (to_hash) { 3540 struct dpaa2_sg_entry *sg_table; 3541 int src_len = req->nbytes - *next_buflen; 3542 3543 src_nents = sg_nents_for_len(req->src, src_len); 3544 if (src_nents < 0) { 3545 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3546 return src_nents; 3547 } 3548 3549 if (src_nents) { 3550 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3551 DMA_TO_DEVICE); 3552 if (!mapped_nents) { 3553 dev_err(ctx->dev, "unable to DMA map source\n"); 3554 return -ENOMEM; 3555 } 3556 } else { 3557 mapped_nents = 0; 3558 } 3559 3560 /* allocate space for base edesc and link tables */ 3561 edesc = qi_cache_zalloc(GFP_DMA | flags); 3562 if (!edesc) { 3563 dma_unmap_sg(ctx->dev, req->src, src_nents, 3564 DMA_TO_DEVICE); 3565 return -ENOMEM; 3566 } 3567 3568 edesc->src_nents = src_nents; 3569 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3570 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3571 sizeof(*sg_table); 3572 sg_table = &edesc->sgt[0]; 3573 3574 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3575 DMA_BIDIRECTIONAL); 3576 if (ret) 3577 goto unmap_ctx; 3578 3579 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3580 if (ret) 3581 goto unmap_ctx; 3582 3583 if (mapped_nents) { 3584 sg_to_qm_sg_last(req->src, src_len, 3585 sg_table + qm_sg_src_index, 0); 3586 } else { 3587 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3588 true); 3589 } 3590 3591 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3592 qm_sg_bytes, DMA_TO_DEVICE); 3593 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3594 dev_err(ctx->dev, "unable to map S/G table\n"); 3595 ret = -ENOMEM; 3596 goto unmap_ctx; 3597 } 3598 edesc->qm_sg_bytes = qm_sg_bytes; 3599 3600 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3601 dpaa2_fl_set_final(in_fle, true); 3602 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3603 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3604 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3605 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3606 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3607 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3608 3609 req_ctx->flc = &ctx->flc[UPDATE]; 3610 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3611 req_ctx->cbk = ahash_done_bi; 3612 req_ctx->ctx = &req->base; 3613 req_ctx->edesc = edesc; 3614 3615 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3616 if (ret != -EINPROGRESS && 3617 !(ret == -EBUSY && 3618 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3619 goto unmap_ctx; 3620 } else if (*next_buflen) { 3621 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3622 req->nbytes, 0); 3623 *buflen = *next_buflen; 3624 3625 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3626 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3627 *buflen, 1); 3628 } 3629 3630 return ret; 3631 unmap_ctx: 3632 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3633 qi_cache_free(edesc); 3634 return ret; 3635 } 3636 3637 static int ahash_final_ctx(struct ahash_request *req) 3638 { 3639 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3640 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3641 struct caam_hash_state *state = ahash_request_ctx(req); 3642 struct caam_request *req_ctx = &state->caam_req; 3643 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3644 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3645 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3646 GFP_KERNEL : GFP_ATOMIC; 3647 int buflen = state->buflen; 3648 int qm_sg_bytes; 3649 int digestsize = crypto_ahash_digestsize(ahash); 3650 struct ahash_edesc *edesc; 3651 struct dpaa2_sg_entry *sg_table; 3652 int ret; 3653 3654 /* allocate space for base edesc and link tables */ 3655 edesc = qi_cache_zalloc(GFP_DMA | flags); 3656 if (!edesc) 3657 return -ENOMEM; 3658 3659 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3660 sg_table = &edesc->sgt[0]; 3661 3662 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3663 DMA_BIDIRECTIONAL); 3664 if (ret) 3665 goto unmap_ctx; 3666 3667 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3668 if (ret) 3669 goto unmap_ctx; 3670 3671 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3672 3673 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3674 DMA_TO_DEVICE); 3675 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3676 dev_err(ctx->dev, "unable to map S/G table\n"); 3677 ret = -ENOMEM; 3678 goto unmap_ctx; 3679 } 3680 edesc->qm_sg_bytes = qm_sg_bytes; 3681 3682 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3683 dpaa2_fl_set_final(in_fle, true); 3684 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3685 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3686 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3687 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3688 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3689 dpaa2_fl_set_len(out_fle, digestsize); 3690 3691 req_ctx->flc = &ctx->flc[FINALIZE]; 3692 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3693 req_ctx->cbk = ahash_done_ctx_src; 3694 req_ctx->ctx = &req->base; 3695 req_ctx->edesc = edesc; 3696 3697 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3698 if (ret == -EINPROGRESS || 3699 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3700 return ret; 3701 3702 unmap_ctx: 3703 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3704 qi_cache_free(edesc); 3705 return ret; 3706 } 3707 3708 static int ahash_finup_ctx(struct ahash_request *req) 3709 { 3710 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3711 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3712 struct caam_hash_state *state = ahash_request_ctx(req); 3713 struct caam_request *req_ctx = &state->caam_req; 3714 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3715 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3716 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3717 GFP_KERNEL : GFP_ATOMIC; 3718 int buflen = state->buflen; 3719 int qm_sg_bytes, qm_sg_src_index; 3720 int src_nents, mapped_nents; 3721 int digestsize = crypto_ahash_digestsize(ahash); 3722 struct ahash_edesc *edesc; 3723 struct dpaa2_sg_entry *sg_table; 3724 int ret; 3725 3726 src_nents = sg_nents_for_len(req->src, req->nbytes); 3727 if (src_nents < 0) { 3728 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3729 return src_nents; 3730 } 3731 3732 if (src_nents) { 3733 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3734 DMA_TO_DEVICE); 3735 if (!mapped_nents) { 3736 dev_err(ctx->dev, "unable to DMA map source\n"); 3737 return -ENOMEM; 3738 } 3739 } else { 3740 mapped_nents = 0; 3741 } 3742 3743 /* allocate space for base edesc and link tables */ 3744 edesc = qi_cache_zalloc(GFP_DMA | flags); 3745 if (!edesc) { 3746 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3747 return -ENOMEM; 3748 } 3749 3750 edesc->src_nents = src_nents; 3751 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3752 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3753 sizeof(*sg_table); 3754 sg_table = &edesc->sgt[0]; 3755 3756 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3757 DMA_BIDIRECTIONAL); 3758 if (ret) 3759 goto unmap_ctx; 3760 3761 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3762 if (ret) 3763 goto unmap_ctx; 3764 3765 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3766 3767 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3768 DMA_TO_DEVICE); 3769 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3770 dev_err(ctx->dev, "unable to map S/G table\n"); 3771 ret = -ENOMEM; 3772 goto unmap_ctx; 3773 } 3774 edesc->qm_sg_bytes = qm_sg_bytes; 3775 3776 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3777 dpaa2_fl_set_final(in_fle, true); 3778 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3779 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3780 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3781 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3782 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3783 dpaa2_fl_set_len(out_fle, digestsize); 3784 3785 req_ctx->flc = &ctx->flc[FINALIZE]; 3786 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3787 req_ctx->cbk = ahash_done_ctx_src; 3788 req_ctx->ctx = &req->base; 3789 req_ctx->edesc = edesc; 3790 3791 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3792 if (ret == -EINPROGRESS || 3793 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3794 return ret; 3795 3796 unmap_ctx: 3797 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3798 qi_cache_free(edesc); 3799 return ret; 3800 } 3801 3802 static int ahash_digest(struct ahash_request *req) 3803 { 3804 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3805 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3806 struct caam_hash_state *state = ahash_request_ctx(req); 3807 struct caam_request *req_ctx = &state->caam_req; 3808 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3809 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3810 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3811 GFP_KERNEL : GFP_ATOMIC; 3812 int digestsize = crypto_ahash_digestsize(ahash); 3813 int src_nents, mapped_nents; 3814 struct ahash_edesc *edesc; 3815 int ret = -ENOMEM; 3816 3817 state->buf_dma = 0; 3818 3819 src_nents = sg_nents_for_len(req->src, req->nbytes); 3820 if (src_nents < 0) { 3821 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3822 return src_nents; 3823 } 3824 3825 if (src_nents) { 3826 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3827 DMA_TO_DEVICE); 3828 if (!mapped_nents) { 3829 dev_err(ctx->dev, "unable to map source for DMA\n"); 3830 return ret; 3831 } 3832 } else { 3833 mapped_nents = 0; 3834 } 3835 3836 /* allocate space for base edesc and link tables */ 3837 edesc = qi_cache_zalloc(GFP_DMA | flags); 3838 if (!edesc) { 3839 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3840 return ret; 3841 } 3842 3843 edesc->src_nents = src_nents; 3844 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3845 3846 if (mapped_nents > 1) { 3847 int qm_sg_bytes; 3848 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3849 3850 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3851 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3852 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3853 qm_sg_bytes, DMA_TO_DEVICE); 3854 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3855 dev_err(ctx->dev, "unable to map S/G table\n"); 3856 goto unmap; 3857 } 3858 edesc->qm_sg_bytes = qm_sg_bytes; 3859 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3860 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3861 } else { 3862 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3863 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3864 } 3865 3866 state->ctx_dma_len = digestsize; 3867 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3868 DMA_FROM_DEVICE); 3869 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3870 dev_err(ctx->dev, "unable to map ctx\n"); 3871 state->ctx_dma = 0; 3872 goto unmap; 3873 } 3874 3875 dpaa2_fl_set_final(in_fle, true); 3876 dpaa2_fl_set_len(in_fle, req->nbytes); 3877 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3878 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3879 dpaa2_fl_set_len(out_fle, digestsize); 3880 3881 req_ctx->flc = &ctx->flc[DIGEST]; 3882 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3883 req_ctx->cbk = ahash_done; 3884 req_ctx->ctx = &req->base; 3885 req_ctx->edesc = edesc; 3886 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3887 if (ret == -EINPROGRESS || 3888 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3889 return ret; 3890 3891 unmap: 3892 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3893 qi_cache_free(edesc); 3894 return ret; 3895 } 3896 3897 static int ahash_final_no_ctx(struct ahash_request *req) 3898 { 3899 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3900 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3901 struct caam_hash_state *state = ahash_request_ctx(req); 3902 struct caam_request *req_ctx = &state->caam_req; 3903 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3904 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3905 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3906 GFP_KERNEL : GFP_ATOMIC; 3907 u8 *buf = state->buf; 3908 int buflen = state->buflen; 3909 int digestsize = crypto_ahash_digestsize(ahash); 3910 struct ahash_edesc *edesc; 3911 int ret = -ENOMEM; 3912 3913 /* allocate space for base edesc and link tables */ 3914 edesc = qi_cache_zalloc(GFP_DMA | flags); 3915 if (!edesc) 3916 return ret; 3917 3918 if (buflen) { 3919 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3920 DMA_TO_DEVICE); 3921 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3922 dev_err(ctx->dev, "unable to map src\n"); 3923 goto unmap; 3924 } 3925 } 3926 3927 state->ctx_dma_len = digestsize; 3928 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3929 DMA_FROM_DEVICE); 3930 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3931 dev_err(ctx->dev, "unable to map ctx\n"); 3932 state->ctx_dma = 0; 3933 goto unmap; 3934 } 3935 3936 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3937 dpaa2_fl_set_final(in_fle, true); 3938 /* 3939 * crypto engine requires the input entry to be present when 3940 * "frame list" FD is used. 3941 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3942 * in_fle zeroized (except for "Final" flag) is the best option. 3943 */ 3944 if (buflen) { 3945 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3946 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3947 dpaa2_fl_set_len(in_fle, buflen); 3948 } 3949 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3950 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3951 dpaa2_fl_set_len(out_fle, digestsize); 3952 3953 req_ctx->flc = &ctx->flc[DIGEST]; 3954 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3955 req_ctx->cbk = ahash_done; 3956 req_ctx->ctx = &req->base; 3957 req_ctx->edesc = edesc; 3958 3959 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3960 if (ret == -EINPROGRESS || 3961 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3962 return ret; 3963 3964 unmap: 3965 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3966 qi_cache_free(edesc); 3967 return ret; 3968 } 3969 3970 static int ahash_update_no_ctx(struct ahash_request *req) 3971 { 3972 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3973 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3974 struct caam_hash_state *state = ahash_request_ctx(req); 3975 struct caam_request *req_ctx = &state->caam_req; 3976 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3977 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3978 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3979 GFP_KERNEL : GFP_ATOMIC; 3980 u8 *buf = state->buf; 3981 int *buflen = &state->buflen; 3982 int *next_buflen = &state->next_buflen; 3983 int in_len = *buflen + req->nbytes, to_hash; 3984 int qm_sg_bytes, src_nents, mapped_nents; 3985 struct ahash_edesc *edesc; 3986 int ret = 0; 3987 3988 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3989 to_hash = in_len - *next_buflen; 3990 3991 if (to_hash) { 3992 struct dpaa2_sg_entry *sg_table; 3993 int src_len = req->nbytes - *next_buflen; 3994 3995 src_nents = sg_nents_for_len(req->src, src_len); 3996 if (src_nents < 0) { 3997 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3998 return src_nents; 3999 } 4000 4001 if (src_nents) { 4002 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4003 DMA_TO_DEVICE); 4004 if (!mapped_nents) { 4005 dev_err(ctx->dev, "unable to DMA map source\n"); 4006 return -ENOMEM; 4007 } 4008 } else { 4009 mapped_nents = 0; 4010 } 4011 4012 /* allocate space for base edesc and link tables */ 4013 edesc = qi_cache_zalloc(GFP_DMA | flags); 4014 if (!edesc) { 4015 dma_unmap_sg(ctx->dev, req->src, src_nents, 4016 DMA_TO_DEVICE); 4017 return -ENOMEM; 4018 } 4019 4020 edesc->src_nents = src_nents; 4021 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 4022 sizeof(*sg_table); 4023 sg_table = &edesc->sgt[0]; 4024 4025 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4026 if (ret) 4027 goto unmap_ctx; 4028 4029 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 4030 4031 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4032 qm_sg_bytes, DMA_TO_DEVICE); 4033 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4034 dev_err(ctx->dev, "unable to map S/G table\n"); 4035 ret = -ENOMEM; 4036 goto unmap_ctx; 4037 } 4038 edesc->qm_sg_bytes = qm_sg_bytes; 4039 4040 state->ctx_dma_len = ctx->ctx_len; 4041 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4042 ctx->ctx_len, DMA_FROM_DEVICE); 4043 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4044 dev_err(ctx->dev, "unable to map ctx\n"); 4045 state->ctx_dma = 0; 4046 ret = -ENOMEM; 4047 goto unmap_ctx; 4048 } 4049 4050 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4051 dpaa2_fl_set_final(in_fle, true); 4052 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4053 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4054 dpaa2_fl_set_len(in_fle, to_hash); 4055 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4056 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4057 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4058 4059 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4060 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4061 req_ctx->cbk = ahash_done_ctx_dst; 4062 req_ctx->ctx = &req->base; 4063 req_ctx->edesc = edesc; 4064 4065 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4066 if (ret != -EINPROGRESS && 4067 !(ret == -EBUSY && 4068 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4069 goto unmap_ctx; 4070 4071 state->update = ahash_update_ctx; 4072 state->finup = ahash_finup_ctx; 4073 state->final = ahash_final_ctx; 4074 } else if (*next_buflen) { 4075 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 4076 req->nbytes, 0); 4077 *buflen = *next_buflen; 4078 4079 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4080 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4081 *buflen, 1); 4082 } 4083 4084 return ret; 4085 unmap_ctx: 4086 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4087 qi_cache_free(edesc); 4088 return ret; 4089 } 4090 4091 static int ahash_finup_no_ctx(struct ahash_request *req) 4092 { 4093 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4094 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4095 struct caam_hash_state *state = ahash_request_ctx(req); 4096 struct caam_request *req_ctx = &state->caam_req; 4097 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4098 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4099 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4100 GFP_KERNEL : GFP_ATOMIC; 4101 int buflen = state->buflen; 4102 int qm_sg_bytes, src_nents, mapped_nents; 4103 int digestsize = crypto_ahash_digestsize(ahash); 4104 struct ahash_edesc *edesc; 4105 struct dpaa2_sg_entry *sg_table; 4106 int ret = -ENOMEM; 4107 4108 src_nents = sg_nents_for_len(req->src, req->nbytes); 4109 if (src_nents < 0) { 4110 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4111 return src_nents; 4112 } 4113 4114 if (src_nents) { 4115 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4116 DMA_TO_DEVICE); 4117 if (!mapped_nents) { 4118 dev_err(ctx->dev, "unable to DMA map source\n"); 4119 return ret; 4120 } 4121 } else { 4122 mapped_nents = 0; 4123 } 4124 4125 /* allocate space for base edesc and link tables */ 4126 edesc = qi_cache_zalloc(GFP_DMA | flags); 4127 if (!edesc) { 4128 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4129 return ret; 4130 } 4131 4132 edesc->src_nents = src_nents; 4133 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4134 sg_table = &edesc->sgt[0]; 4135 4136 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4137 if (ret) 4138 goto unmap; 4139 4140 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4141 4142 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4143 DMA_TO_DEVICE); 4144 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4145 dev_err(ctx->dev, "unable to map S/G table\n"); 4146 ret = -ENOMEM; 4147 goto unmap; 4148 } 4149 edesc->qm_sg_bytes = qm_sg_bytes; 4150 4151 state->ctx_dma_len = digestsize; 4152 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4153 DMA_FROM_DEVICE); 4154 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4155 dev_err(ctx->dev, "unable to map ctx\n"); 4156 state->ctx_dma = 0; 4157 ret = -ENOMEM; 4158 goto unmap; 4159 } 4160 4161 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4162 dpaa2_fl_set_final(in_fle, true); 4163 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4164 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4165 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4166 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4167 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4168 dpaa2_fl_set_len(out_fle, digestsize); 4169 4170 req_ctx->flc = &ctx->flc[DIGEST]; 4171 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4172 req_ctx->cbk = ahash_done; 4173 req_ctx->ctx = &req->base; 4174 req_ctx->edesc = edesc; 4175 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4176 if (ret != -EINPROGRESS && 4177 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4178 goto unmap; 4179 4180 return ret; 4181 unmap: 4182 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4183 qi_cache_free(edesc); 4184 return ret; 4185 } 4186 4187 static int ahash_update_first(struct ahash_request *req) 4188 { 4189 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4190 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4191 struct caam_hash_state *state = ahash_request_ctx(req); 4192 struct caam_request *req_ctx = &state->caam_req; 4193 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4194 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4195 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4196 GFP_KERNEL : GFP_ATOMIC; 4197 u8 *buf = state->buf; 4198 int *buflen = &state->buflen; 4199 int *next_buflen = &state->next_buflen; 4200 int to_hash; 4201 int src_nents, mapped_nents; 4202 struct ahash_edesc *edesc; 4203 int ret = 0; 4204 4205 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4206 1); 4207 to_hash = req->nbytes - *next_buflen; 4208 4209 if (to_hash) { 4210 struct dpaa2_sg_entry *sg_table; 4211 int src_len = req->nbytes - *next_buflen; 4212 4213 src_nents = sg_nents_for_len(req->src, src_len); 4214 if (src_nents < 0) { 4215 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4216 return src_nents; 4217 } 4218 4219 if (src_nents) { 4220 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4221 DMA_TO_DEVICE); 4222 if (!mapped_nents) { 4223 dev_err(ctx->dev, "unable to map source for DMA\n"); 4224 return -ENOMEM; 4225 } 4226 } else { 4227 mapped_nents = 0; 4228 } 4229 4230 /* allocate space for base edesc and link tables */ 4231 edesc = qi_cache_zalloc(GFP_DMA | flags); 4232 if (!edesc) { 4233 dma_unmap_sg(ctx->dev, req->src, src_nents, 4234 DMA_TO_DEVICE); 4235 return -ENOMEM; 4236 } 4237 4238 edesc->src_nents = src_nents; 4239 sg_table = &edesc->sgt[0]; 4240 4241 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4242 dpaa2_fl_set_final(in_fle, true); 4243 dpaa2_fl_set_len(in_fle, to_hash); 4244 4245 if (mapped_nents > 1) { 4246 int qm_sg_bytes; 4247 4248 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4249 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4250 sizeof(*sg_table); 4251 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4252 qm_sg_bytes, 4253 DMA_TO_DEVICE); 4254 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4255 dev_err(ctx->dev, "unable to map S/G table\n"); 4256 ret = -ENOMEM; 4257 goto unmap_ctx; 4258 } 4259 edesc->qm_sg_bytes = qm_sg_bytes; 4260 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4261 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4262 } else { 4263 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4264 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4265 } 4266 4267 state->ctx_dma_len = ctx->ctx_len; 4268 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4269 ctx->ctx_len, DMA_FROM_DEVICE); 4270 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4271 dev_err(ctx->dev, "unable to map ctx\n"); 4272 state->ctx_dma = 0; 4273 ret = -ENOMEM; 4274 goto unmap_ctx; 4275 } 4276 4277 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4278 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4279 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4280 4281 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4282 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4283 req_ctx->cbk = ahash_done_ctx_dst; 4284 req_ctx->ctx = &req->base; 4285 req_ctx->edesc = edesc; 4286 4287 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4288 if (ret != -EINPROGRESS && 4289 !(ret == -EBUSY && req->base.flags & 4290 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4291 goto unmap_ctx; 4292 4293 state->update = ahash_update_ctx; 4294 state->finup = ahash_finup_ctx; 4295 state->final = ahash_final_ctx; 4296 } else if (*next_buflen) { 4297 state->update = ahash_update_no_ctx; 4298 state->finup = ahash_finup_no_ctx; 4299 state->final = ahash_final_no_ctx; 4300 scatterwalk_map_and_copy(buf, req->src, 0, 4301 req->nbytes, 0); 4302 *buflen = *next_buflen; 4303 4304 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4305 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4306 *buflen, 1); 4307 } 4308 4309 return ret; 4310 unmap_ctx: 4311 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4312 qi_cache_free(edesc); 4313 return ret; 4314 } 4315 4316 static int ahash_finup_first(struct ahash_request *req) 4317 { 4318 return ahash_digest(req); 4319 } 4320 4321 static int ahash_init(struct ahash_request *req) 4322 { 4323 struct caam_hash_state *state = ahash_request_ctx(req); 4324 4325 state->update = ahash_update_first; 4326 state->finup = ahash_finup_first; 4327 state->final = ahash_final_no_ctx; 4328 4329 state->ctx_dma = 0; 4330 state->ctx_dma_len = 0; 4331 state->buf_dma = 0; 4332 state->buflen = 0; 4333 state->next_buflen = 0; 4334 4335 return 0; 4336 } 4337 4338 static int ahash_update(struct ahash_request *req) 4339 { 4340 struct caam_hash_state *state = ahash_request_ctx(req); 4341 4342 return state->update(req); 4343 } 4344 4345 static int ahash_finup(struct ahash_request *req) 4346 { 4347 struct caam_hash_state *state = ahash_request_ctx(req); 4348 4349 return state->finup(req); 4350 } 4351 4352 static int ahash_final(struct ahash_request *req) 4353 { 4354 struct caam_hash_state *state = ahash_request_ctx(req); 4355 4356 return state->final(req); 4357 } 4358 4359 static int ahash_export(struct ahash_request *req, void *out) 4360 { 4361 struct caam_hash_state *state = ahash_request_ctx(req); 4362 struct caam_export_state *export = out; 4363 u8 *buf = state->buf; 4364 int len = state->buflen; 4365 4366 memcpy(export->buf, buf, len); 4367 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4368 export->buflen = len; 4369 export->update = state->update; 4370 export->final = state->final; 4371 export->finup = state->finup; 4372 4373 return 0; 4374 } 4375 4376 static int ahash_import(struct ahash_request *req, const void *in) 4377 { 4378 struct caam_hash_state *state = ahash_request_ctx(req); 4379 const struct caam_export_state *export = in; 4380 4381 memset(state, 0, sizeof(*state)); 4382 memcpy(state->buf, export->buf, export->buflen); 4383 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4384 state->buflen = export->buflen; 4385 state->update = export->update; 4386 state->final = export->final; 4387 state->finup = export->finup; 4388 4389 return 0; 4390 } 4391 4392 struct caam_hash_template { 4393 char name[CRYPTO_MAX_ALG_NAME]; 4394 char driver_name[CRYPTO_MAX_ALG_NAME]; 4395 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4396 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4397 unsigned int blocksize; 4398 struct ahash_alg template_ahash; 4399 u32 alg_type; 4400 }; 4401 4402 /* ahash descriptors */ 4403 static struct caam_hash_template driver_hash[] = { 4404 { 4405 .name = "sha1", 4406 .driver_name = "sha1-caam-qi2", 4407 .hmac_name = "hmac(sha1)", 4408 .hmac_driver_name = "hmac-sha1-caam-qi2", 4409 .blocksize = SHA1_BLOCK_SIZE, 4410 .template_ahash = { 4411 .init = ahash_init, 4412 .update = ahash_update, 4413 .final = ahash_final, 4414 .finup = ahash_finup, 4415 .digest = ahash_digest, 4416 .export = ahash_export, 4417 .import = ahash_import, 4418 .setkey = ahash_setkey, 4419 .halg = { 4420 .digestsize = SHA1_DIGEST_SIZE, 4421 .statesize = sizeof(struct caam_export_state), 4422 }, 4423 }, 4424 .alg_type = OP_ALG_ALGSEL_SHA1, 4425 }, { 4426 .name = "sha224", 4427 .driver_name = "sha224-caam-qi2", 4428 .hmac_name = "hmac(sha224)", 4429 .hmac_driver_name = "hmac-sha224-caam-qi2", 4430 .blocksize = SHA224_BLOCK_SIZE, 4431 .template_ahash = { 4432 .init = ahash_init, 4433 .update = ahash_update, 4434 .final = ahash_final, 4435 .finup = ahash_finup, 4436 .digest = ahash_digest, 4437 .export = ahash_export, 4438 .import = ahash_import, 4439 .setkey = ahash_setkey, 4440 .halg = { 4441 .digestsize = SHA224_DIGEST_SIZE, 4442 .statesize = sizeof(struct caam_export_state), 4443 }, 4444 }, 4445 .alg_type = OP_ALG_ALGSEL_SHA224, 4446 }, { 4447 .name = "sha256", 4448 .driver_name = "sha256-caam-qi2", 4449 .hmac_name = "hmac(sha256)", 4450 .hmac_driver_name = "hmac-sha256-caam-qi2", 4451 .blocksize = SHA256_BLOCK_SIZE, 4452 .template_ahash = { 4453 .init = ahash_init, 4454 .update = ahash_update, 4455 .final = ahash_final, 4456 .finup = ahash_finup, 4457 .digest = ahash_digest, 4458 .export = ahash_export, 4459 .import = ahash_import, 4460 .setkey = ahash_setkey, 4461 .halg = { 4462 .digestsize = SHA256_DIGEST_SIZE, 4463 .statesize = sizeof(struct caam_export_state), 4464 }, 4465 }, 4466 .alg_type = OP_ALG_ALGSEL_SHA256, 4467 }, { 4468 .name = "sha384", 4469 .driver_name = "sha384-caam-qi2", 4470 .hmac_name = "hmac(sha384)", 4471 .hmac_driver_name = "hmac-sha384-caam-qi2", 4472 .blocksize = SHA384_BLOCK_SIZE, 4473 .template_ahash = { 4474 .init = ahash_init, 4475 .update = ahash_update, 4476 .final = ahash_final, 4477 .finup = ahash_finup, 4478 .digest = ahash_digest, 4479 .export = ahash_export, 4480 .import = ahash_import, 4481 .setkey = ahash_setkey, 4482 .halg = { 4483 .digestsize = SHA384_DIGEST_SIZE, 4484 .statesize = sizeof(struct caam_export_state), 4485 }, 4486 }, 4487 .alg_type = OP_ALG_ALGSEL_SHA384, 4488 }, { 4489 .name = "sha512", 4490 .driver_name = "sha512-caam-qi2", 4491 .hmac_name = "hmac(sha512)", 4492 .hmac_driver_name = "hmac-sha512-caam-qi2", 4493 .blocksize = SHA512_BLOCK_SIZE, 4494 .template_ahash = { 4495 .init = ahash_init, 4496 .update = ahash_update, 4497 .final = ahash_final, 4498 .finup = ahash_finup, 4499 .digest = ahash_digest, 4500 .export = ahash_export, 4501 .import = ahash_import, 4502 .setkey = ahash_setkey, 4503 .halg = { 4504 .digestsize = SHA512_DIGEST_SIZE, 4505 .statesize = sizeof(struct caam_export_state), 4506 }, 4507 }, 4508 .alg_type = OP_ALG_ALGSEL_SHA512, 4509 }, { 4510 .name = "md5", 4511 .driver_name = "md5-caam-qi2", 4512 .hmac_name = "hmac(md5)", 4513 .hmac_driver_name = "hmac-md5-caam-qi2", 4514 .blocksize = MD5_BLOCK_WORDS * 4, 4515 .template_ahash = { 4516 .init = ahash_init, 4517 .update = ahash_update, 4518 .final = ahash_final, 4519 .finup = ahash_finup, 4520 .digest = ahash_digest, 4521 .export = ahash_export, 4522 .import = ahash_import, 4523 .setkey = ahash_setkey, 4524 .halg = { 4525 .digestsize = MD5_DIGEST_SIZE, 4526 .statesize = sizeof(struct caam_export_state), 4527 }, 4528 }, 4529 .alg_type = OP_ALG_ALGSEL_MD5, 4530 } 4531 }; 4532 4533 struct caam_hash_alg { 4534 struct list_head entry; 4535 struct device *dev; 4536 int alg_type; 4537 struct ahash_alg ahash_alg; 4538 }; 4539 4540 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4541 { 4542 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4543 struct crypto_alg *base = tfm->__crt_alg; 4544 struct hash_alg_common *halg = 4545 container_of(base, struct hash_alg_common, base); 4546 struct ahash_alg *alg = 4547 container_of(halg, struct ahash_alg, halg); 4548 struct caam_hash_alg *caam_hash = 4549 container_of(alg, struct caam_hash_alg, ahash_alg); 4550 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4551 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4552 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4553 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4554 HASH_MSG_LEN + 32, 4555 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4556 HASH_MSG_LEN + 64, 4557 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4558 dma_addr_t dma_addr; 4559 int i; 4560 4561 ctx->dev = caam_hash->dev; 4562 4563 if (alg->setkey) { 4564 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4565 ARRAY_SIZE(ctx->key), 4566 DMA_TO_DEVICE, 4567 DMA_ATTR_SKIP_CPU_SYNC); 4568 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4569 dev_err(ctx->dev, "unable to map key\n"); 4570 return -ENOMEM; 4571 } 4572 } 4573 4574 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4575 DMA_BIDIRECTIONAL, 4576 DMA_ATTR_SKIP_CPU_SYNC); 4577 if (dma_mapping_error(ctx->dev, dma_addr)) { 4578 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4579 if (ctx->adata.key_dma) 4580 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4581 ARRAY_SIZE(ctx->key), 4582 DMA_TO_DEVICE, 4583 DMA_ATTR_SKIP_CPU_SYNC); 4584 return -ENOMEM; 4585 } 4586 4587 for (i = 0; i < HASH_NUM_OP; i++) 4588 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4589 4590 /* copy descriptor header template value */ 4591 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4592 4593 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4594 OP_ALG_ALGSEL_SUBMASK) >> 4595 OP_ALG_ALGSEL_SHIFT]; 4596 4597 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 4598 sizeof(struct caam_hash_state)); 4599 4600 /* 4601 * For keyed hash algorithms shared descriptors 4602 * will be created later in setkey() callback 4603 */ 4604 return alg->setkey ? 0 : ahash_set_sh_desc(ahash); 4605 } 4606 4607 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4608 { 4609 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4610 4611 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4612 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4613 if (ctx->adata.key_dma) 4614 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4615 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4616 DMA_ATTR_SKIP_CPU_SYNC); 4617 } 4618 4619 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4620 struct caam_hash_template *template, bool keyed) 4621 { 4622 struct caam_hash_alg *t_alg; 4623 struct ahash_alg *halg; 4624 struct crypto_alg *alg; 4625 4626 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4627 if (!t_alg) 4628 return ERR_PTR(-ENOMEM); 4629 4630 t_alg->ahash_alg = template->template_ahash; 4631 halg = &t_alg->ahash_alg; 4632 alg = &halg->halg.base; 4633 4634 if (keyed) { 4635 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4636 template->hmac_name); 4637 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4638 template->hmac_driver_name); 4639 } else { 4640 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4641 template->name); 4642 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4643 template->driver_name); 4644 t_alg->ahash_alg.setkey = NULL; 4645 } 4646 alg->cra_module = THIS_MODULE; 4647 alg->cra_init = caam_hash_cra_init; 4648 alg->cra_exit = caam_hash_cra_exit; 4649 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 4650 alg->cra_priority = CAAM_CRA_PRIORITY; 4651 alg->cra_blocksize = template->blocksize; 4652 alg->cra_alignmask = 0; 4653 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4654 4655 t_alg->alg_type = template->alg_type; 4656 t_alg->dev = dev; 4657 4658 return t_alg; 4659 } 4660 4661 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4662 { 4663 struct dpaa2_caam_priv_per_cpu *ppriv; 4664 4665 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4666 napi_schedule_irqoff(&ppriv->napi); 4667 } 4668 4669 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4670 { 4671 struct device *dev = priv->dev; 4672 struct dpaa2_io_notification_ctx *nctx; 4673 struct dpaa2_caam_priv_per_cpu *ppriv; 4674 int err, i = 0, cpu; 4675 4676 for_each_online_cpu(cpu) { 4677 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4678 ppriv->priv = priv; 4679 nctx = &ppriv->nctx; 4680 nctx->is_cdan = 0; 4681 nctx->id = ppriv->rsp_fqid; 4682 nctx->desired_cpu = cpu; 4683 nctx->cb = dpaa2_caam_fqdan_cb; 4684 4685 /* Register notification callbacks */ 4686 ppriv->dpio = dpaa2_io_service_select(cpu); 4687 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4688 if (unlikely(err)) { 4689 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4690 nctx->cb = NULL; 4691 /* 4692 * If no affine DPIO for this core, there's probably 4693 * none available for next cores either. Signal we want 4694 * to retry later, in case the DPIO devices weren't 4695 * probed yet. 4696 */ 4697 err = -EPROBE_DEFER; 4698 goto err; 4699 } 4700 4701 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4702 dev); 4703 if (unlikely(!ppriv->store)) { 4704 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4705 err = -ENOMEM; 4706 goto err; 4707 } 4708 4709 if (++i == priv->num_pairs) 4710 break; 4711 } 4712 4713 return 0; 4714 4715 err: 4716 for_each_online_cpu(cpu) { 4717 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4718 if (!ppriv->nctx.cb) 4719 break; 4720 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4721 } 4722 4723 for_each_online_cpu(cpu) { 4724 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4725 if (!ppriv->store) 4726 break; 4727 dpaa2_io_store_destroy(ppriv->store); 4728 } 4729 4730 return err; 4731 } 4732 4733 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4734 { 4735 struct dpaa2_caam_priv_per_cpu *ppriv; 4736 int i = 0, cpu; 4737 4738 for_each_online_cpu(cpu) { 4739 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4740 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4741 priv->dev); 4742 dpaa2_io_store_destroy(ppriv->store); 4743 4744 if (++i == priv->num_pairs) 4745 return; 4746 } 4747 } 4748 4749 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4750 { 4751 struct dpseci_rx_queue_cfg rx_queue_cfg; 4752 struct device *dev = priv->dev; 4753 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4754 struct dpaa2_caam_priv_per_cpu *ppriv; 4755 int err = 0, i = 0, cpu; 4756 4757 /* Configure Rx queues */ 4758 for_each_online_cpu(cpu) { 4759 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4760 4761 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4762 DPSECI_QUEUE_OPT_USER_CTX; 4763 rx_queue_cfg.order_preservation_en = 0; 4764 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4765 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4766 /* 4767 * Rx priority (WQ) doesn't really matter, since we use 4768 * pull mode, i.e. volatile dequeues from specific FQs 4769 */ 4770 rx_queue_cfg.dest_cfg.priority = 0; 4771 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4772 4773 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4774 &rx_queue_cfg); 4775 if (err) { 4776 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4777 err); 4778 return err; 4779 } 4780 4781 if (++i == priv->num_pairs) 4782 break; 4783 } 4784 4785 return err; 4786 } 4787 4788 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4789 { 4790 struct device *dev = priv->dev; 4791 4792 if (!priv->cscn_mem) 4793 return; 4794 4795 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4796 kfree(priv->cscn_mem); 4797 } 4798 4799 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4800 { 4801 struct device *dev = priv->dev; 4802 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4803 int err; 4804 4805 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 4806 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 4807 if (err) 4808 dev_err(dev, "dpseci_reset() failed\n"); 4809 } 4810 4811 dpaa2_dpseci_congestion_free(priv); 4812 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4813 } 4814 4815 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4816 const struct dpaa2_fd *fd) 4817 { 4818 struct caam_request *req; 4819 u32 fd_err; 4820 4821 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4822 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4823 return; 4824 } 4825 4826 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4827 if (unlikely(fd_err)) 4828 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4829 4830 /* 4831 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4832 * in FD[ERR] or FD[FRC]. 4833 */ 4834 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4835 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4836 DMA_BIDIRECTIONAL); 4837 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4838 } 4839 4840 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4841 { 4842 int err; 4843 4844 /* Retry while portal is busy */ 4845 do { 4846 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4847 ppriv->store); 4848 } while (err == -EBUSY); 4849 4850 if (unlikely(err)) 4851 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4852 4853 return err; 4854 } 4855 4856 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4857 { 4858 struct dpaa2_dq *dq; 4859 int cleaned = 0, is_last; 4860 4861 do { 4862 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4863 if (unlikely(!dq)) { 4864 if (unlikely(!is_last)) { 4865 dev_dbg(ppriv->priv->dev, 4866 "FQ %d returned no valid frames\n", 4867 ppriv->rsp_fqid); 4868 /* 4869 * MUST retry until we get some sort of 4870 * valid response token (be it "empty dequeue" 4871 * or a valid frame). 4872 */ 4873 continue; 4874 } 4875 break; 4876 } 4877 4878 /* Process FD */ 4879 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4880 cleaned++; 4881 } while (!is_last); 4882 4883 return cleaned; 4884 } 4885 4886 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4887 { 4888 struct dpaa2_caam_priv_per_cpu *ppriv; 4889 struct dpaa2_caam_priv *priv; 4890 int err, cleaned = 0, store_cleaned; 4891 4892 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4893 priv = ppriv->priv; 4894 4895 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4896 return 0; 4897 4898 do { 4899 store_cleaned = dpaa2_caam_store_consume(ppriv); 4900 cleaned += store_cleaned; 4901 4902 if (store_cleaned == 0 || 4903 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4904 break; 4905 4906 /* Try to dequeue some more */ 4907 err = dpaa2_caam_pull_fq(ppriv); 4908 if (unlikely(err)) 4909 break; 4910 } while (1); 4911 4912 if (cleaned < budget) { 4913 napi_complete_done(napi, cleaned); 4914 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4915 if (unlikely(err)) 4916 dev_err(priv->dev, "Notification rearm failed: %d\n", 4917 err); 4918 } 4919 4920 return cleaned; 4921 } 4922 4923 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4924 u16 token) 4925 { 4926 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4927 struct device *dev = priv->dev; 4928 int err; 4929 4930 /* 4931 * Congestion group feature supported starting with DPSECI API v5.1 4932 * and only when object has been created with this capability. 4933 */ 4934 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4935 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4936 return 0; 4937 4938 priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, 4939 GFP_KERNEL | GFP_DMA); 4940 if (!priv->cscn_mem) 4941 return -ENOMEM; 4942 4943 priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); 4944 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, 4945 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4946 if (dma_mapping_error(dev, priv->cscn_dma)) { 4947 dev_err(dev, "Error mapping CSCN memory area\n"); 4948 err = -ENOMEM; 4949 goto err_dma_map; 4950 } 4951 4952 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4953 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4954 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4955 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4956 cong_notif_cfg.message_iova = priv->cscn_dma; 4957 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4958 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4959 DPSECI_CGN_MODE_COHERENT_WRITE; 4960 4961 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4962 &cong_notif_cfg); 4963 if (err) { 4964 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4965 goto err_set_cong; 4966 } 4967 4968 return 0; 4969 4970 err_set_cong: 4971 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4972 err_dma_map: 4973 kfree(priv->cscn_mem); 4974 4975 return err; 4976 } 4977 4978 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4979 { 4980 struct device *dev = &ls_dev->dev; 4981 struct dpaa2_caam_priv *priv; 4982 struct dpaa2_caam_priv_per_cpu *ppriv; 4983 int err, cpu; 4984 u8 i; 4985 4986 priv = dev_get_drvdata(dev); 4987 4988 priv->dev = dev; 4989 priv->dpsec_id = ls_dev->obj_desc.id; 4990 4991 /* Get a handle for the DPSECI this interface is associate with */ 4992 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 4993 if (err) { 4994 dev_err(dev, "dpseci_open() failed: %d\n", err); 4995 goto err_open; 4996 } 4997 4998 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 4999 &priv->minor_ver); 5000 if (err) { 5001 dev_err(dev, "dpseci_get_api_version() failed\n"); 5002 goto err_get_vers; 5003 } 5004 5005 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 5006 5007 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 5008 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 5009 if (err) { 5010 dev_err(dev, "dpseci_reset() failed\n"); 5011 goto err_get_vers; 5012 } 5013 } 5014 5015 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 5016 &priv->dpseci_attr); 5017 if (err) { 5018 dev_err(dev, "dpseci_get_attributes() failed\n"); 5019 goto err_get_vers; 5020 } 5021 5022 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 5023 &priv->sec_attr); 5024 if (err) { 5025 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 5026 goto err_get_vers; 5027 } 5028 5029 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 5030 if (err) { 5031 dev_err(dev, "setup_congestion() failed\n"); 5032 goto err_get_vers; 5033 } 5034 5035 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 5036 priv->dpseci_attr.num_tx_queues); 5037 if (priv->num_pairs > num_online_cpus()) { 5038 dev_warn(dev, "%d queues won't be used\n", 5039 priv->num_pairs - num_online_cpus()); 5040 priv->num_pairs = num_online_cpus(); 5041 } 5042 5043 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 5044 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5045 &priv->rx_queue_attr[i]); 5046 if (err) { 5047 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 5048 goto err_get_rx_queue; 5049 } 5050 } 5051 5052 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 5053 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5054 &priv->tx_queue_attr[i]); 5055 if (err) { 5056 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 5057 goto err_get_rx_queue; 5058 } 5059 } 5060 5061 i = 0; 5062 for_each_online_cpu(cpu) { 5063 u8 j; 5064 5065 j = i % priv->num_pairs; 5066 5067 ppriv = per_cpu_ptr(priv->ppriv, cpu); 5068 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 5069 5070 /* 5071 * Allow all cores to enqueue, while only some of them 5072 * will take part in dequeuing. 5073 */ 5074 if (++i > priv->num_pairs) 5075 continue; 5076 5077 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 5078 ppriv->prio = j; 5079 5080 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 5081 priv->rx_queue_attr[j].fqid, 5082 priv->tx_queue_attr[j].fqid); 5083 5084 ppriv->net_dev.dev = *dev; 5085 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 5086 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, 5087 DPAA2_CAAM_NAPI_WEIGHT); 5088 } 5089 5090 return 0; 5091 5092 err_get_rx_queue: 5093 dpaa2_dpseci_congestion_free(priv); 5094 err_get_vers: 5095 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5096 err_open: 5097 return err; 5098 } 5099 5100 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 5101 { 5102 struct device *dev = priv->dev; 5103 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5104 struct dpaa2_caam_priv_per_cpu *ppriv; 5105 int i; 5106 5107 for (i = 0; i < priv->num_pairs; i++) { 5108 ppriv = per_cpu_ptr(priv->ppriv, i); 5109 napi_enable(&ppriv->napi); 5110 } 5111 5112 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 5113 } 5114 5115 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 5116 { 5117 struct device *dev = priv->dev; 5118 struct dpaa2_caam_priv_per_cpu *ppriv; 5119 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5120 int i, err = 0, enabled; 5121 5122 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5123 if (err) { 5124 dev_err(dev, "dpseci_disable() failed\n"); 5125 return err; 5126 } 5127 5128 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5129 if (err) { 5130 dev_err(dev, "dpseci_is_enabled() failed\n"); 5131 return err; 5132 } 5133 5134 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 5135 5136 for (i = 0; i < priv->num_pairs; i++) { 5137 ppriv = per_cpu_ptr(priv->ppriv, i); 5138 napi_disable(&ppriv->napi); 5139 netif_napi_del(&ppriv->napi); 5140 } 5141 5142 return 0; 5143 } 5144 5145 static struct list_head hash_list; 5146 5147 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5148 { 5149 struct device *dev; 5150 struct dpaa2_caam_priv *priv; 5151 int i, err = 0; 5152 bool registered = false; 5153 5154 /* 5155 * There is no way to get CAAM endianness - there is no direct register 5156 * space access and MC f/w does not provide this attribute. 5157 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5158 * property. 5159 */ 5160 caam_little_end = true; 5161 5162 caam_imx = false; 5163 5164 dev = &dpseci_dev->dev; 5165 5166 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5167 if (!priv) 5168 return -ENOMEM; 5169 5170 dev_set_drvdata(dev, priv); 5171 5172 priv->domain = iommu_get_domain_for_dev(dev); 5173 5174 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5175 0, SLAB_CACHE_DMA, NULL); 5176 if (!qi_cache) { 5177 dev_err(dev, "Can't allocate SEC cache\n"); 5178 return -ENOMEM; 5179 } 5180 5181 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5182 if (err) { 5183 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5184 goto err_dma_mask; 5185 } 5186 5187 /* Obtain a MC portal */ 5188 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5189 if (err) { 5190 if (err == -ENXIO) 5191 err = -EPROBE_DEFER; 5192 else 5193 dev_err(dev, "MC portal allocation failed\n"); 5194 5195 goto err_dma_mask; 5196 } 5197 5198 priv->ppriv = alloc_percpu(*priv->ppriv); 5199 if (!priv->ppriv) { 5200 dev_err(dev, "alloc_percpu() failed\n"); 5201 err = -ENOMEM; 5202 goto err_alloc_ppriv; 5203 } 5204 5205 /* DPSECI initialization */ 5206 err = dpaa2_dpseci_setup(dpseci_dev); 5207 if (err) { 5208 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5209 goto err_dpseci_setup; 5210 } 5211 5212 /* DPIO */ 5213 err = dpaa2_dpseci_dpio_setup(priv); 5214 if (err) { 5215 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); 5216 goto err_dpio_setup; 5217 } 5218 5219 /* DPSECI binding to DPIO */ 5220 err = dpaa2_dpseci_bind(priv); 5221 if (err) { 5222 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5223 goto err_bind; 5224 } 5225 5226 /* DPSECI enable */ 5227 err = dpaa2_dpseci_enable(priv); 5228 if (err) { 5229 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5230 goto err_bind; 5231 } 5232 5233 dpaa2_dpseci_debugfs_init(priv); 5234 5235 /* register crypto algorithms the device supports */ 5236 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5237 struct caam_skcipher_alg *t_alg = driver_algs + i; 5238 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5239 5240 /* Skip DES algorithms if not supported by device */ 5241 if (!priv->sec_attr.des_acc_num && 5242 (alg_sel == OP_ALG_ALGSEL_3DES || 5243 alg_sel == OP_ALG_ALGSEL_DES)) 5244 continue; 5245 5246 /* Skip AES algorithms if not supported by device */ 5247 if (!priv->sec_attr.aes_acc_num && 5248 alg_sel == OP_ALG_ALGSEL_AES) 5249 continue; 5250 5251 /* Skip CHACHA20 algorithms if not supported by device */ 5252 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5253 !priv->sec_attr.ccha_acc_num) 5254 continue; 5255 5256 t_alg->caam.dev = dev; 5257 caam_skcipher_alg_init(t_alg); 5258 5259 err = crypto_register_skcipher(&t_alg->skcipher); 5260 if (err) { 5261 dev_warn(dev, "%s alg registration failed: %d\n", 5262 t_alg->skcipher.base.cra_driver_name, err); 5263 continue; 5264 } 5265 5266 t_alg->registered = true; 5267 registered = true; 5268 } 5269 5270 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5271 struct caam_aead_alg *t_alg = driver_aeads + i; 5272 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5273 OP_ALG_ALGSEL_MASK; 5274 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5275 OP_ALG_ALGSEL_MASK; 5276 5277 /* Skip DES algorithms if not supported by device */ 5278 if (!priv->sec_attr.des_acc_num && 5279 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5280 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5281 continue; 5282 5283 /* Skip AES algorithms if not supported by device */ 5284 if (!priv->sec_attr.aes_acc_num && 5285 c1_alg_sel == OP_ALG_ALGSEL_AES) 5286 continue; 5287 5288 /* Skip CHACHA20 algorithms if not supported by device */ 5289 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5290 !priv->sec_attr.ccha_acc_num) 5291 continue; 5292 5293 /* Skip POLY1305 algorithms if not supported by device */ 5294 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5295 !priv->sec_attr.ptha_acc_num) 5296 continue; 5297 5298 /* 5299 * Skip algorithms requiring message digests 5300 * if MD not supported by device. 5301 */ 5302 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5303 !priv->sec_attr.md_acc_num) 5304 continue; 5305 5306 t_alg->caam.dev = dev; 5307 caam_aead_alg_init(t_alg); 5308 5309 err = crypto_register_aead(&t_alg->aead); 5310 if (err) { 5311 dev_warn(dev, "%s alg registration failed: %d\n", 5312 t_alg->aead.base.cra_driver_name, err); 5313 continue; 5314 } 5315 5316 t_alg->registered = true; 5317 registered = true; 5318 } 5319 if (registered) 5320 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5321 5322 /* register hash algorithms the device supports */ 5323 INIT_LIST_HEAD(&hash_list); 5324 5325 /* 5326 * Skip registration of any hashing algorithms if MD block 5327 * is not present. 5328 */ 5329 if (!priv->sec_attr.md_acc_num) 5330 return 0; 5331 5332 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5333 struct caam_hash_alg *t_alg; 5334 struct caam_hash_template *alg = driver_hash + i; 5335 5336 /* register hmac version */ 5337 t_alg = caam_hash_alloc(dev, alg, true); 5338 if (IS_ERR(t_alg)) { 5339 err = PTR_ERR(t_alg); 5340 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5341 alg->hmac_driver_name, err); 5342 continue; 5343 } 5344 5345 err = crypto_register_ahash(&t_alg->ahash_alg); 5346 if (err) { 5347 dev_warn(dev, "%s alg registration failed: %d\n", 5348 t_alg->ahash_alg.halg.base.cra_driver_name, 5349 err); 5350 kfree(t_alg); 5351 } else { 5352 list_add_tail(&t_alg->entry, &hash_list); 5353 } 5354 5355 /* register unkeyed version */ 5356 t_alg = caam_hash_alloc(dev, alg, false); 5357 if (IS_ERR(t_alg)) { 5358 err = PTR_ERR(t_alg); 5359 dev_warn(dev, "%s alg allocation failed: %d\n", 5360 alg->driver_name, err); 5361 continue; 5362 } 5363 5364 err = crypto_register_ahash(&t_alg->ahash_alg); 5365 if (err) { 5366 dev_warn(dev, "%s alg registration failed: %d\n", 5367 t_alg->ahash_alg.halg.base.cra_driver_name, 5368 err); 5369 kfree(t_alg); 5370 } else { 5371 list_add_tail(&t_alg->entry, &hash_list); 5372 } 5373 } 5374 if (!list_empty(&hash_list)) 5375 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5376 5377 return err; 5378 5379 err_bind: 5380 dpaa2_dpseci_dpio_free(priv); 5381 err_dpio_setup: 5382 dpaa2_dpseci_free(priv); 5383 err_dpseci_setup: 5384 free_percpu(priv->ppriv); 5385 err_alloc_ppriv: 5386 fsl_mc_portal_free(priv->mc_io); 5387 err_dma_mask: 5388 kmem_cache_destroy(qi_cache); 5389 5390 return err; 5391 } 5392 5393 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5394 { 5395 struct device *dev; 5396 struct dpaa2_caam_priv *priv; 5397 int i; 5398 5399 dev = &ls_dev->dev; 5400 priv = dev_get_drvdata(dev); 5401 5402 dpaa2_dpseci_debugfs_exit(priv); 5403 5404 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5405 struct caam_aead_alg *t_alg = driver_aeads + i; 5406 5407 if (t_alg->registered) 5408 crypto_unregister_aead(&t_alg->aead); 5409 } 5410 5411 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5412 struct caam_skcipher_alg *t_alg = driver_algs + i; 5413 5414 if (t_alg->registered) 5415 crypto_unregister_skcipher(&t_alg->skcipher); 5416 } 5417 5418 if (hash_list.next) { 5419 struct caam_hash_alg *t_hash_alg, *p; 5420 5421 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5422 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5423 list_del(&t_hash_alg->entry); 5424 kfree(t_hash_alg); 5425 } 5426 } 5427 5428 dpaa2_dpseci_disable(priv); 5429 dpaa2_dpseci_dpio_free(priv); 5430 dpaa2_dpseci_free(priv); 5431 free_percpu(priv->ppriv); 5432 fsl_mc_portal_free(priv->mc_io); 5433 kmem_cache_destroy(qi_cache); 5434 5435 return 0; 5436 } 5437 5438 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5439 { 5440 struct dpaa2_fd fd; 5441 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5442 struct dpaa2_caam_priv_per_cpu *ppriv; 5443 int err = 0, i; 5444 5445 if (IS_ERR(req)) 5446 return PTR_ERR(req); 5447 5448 if (priv->cscn_mem) { 5449 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5450 DPAA2_CSCN_SIZE, 5451 DMA_FROM_DEVICE); 5452 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { 5453 dev_dbg_ratelimited(dev, "Dropping request\n"); 5454 return -EBUSY; 5455 } 5456 } 5457 5458 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5459 5460 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5461 DMA_BIDIRECTIONAL); 5462 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5463 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5464 goto err_out; 5465 } 5466 5467 memset(&fd, 0, sizeof(fd)); 5468 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5469 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5470 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5471 dpaa2_fd_set_flc(&fd, req->flc_dma); 5472 5473 ppriv = raw_cpu_ptr(priv->ppriv); 5474 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5475 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5476 &fd); 5477 if (err != -EBUSY) 5478 break; 5479 5480 cpu_relax(); 5481 } 5482 5483 if (unlikely(err)) { 5484 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5485 goto err_out; 5486 } 5487 5488 return -EINPROGRESS; 5489 5490 err_out: 5491 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5492 DMA_BIDIRECTIONAL); 5493 return -EIO; 5494 } 5495 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5496 5497 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5498 { 5499 .vendor = FSL_MC_VENDOR_FREESCALE, 5500 .obj_type = "dpseci", 5501 }, 5502 { .vendor = 0x0 } 5503 }; 5504 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table); 5505 5506 static struct fsl_mc_driver dpaa2_caam_driver = { 5507 .driver = { 5508 .name = KBUILD_MODNAME, 5509 .owner = THIS_MODULE, 5510 }, 5511 .probe = dpaa2_caam_probe, 5512 .remove = dpaa2_caam_remove, 5513 .match_id_table = dpaa2_caam_match_id_table 5514 }; 5515 5516 MODULE_LICENSE("Dual BSD/GPL"); 5517 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5518 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5519 5520 module_fsl_mc_driver(dpaa2_caam_driver); 5521