1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include "dpseci-debugfs.h" 19 #include <linux/dma-mapping.h> 20 #include <linux/fsl/mc.h> 21 #include <linux/kernel.h> 22 #include <linux/string.h> 23 #include <linux/string_choices.h> 24 #include <soc/fsl/dpaa2-io.h> 25 #include <soc/fsl/dpaa2-fd.h> 26 #include <crypto/xts.h> 27 #include <linux/unaligned.h> 28 29 #define CAAM_CRA_PRIORITY 2000 30 31 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 32 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 33 SHA512_DIGEST_SIZE * 2) 34 35 /* 36 * This is a cache of buffers, from which the users of CAAM QI driver 37 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 38 * NOTE: A more elegant solution would be to have some headroom in the frames 39 * being processed. This can be added by the dpaa2-eth driver. This would 40 * pose a problem for userspace application processing which cannot 41 * know of this limitation. So for now, this will work. 42 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 43 */ 44 static struct kmem_cache *qi_cache; 45 46 struct caam_alg_entry { 47 struct device *dev; 48 int class1_alg_type; 49 int class2_alg_type; 50 bool rfc3686; 51 bool geniv; 52 bool nodkp; 53 }; 54 55 struct caam_aead_alg { 56 struct aead_alg aead; 57 struct caam_alg_entry caam; 58 bool registered; 59 }; 60 61 struct caam_skcipher_alg { 62 struct skcipher_alg skcipher; 63 struct caam_alg_entry caam; 64 bool registered; 65 }; 66 67 /** 68 * struct caam_ctx - per-session context 69 * @flc: Flow Contexts array 70 * @key: [authentication key], encryption key 71 * @flc_dma: I/O virtual addresses of the Flow Contexts 72 * @key_dma: I/O virtual address of the key 73 * @dir: DMA direction for mapping key and Flow Contexts 74 * @dev: dpseci device 75 * @adata: authentication algorithm details 76 * @cdata: encryption algorithm details 77 * @authsize: authentication tag (a.k.a. ICV / MAC) size 78 * @xts_key_fallback: true if fallback tfm needs to be used due 79 * to unsupported xts key lengths 80 * @fallback: xts fallback tfm 81 */ 82 struct caam_ctx { 83 struct caam_flc flc[NUM_OP]; 84 u8 key[CAAM_MAX_KEY_SIZE]; 85 dma_addr_t flc_dma[NUM_OP]; 86 dma_addr_t key_dma; 87 enum dma_data_direction dir; 88 struct device *dev; 89 struct alginfo adata; 90 struct alginfo cdata; 91 unsigned int authsize; 92 bool xts_key_fallback; 93 struct crypto_skcipher *fallback; 94 }; 95 96 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 97 dma_addr_t iova_addr) 98 { 99 phys_addr_t phys_addr; 100 101 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 102 iova_addr; 103 104 return phys_to_virt(phys_addr); 105 } 106 107 /* 108 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 109 * 110 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 111 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 112 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 113 * hosting 16 SG entries. 114 * 115 * @flags - flags that would be used for the equivalent kmalloc(..) call 116 * 117 * Returns a pointer to a retrieved buffer on success or NULL on failure. 118 */ 119 static inline void *qi_cache_zalloc(gfp_t flags) 120 { 121 return kmem_cache_zalloc(qi_cache, flags); 122 } 123 124 /* 125 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 126 * 127 * @obj - buffer previously allocated by qi_cache_zalloc 128 * 129 * No checking is being done, the call is a passthrough call to 130 * kmem_cache_free(...) 131 */ 132 static inline void qi_cache_free(void *obj) 133 { 134 kmem_cache_free(qi_cache, obj); 135 } 136 137 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 138 { 139 switch (crypto_tfm_alg_type(areq->tfm)) { 140 case CRYPTO_ALG_TYPE_SKCIPHER: 141 return skcipher_request_ctx_dma(skcipher_request_cast(areq)); 142 case CRYPTO_ALG_TYPE_AEAD: 143 return aead_request_ctx_dma( 144 container_of(areq, struct aead_request, base)); 145 case CRYPTO_ALG_TYPE_AHASH: 146 return ahash_request_ctx_dma(ahash_request_cast(areq)); 147 default: 148 return ERR_PTR(-EINVAL); 149 } 150 } 151 152 static void caam_unmap(struct device *dev, struct scatterlist *src, 153 struct scatterlist *dst, int src_nents, 154 int dst_nents, dma_addr_t iv_dma, int ivsize, 155 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 156 int qm_sg_bytes) 157 { 158 if (dst != src) { 159 if (src_nents) 160 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 161 if (dst_nents) 162 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 163 } else { 164 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 165 } 166 167 if (iv_dma) 168 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 169 170 if (qm_sg_bytes) 171 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 172 } 173 174 static int aead_set_sh_desc(struct crypto_aead *aead) 175 { 176 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 177 typeof(*alg), aead); 178 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 179 unsigned int ivsize = crypto_aead_ivsize(aead); 180 struct device *dev = ctx->dev; 181 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 182 struct caam_flc *flc; 183 u32 *desc; 184 u32 ctx1_iv_off = 0; 185 u32 *nonce = NULL; 186 unsigned int data_len[2]; 187 u32 inl_mask; 188 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 189 OP_ALG_AAI_CTR_MOD128); 190 const bool is_rfc3686 = alg->caam.rfc3686; 191 192 if (!ctx->cdata.keylen || !ctx->authsize) 193 return 0; 194 195 /* 196 * AES-CTR needs to load IV in CONTEXT1 reg 197 * at an offset of 128bits (16bytes) 198 * CONTEXT1[255:128] = IV 199 */ 200 if (ctr_mode) 201 ctx1_iv_off = 16; 202 203 /* 204 * RFC3686 specific: 205 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 206 */ 207 if (is_rfc3686) { 208 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 209 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 210 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 211 } 212 213 /* 214 * In case |user key| > |derived key|, using DKP<imm,imm> would result 215 * in invalid opcodes (last bytes of user key) in the resulting 216 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 217 * addresses are needed. 218 */ 219 ctx->adata.key_virt = ctx->key; 220 ctx->adata.key_dma = ctx->key_dma; 221 222 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 223 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 224 225 data_len[0] = ctx->adata.keylen_pad; 226 data_len[1] = ctx->cdata.keylen; 227 228 /* aead_encrypt shared descriptor */ 229 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 230 DESC_QI_AEAD_ENC_LEN) + 231 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 232 DESC_JOB_IO_LEN, data_len, &inl_mask, 233 ARRAY_SIZE(data_len)) < 0) 234 return -EINVAL; 235 236 ctx->adata.key_inline = !!(inl_mask & 1); 237 ctx->cdata.key_inline = !!(inl_mask & 2); 238 239 flc = &ctx->flc[ENCRYPT]; 240 desc = flc->sh_desc; 241 242 if (alg->caam.geniv) 243 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 244 ivsize, ctx->authsize, is_rfc3686, 245 nonce, ctx1_iv_off, true, 246 priv->sec_attr.era); 247 else 248 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 249 ivsize, ctx->authsize, is_rfc3686, nonce, 250 ctx1_iv_off, true, priv->sec_attr.era); 251 252 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 253 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 254 sizeof(flc->flc) + desc_bytes(desc), 255 ctx->dir); 256 257 /* aead_decrypt shared descriptor */ 258 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 259 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 260 DESC_JOB_IO_LEN, data_len, &inl_mask, 261 ARRAY_SIZE(data_len)) < 0) 262 return -EINVAL; 263 264 ctx->adata.key_inline = !!(inl_mask & 1); 265 ctx->cdata.key_inline = !!(inl_mask & 2); 266 267 flc = &ctx->flc[DECRYPT]; 268 desc = flc->sh_desc; 269 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 270 ivsize, ctx->authsize, alg->caam.geniv, 271 is_rfc3686, nonce, ctx1_iv_off, true, 272 priv->sec_attr.era); 273 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 274 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 275 sizeof(flc->flc) + desc_bytes(desc), 276 ctx->dir); 277 278 return 0; 279 } 280 281 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 282 { 283 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 284 285 ctx->authsize = authsize; 286 aead_set_sh_desc(authenc); 287 288 return 0; 289 } 290 291 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 292 unsigned int keylen) 293 { 294 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 295 struct device *dev = ctx->dev; 296 struct crypto_authenc_keys keys; 297 298 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 299 goto badkey; 300 301 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 302 keys.authkeylen + keys.enckeylen, keys.enckeylen, 303 keys.authkeylen); 304 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 305 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 306 307 ctx->adata.keylen = keys.authkeylen; 308 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 309 OP_ALG_ALGSEL_MASK); 310 311 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 312 goto badkey; 313 314 memcpy(ctx->key, keys.authkey, keys.authkeylen); 315 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 316 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 317 keys.enckeylen, ctx->dir); 318 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 319 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 320 ctx->adata.keylen_pad + keys.enckeylen, 1); 321 322 ctx->cdata.keylen = keys.enckeylen; 323 324 memzero_explicit(&keys, sizeof(keys)); 325 return aead_set_sh_desc(aead); 326 badkey: 327 memzero_explicit(&keys, sizeof(keys)); 328 return -EINVAL; 329 } 330 331 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 332 unsigned int keylen) 333 { 334 struct crypto_authenc_keys keys; 335 int err; 336 337 err = crypto_authenc_extractkeys(&keys, key, keylen); 338 if (unlikely(err)) 339 goto out; 340 341 err = -EINVAL; 342 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 343 goto out; 344 345 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 346 aead_setkey(aead, key, keylen); 347 348 out: 349 memzero_explicit(&keys, sizeof(keys)); 350 return err; 351 } 352 353 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 354 bool encrypt) 355 { 356 struct crypto_aead *aead = crypto_aead_reqtfm(req); 357 struct caam_request *req_ctx = aead_request_ctx_dma(req); 358 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 359 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 360 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 361 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 362 typeof(*alg), aead); 363 struct device *dev = ctx->dev; 364 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 365 GFP_KERNEL : GFP_ATOMIC; 366 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 367 int src_len, dst_len = 0; 368 struct aead_edesc *edesc; 369 dma_addr_t qm_sg_dma, iv_dma = 0; 370 int ivsize = 0; 371 unsigned int authsize = ctx->authsize; 372 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 373 int in_len, out_len; 374 struct dpaa2_sg_entry *sg_table; 375 376 /* allocate space for base edesc, link tables and IV */ 377 edesc = qi_cache_zalloc(flags); 378 if (unlikely(!edesc)) { 379 dev_err(dev, "could not allocate extended descriptor\n"); 380 return ERR_PTR(-ENOMEM); 381 } 382 383 if (unlikely(req->dst != req->src)) { 384 src_len = req->assoclen + req->cryptlen; 385 dst_len = src_len + (encrypt ? authsize : (-authsize)); 386 387 src_nents = sg_nents_for_len(req->src, src_len); 388 if (unlikely(src_nents < 0)) { 389 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 390 src_len); 391 qi_cache_free(edesc); 392 return ERR_PTR(src_nents); 393 } 394 395 dst_nents = sg_nents_for_len(req->dst, dst_len); 396 if (unlikely(dst_nents < 0)) { 397 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 398 dst_len); 399 qi_cache_free(edesc); 400 return ERR_PTR(dst_nents); 401 } 402 403 if (src_nents) { 404 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 405 DMA_TO_DEVICE); 406 if (unlikely(!mapped_src_nents)) { 407 dev_err(dev, "unable to map source\n"); 408 qi_cache_free(edesc); 409 return ERR_PTR(-ENOMEM); 410 } 411 } else { 412 mapped_src_nents = 0; 413 } 414 415 if (dst_nents) { 416 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 417 DMA_FROM_DEVICE); 418 if (unlikely(!mapped_dst_nents)) { 419 dev_err(dev, "unable to map destination\n"); 420 dma_unmap_sg(dev, req->src, src_nents, 421 DMA_TO_DEVICE); 422 qi_cache_free(edesc); 423 return ERR_PTR(-ENOMEM); 424 } 425 } else { 426 mapped_dst_nents = 0; 427 } 428 } else { 429 src_len = req->assoclen + req->cryptlen + 430 (encrypt ? authsize : 0); 431 432 src_nents = sg_nents_for_len(req->src, src_len); 433 if (unlikely(src_nents < 0)) { 434 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 435 src_len); 436 qi_cache_free(edesc); 437 return ERR_PTR(src_nents); 438 } 439 440 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 441 DMA_BIDIRECTIONAL); 442 if (unlikely(!mapped_src_nents)) { 443 dev_err(dev, "unable to map source\n"); 444 qi_cache_free(edesc); 445 return ERR_PTR(-ENOMEM); 446 } 447 } 448 449 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 450 ivsize = crypto_aead_ivsize(aead); 451 452 /* 453 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 454 * Input is not contiguous. 455 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 456 * the end of the table by allocating more S/G entries. Logic: 457 * if (src != dst && output S/G) 458 * pad output S/G, if needed 459 * else if (src == dst && S/G) 460 * overlapping S/Gs; pad one of them 461 * else if (input S/G) ... 462 * pad input S/G, if needed 463 */ 464 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 465 if (mapped_dst_nents > 1) 466 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 467 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 468 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 469 1 + !!ivsize + 470 pad_sg_nents(mapped_src_nents)); 471 else 472 qm_sg_nents = pad_sg_nents(qm_sg_nents); 473 474 sg_table = &edesc->sgt[0]; 475 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 476 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 477 CAAM_QI_MEMCACHE_SIZE)) { 478 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 479 qm_sg_nents, ivsize); 480 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 481 0, DMA_NONE, 0, 0); 482 qi_cache_free(edesc); 483 return ERR_PTR(-ENOMEM); 484 } 485 486 if (ivsize) { 487 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 488 489 /* Make sure IV is located in a DMAable area */ 490 memcpy(iv, req->iv, ivsize); 491 492 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 493 if (dma_mapping_error(dev, iv_dma)) { 494 dev_err(dev, "unable to map IV\n"); 495 caam_unmap(dev, req->src, req->dst, src_nents, 496 dst_nents, 0, 0, DMA_NONE, 0, 0); 497 qi_cache_free(edesc); 498 return ERR_PTR(-ENOMEM); 499 } 500 } 501 502 edesc->src_nents = src_nents; 503 edesc->dst_nents = dst_nents; 504 edesc->iv_dma = iv_dma; 505 506 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 507 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 508 /* 509 * The associated data comes already with the IV but we need 510 * to skip it when we authenticate or encrypt... 511 */ 512 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 513 else 514 edesc->assoclen = cpu_to_caam32(req->assoclen); 515 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 516 DMA_TO_DEVICE); 517 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 518 dev_err(dev, "unable to map assoclen\n"); 519 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 520 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 521 qi_cache_free(edesc); 522 return ERR_PTR(-ENOMEM); 523 } 524 525 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 526 qm_sg_index++; 527 if (ivsize) { 528 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 529 qm_sg_index++; 530 } 531 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 532 qm_sg_index += mapped_src_nents; 533 534 if (mapped_dst_nents > 1) 535 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 536 537 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 538 if (dma_mapping_error(dev, qm_sg_dma)) { 539 dev_err(dev, "unable to map S/G table\n"); 540 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 541 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 542 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 543 qi_cache_free(edesc); 544 return ERR_PTR(-ENOMEM); 545 } 546 547 edesc->qm_sg_dma = qm_sg_dma; 548 edesc->qm_sg_bytes = qm_sg_bytes; 549 550 out_len = req->assoclen + req->cryptlen + 551 (encrypt ? ctx->authsize : (-ctx->authsize)); 552 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 553 554 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 555 dpaa2_fl_set_final(in_fle, true); 556 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 557 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 558 dpaa2_fl_set_len(in_fle, in_len); 559 560 if (req->dst == req->src) { 561 if (mapped_src_nents == 1) { 562 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 563 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 564 } else { 565 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 566 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 567 (1 + !!ivsize) * sizeof(*sg_table)); 568 } 569 } else if (!mapped_dst_nents) { 570 /* 571 * crypto engine requires the output entry to be present when 572 * "frame list" FD is used. 573 * Since engine does not support FMT=2'b11 (unused entry type), 574 * leaving out_fle zeroized is the best option. 575 */ 576 goto skip_out_fle; 577 } else if (mapped_dst_nents == 1) { 578 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 579 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 580 } else { 581 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 582 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 583 sizeof(*sg_table)); 584 } 585 586 dpaa2_fl_set_len(out_fle, out_len); 587 588 skip_out_fle: 589 return edesc; 590 } 591 592 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 593 { 594 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 595 unsigned int ivsize = crypto_aead_ivsize(aead); 596 struct device *dev = ctx->dev; 597 struct caam_flc *flc; 598 u32 *desc; 599 600 if (!ctx->cdata.keylen || !ctx->authsize) 601 return 0; 602 603 flc = &ctx->flc[ENCRYPT]; 604 desc = flc->sh_desc; 605 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 606 ctx->authsize, true, true); 607 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 608 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 609 sizeof(flc->flc) + desc_bytes(desc), 610 ctx->dir); 611 612 flc = &ctx->flc[DECRYPT]; 613 desc = flc->sh_desc; 614 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 615 ctx->authsize, false, true); 616 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 617 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 618 sizeof(flc->flc) + desc_bytes(desc), 619 ctx->dir); 620 621 return 0; 622 } 623 624 static int chachapoly_setauthsize(struct crypto_aead *aead, 625 unsigned int authsize) 626 { 627 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 628 629 if (authsize != POLY1305_DIGEST_SIZE) 630 return -EINVAL; 631 632 ctx->authsize = authsize; 633 return chachapoly_set_sh_desc(aead); 634 } 635 636 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 637 unsigned int keylen) 638 { 639 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 640 unsigned int ivsize = crypto_aead_ivsize(aead); 641 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 642 643 if (keylen != CHACHA_KEY_SIZE + saltlen) 644 return -EINVAL; 645 646 memcpy(ctx->key, key, keylen); 647 ctx->cdata.key_virt = ctx->key; 648 ctx->cdata.keylen = keylen - saltlen; 649 650 return chachapoly_set_sh_desc(aead); 651 } 652 653 static int gcm_set_sh_desc(struct crypto_aead *aead) 654 { 655 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 656 struct device *dev = ctx->dev; 657 unsigned int ivsize = crypto_aead_ivsize(aead); 658 struct caam_flc *flc; 659 u32 *desc; 660 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 661 ctx->cdata.keylen; 662 663 if (!ctx->cdata.keylen || !ctx->authsize) 664 return 0; 665 666 /* 667 * AES GCM encrypt shared descriptor 668 * Job Descriptor and Shared Descriptor 669 * must fit into the 64-word Descriptor h/w Buffer 670 */ 671 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 672 ctx->cdata.key_inline = true; 673 ctx->cdata.key_virt = ctx->key; 674 } else { 675 ctx->cdata.key_inline = false; 676 ctx->cdata.key_dma = ctx->key_dma; 677 } 678 679 flc = &ctx->flc[ENCRYPT]; 680 desc = flc->sh_desc; 681 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 682 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 683 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 684 sizeof(flc->flc) + desc_bytes(desc), 685 ctx->dir); 686 687 /* 688 * Job Descriptor and Shared Descriptors 689 * must all fit into the 64-word Descriptor h/w Buffer 690 */ 691 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 692 ctx->cdata.key_inline = true; 693 ctx->cdata.key_virt = ctx->key; 694 } else { 695 ctx->cdata.key_inline = false; 696 ctx->cdata.key_dma = ctx->key_dma; 697 } 698 699 flc = &ctx->flc[DECRYPT]; 700 desc = flc->sh_desc; 701 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 702 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 703 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 704 sizeof(flc->flc) + desc_bytes(desc), 705 ctx->dir); 706 707 return 0; 708 } 709 710 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 711 { 712 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 713 int err; 714 715 err = crypto_gcm_check_authsize(authsize); 716 if (err) 717 return err; 718 719 ctx->authsize = authsize; 720 gcm_set_sh_desc(authenc); 721 722 return 0; 723 } 724 725 static int gcm_setkey(struct crypto_aead *aead, 726 const u8 *key, unsigned int keylen) 727 { 728 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 729 struct device *dev = ctx->dev; 730 int ret; 731 732 ret = aes_check_keylen(keylen); 733 if (ret) 734 return ret; 735 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 736 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 737 738 memcpy(ctx->key, key, keylen); 739 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 740 ctx->cdata.keylen = keylen; 741 742 return gcm_set_sh_desc(aead); 743 } 744 745 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 746 { 747 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 748 struct device *dev = ctx->dev; 749 unsigned int ivsize = crypto_aead_ivsize(aead); 750 struct caam_flc *flc; 751 u32 *desc; 752 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 753 ctx->cdata.keylen; 754 755 if (!ctx->cdata.keylen || !ctx->authsize) 756 return 0; 757 758 ctx->cdata.key_virt = ctx->key; 759 760 /* 761 * RFC4106 encrypt shared descriptor 762 * Job Descriptor and Shared Descriptor 763 * must fit into the 64-word Descriptor h/w Buffer 764 */ 765 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 766 ctx->cdata.key_inline = true; 767 } else { 768 ctx->cdata.key_inline = false; 769 ctx->cdata.key_dma = ctx->key_dma; 770 } 771 772 flc = &ctx->flc[ENCRYPT]; 773 desc = flc->sh_desc; 774 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 775 true); 776 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 777 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 778 sizeof(flc->flc) + desc_bytes(desc), 779 ctx->dir); 780 781 /* 782 * Job Descriptor and Shared Descriptors 783 * must all fit into the 64-word Descriptor h/w Buffer 784 */ 785 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 786 ctx->cdata.key_inline = true; 787 } else { 788 ctx->cdata.key_inline = false; 789 ctx->cdata.key_dma = ctx->key_dma; 790 } 791 792 flc = &ctx->flc[DECRYPT]; 793 desc = flc->sh_desc; 794 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 795 true); 796 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 797 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 798 sizeof(flc->flc) + desc_bytes(desc), 799 ctx->dir); 800 801 return 0; 802 } 803 804 static int rfc4106_setauthsize(struct crypto_aead *authenc, 805 unsigned int authsize) 806 { 807 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 808 int err; 809 810 err = crypto_rfc4106_check_authsize(authsize); 811 if (err) 812 return err; 813 814 ctx->authsize = authsize; 815 rfc4106_set_sh_desc(authenc); 816 817 return 0; 818 } 819 820 static int rfc4106_setkey(struct crypto_aead *aead, 821 const u8 *key, unsigned int keylen) 822 { 823 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 824 struct device *dev = ctx->dev; 825 int ret; 826 827 ret = aes_check_keylen(keylen - 4); 828 if (ret) 829 return ret; 830 831 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 832 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 833 834 memcpy(ctx->key, key, keylen); 835 /* 836 * The last four bytes of the key material are used as the salt value 837 * in the nonce. Update the AES key length. 838 */ 839 ctx->cdata.keylen = keylen - 4; 840 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 841 ctx->dir); 842 843 return rfc4106_set_sh_desc(aead); 844 } 845 846 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 847 { 848 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 849 struct device *dev = ctx->dev; 850 unsigned int ivsize = crypto_aead_ivsize(aead); 851 struct caam_flc *flc; 852 u32 *desc; 853 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 854 ctx->cdata.keylen; 855 856 if (!ctx->cdata.keylen || !ctx->authsize) 857 return 0; 858 859 ctx->cdata.key_virt = ctx->key; 860 861 /* 862 * RFC4543 encrypt shared descriptor 863 * Job Descriptor and Shared Descriptor 864 * must fit into the 64-word Descriptor h/w Buffer 865 */ 866 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 867 ctx->cdata.key_inline = true; 868 } else { 869 ctx->cdata.key_inline = false; 870 ctx->cdata.key_dma = ctx->key_dma; 871 } 872 873 flc = &ctx->flc[ENCRYPT]; 874 desc = flc->sh_desc; 875 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 876 true); 877 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 878 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 879 sizeof(flc->flc) + desc_bytes(desc), 880 ctx->dir); 881 882 /* 883 * Job Descriptor and Shared Descriptors 884 * must all fit into the 64-word Descriptor h/w Buffer 885 */ 886 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 887 ctx->cdata.key_inline = true; 888 } else { 889 ctx->cdata.key_inline = false; 890 ctx->cdata.key_dma = ctx->key_dma; 891 } 892 893 flc = &ctx->flc[DECRYPT]; 894 desc = flc->sh_desc; 895 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 896 true); 897 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 898 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 899 sizeof(flc->flc) + desc_bytes(desc), 900 ctx->dir); 901 902 return 0; 903 } 904 905 static int rfc4543_setauthsize(struct crypto_aead *authenc, 906 unsigned int authsize) 907 { 908 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 909 910 if (authsize != 16) 911 return -EINVAL; 912 913 ctx->authsize = authsize; 914 rfc4543_set_sh_desc(authenc); 915 916 return 0; 917 } 918 919 static int rfc4543_setkey(struct crypto_aead *aead, 920 const u8 *key, unsigned int keylen) 921 { 922 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 923 struct device *dev = ctx->dev; 924 int ret; 925 926 ret = aes_check_keylen(keylen - 4); 927 if (ret) 928 return ret; 929 930 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 931 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 932 933 memcpy(ctx->key, key, keylen); 934 /* 935 * The last four bytes of the key material are used as the salt value 936 * in the nonce. Update the AES key length. 937 */ 938 ctx->cdata.keylen = keylen - 4; 939 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 940 ctx->dir); 941 942 return rfc4543_set_sh_desc(aead); 943 } 944 945 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 946 unsigned int keylen, const u32 ctx1_iv_off) 947 { 948 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 949 struct caam_skcipher_alg *alg = 950 container_of(crypto_skcipher_alg(skcipher), 951 struct caam_skcipher_alg, skcipher); 952 struct device *dev = ctx->dev; 953 struct caam_flc *flc; 954 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 955 u32 *desc; 956 const bool is_rfc3686 = alg->caam.rfc3686; 957 958 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 959 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 960 961 ctx->cdata.keylen = keylen; 962 ctx->cdata.key_virt = key; 963 ctx->cdata.key_inline = true; 964 965 /* skcipher_encrypt shared descriptor */ 966 flc = &ctx->flc[ENCRYPT]; 967 desc = flc->sh_desc; 968 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 969 ctx1_iv_off); 970 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 971 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 972 sizeof(flc->flc) + desc_bytes(desc), 973 ctx->dir); 974 975 /* skcipher_decrypt shared descriptor */ 976 flc = &ctx->flc[DECRYPT]; 977 desc = flc->sh_desc; 978 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 979 ctx1_iv_off); 980 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 981 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 982 sizeof(flc->flc) + desc_bytes(desc), 983 ctx->dir); 984 985 return 0; 986 } 987 988 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 989 const u8 *key, unsigned int keylen) 990 { 991 int err; 992 993 err = aes_check_keylen(keylen); 994 if (err) 995 return err; 996 997 return skcipher_setkey(skcipher, key, keylen, 0); 998 } 999 1000 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 1001 const u8 *key, unsigned int keylen) 1002 { 1003 u32 ctx1_iv_off; 1004 int err; 1005 1006 /* 1007 * RFC3686 specific: 1008 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1009 * | *key = {KEY, NONCE} 1010 */ 1011 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1012 keylen -= CTR_RFC3686_NONCE_SIZE; 1013 1014 err = aes_check_keylen(keylen); 1015 if (err) 1016 return err; 1017 1018 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1019 } 1020 1021 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1022 const u8 *key, unsigned int keylen) 1023 { 1024 u32 ctx1_iv_off; 1025 int err; 1026 1027 /* 1028 * AES-CTR needs to load IV in CONTEXT1 reg 1029 * at an offset of 128bits (16bytes) 1030 * CONTEXT1[255:128] = IV 1031 */ 1032 ctx1_iv_off = 16; 1033 1034 err = aes_check_keylen(keylen); 1035 if (err) 1036 return err; 1037 1038 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1039 } 1040 1041 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1042 const u8 *key, unsigned int keylen) 1043 { 1044 if (keylen != CHACHA_KEY_SIZE) 1045 return -EINVAL; 1046 1047 return skcipher_setkey(skcipher, key, keylen, 0); 1048 } 1049 1050 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1051 const u8 *key, unsigned int keylen) 1052 { 1053 return verify_skcipher_des_key(skcipher, key) ?: 1054 skcipher_setkey(skcipher, key, keylen, 0); 1055 } 1056 1057 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1058 const u8 *key, unsigned int keylen) 1059 { 1060 return verify_skcipher_des3_key(skcipher, key) ?: 1061 skcipher_setkey(skcipher, key, keylen, 0); 1062 } 1063 1064 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1065 unsigned int keylen) 1066 { 1067 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1068 struct device *dev = ctx->dev; 1069 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 1070 struct caam_flc *flc; 1071 u32 *desc; 1072 int err; 1073 1074 err = xts_verify_key(skcipher, key, keylen); 1075 if (err) { 1076 dev_dbg(dev, "key size mismatch\n"); 1077 return err; 1078 } 1079 1080 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 1081 ctx->xts_key_fallback = true; 1082 1083 if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { 1084 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 1085 if (err) 1086 return err; 1087 } 1088 1089 ctx->cdata.keylen = keylen; 1090 ctx->cdata.key_virt = key; 1091 ctx->cdata.key_inline = true; 1092 1093 /* xts_skcipher_encrypt shared descriptor */ 1094 flc = &ctx->flc[ENCRYPT]; 1095 desc = flc->sh_desc; 1096 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1097 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1098 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1099 sizeof(flc->flc) + desc_bytes(desc), 1100 ctx->dir); 1101 1102 /* xts_skcipher_decrypt shared descriptor */ 1103 flc = &ctx->flc[DECRYPT]; 1104 desc = flc->sh_desc; 1105 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1106 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1107 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1108 sizeof(flc->flc) + desc_bytes(desc), 1109 ctx->dir); 1110 1111 return 0; 1112 } 1113 1114 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1115 { 1116 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1117 struct caam_request *req_ctx = skcipher_request_ctx_dma(req); 1118 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1119 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1120 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1121 struct device *dev = ctx->dev; 1122 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1123 GFP_KERNEL : GFP_ATOMIC; 1124 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1125 struct skcipher_edesc *edesc; 1126 dma_addr_t iv_dma; 1127 u8 *iv; 1128 int ivsize = crypto_skcipher_ivsize(skcipher); 1129 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1130 struct dpaa2_sg_entry *sg_table; 1131 1132 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1133 if (unlikely(src_nents < 0)) { 1134 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1135 req->cryptlen); 1136 return ERR_PTR(src_nents); 1137 } 1138 1139 if (unlikely(req->dst != req->src)) { 1140 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1141 if (unlikely(dst_nents < 0)) { 1142 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1143 req->cryptlen); 1144 return ERR_PTR(dst_nents); 1145 } 1146 1147 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1148 DMA_TO_DEVICE); 1149 if (unlikely(!mapped_src_nents)) { 1150 dev_err(dev, "unable to map source\n"); 1151 return ERR_PTR(-ENOMEM); 1152 } 1153 1154 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1155 DMA_FROM_DEVICE); 1156 if (unlikely(!mapped_dst_nents)) { 1157 dev_err(dev, "unable to map destination\n"); 1158 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1159 return ERR_PTR(-ENOMEM); 1160 } 1161 } else { 1162 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1163 DMA_BIDIRECTIONAL); 1164 if (unlikely(!mapped_src_nents)) { 1165 dev_err(dev, "unable to map source\n"); 1166 return ERR_PTR(-ENOMEM); 1167 } 1168 } 1169 1170 qm_sg_ents = 1 + mapped_src_nents; 1171 dst_sg_idx = qm_sg_ents; 1172 1173 /* 1174 * Input, output HW S/G tables: [IV, src][dst, IV] 1175 * IV entries point to the same buffer 1176 * If src == dst, S/G entries are reused (S/G tables overlap) 1177 * 1178 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1179 * the end of the table by allocating more S/G entries. 1180 */ 1181 if (req->src != req->dst) 1182 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1183 else 1184 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1185 1186 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1187 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1188 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1189 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1190 qm_sg_ents, ivsize); 1191 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1192 0, DMA_NONE, 0, 0); 1193 return ERR_PTR(-ENOMEM); 1194 } 1195 1196 /* allocate space for base edesc, link tables and IV */ 1197 edesc = qi_cache_zalloc(flags); 1198 if (unlikely(!edesc)) { 1199 dev_err(dev, "could not allocate extended descriptor\n"); 1200 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1201 0, DMA_NONE, 0, 0); 1202 return ERR_PTR(-ENOMEM); 1203 } 1204 1205 /* Make sure IV is located in a DMAable area */ 1206 sg_table = &edesc->sgt[0]; 1207 iv = (u8 *)(sg_table + qm_sg_ents); 1208 memcpy(iv, req->iv, ivsize); 1209 1210 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1211 if (dma_mapping_error(dev, iv_dma)) { 1212 dev_err(dev, "unable to map IV\n"); 1213 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1214 0, DMA_NONE, 0, 0); 1215 qi_cache_free(edesc); 1216 return ERR_PTR(-ENOMEM); 1217 } 1218 1219 edesc->src_nents = src_nents; 1220 edesc->dst_nents = dst_nents; 1221 edesc->iv_dma = iv_dma; 1222 edesc->qm_sg_bytes = qm_sg_bytes; 1223 1224 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1225 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1226 1227 if (req->src != req->dst) 1228 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1229 1230 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1231 ivsize, 0); 1232 1233 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1234 DMA_TO_DEVICE); 1235 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1236 dev_err(dev, "unable to map S/G table\n"); 1237 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1238 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1239 qi_cache_free(edesc); 1240 return ERR_PTR(-ENOMEM); 1241 } 1242 1243 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1244 dpaa2_fl_set_final(in_fle, true); 1245 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1246 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1247 1248 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1249 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1250 1251 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1252 1253 if (req->src == req->dst) 1254 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1255 sizeof(*sg_table)); 1256 else 1257 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1258 sizeof(*sg_table)); 1259 1260 return edesc; 1261 } 1262 1263 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1264 struct aead_request *req) 1265 { 1266 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1267 int ivsize = crypto_aead_ivsize(aead); 1268 1269 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1270 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1271 edesc->qm_sg_bytes); 1272 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1273 } 1274 1275 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1276 struct skcipher_request *req) 1277 { 1278 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1279 int ivsize = crypto_skcipher_ivsize(skcipher); 1280 1281 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1282 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1283 edesc->qm_sg_bytes); 1284 } 1285 1286 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1287 { 1288 struct crypto_async_request *areq = cbk_ctx; 1289 struct aead_request *req = container_of(areq, struct aead_request, 1290 base); 1291 struct caam_request *req_ctx = to_caam_req(areq); 1292 struct aead_edesc *edesc = req_ctx->edesc; 1293 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1294 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1295 int ecode = 0; 1296 1297 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1298 1299 if (unlikely(status)) 1300 ecode = caam_qi2_strstatus(ctx->dev, status); 1301 1302 aead_unmap(ctx->dev, edesc, req); 1303 qi_cache_free(edesc); 1304 aead_request_complete(req, ecode); 1305 } 1306 1307 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1308 { 1309 struct crypto_async_request *areq = cbk_ctx; 1310 struct aead_request *req = container_of(areq, struct aead_request, 1311 base); 1312 struct caam_request *req_ctx = to_caam_req(areq); 1313 struct aead_edesc *edesc = req_ctx->edesc; 1314 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1315 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1316 int ecode = 0; 1317 1318 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1319 1320 if (unlikely(status)) 1321 ecode = caam_qi2_strstatus(ctx->dev, status); 1322 1323 aead_unmap(ctx->dev, edesc, req); 1324 qi_cache_free(edesc); 1325 aead_request_complete(req, ecode); 1326 } 1327 1328 static int aead_encrypt(struct aead_request *req) 1329 { 1330 struct aead_edesc *edesc; 1331 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1332 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1333 struct caam_request *caam_req = aead_request_ctx_dma(req); 1334 int ret; 1335 1336 /* allocate extended descriptor */ 1337 edesc = aead_edesc_alloc(req, true); 1338 if (IS_ERR(edesc)) 1339 return PTR_ERR(edesc); 1340 1341 caam_req->flc = &ctx->flc[ENCRYPT]; 1342 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1343 caam_req->cbk = aead_encrypt_done; 1344 caam_req->ctx = &req->base; 1345 caam_req->edesc = edesc; 1346 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1347 if (ret != -EINPROGRESS && 1348 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1349 aead_unmap(ctx->dev, edesc, req); 1350 qi_cache_free(edesc); 1351 } 1352 1353 return ret; 1354 } 1355 1356 static int aead_decrypt(struct aead_request *req) 1357 { 1358 struct aead_edesc *edesc; 1359 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1360 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1361 struct caam_request *caam_req = aead_request_ctx_dma(req); 1362 int ret; 1363 1364 /* allocate extended descriptor */ 1365 edesc = aead_edesc_alloc(req, false); 1366 if (IS_ERR(edesc)) 1367 return PTR_ERR(edesc); 1368 1369 caam_req->flc = &ctx->flc[DECRYPT]; 1370 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1371 caam_req->cbk = aead_decrypt_done; 1372 caam_req->ctx = &req->base; 1373 caam_req->edesc = edesc; 1374 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1375 if (ret != -EINPROGRESS && 1376 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1377 aead_unmap(ctx->dev, edesc, req); 1378 qi_cache_free(edesc); 1379 } 1380 1381 return ret; 1382 } 1383 1384 static int ipsec_gcm_encrypt(struct aead_request *req) 1385 { 1386 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1387 } 1388 1389 static int ipsec_gcm_decrypt(struct aead_request *req) 1390 { 1391 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1392 } 1393 1394 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1395 { 1396 struct crypto_async_request *areq = cbk_ctx; 1397 struct skcipher_request *req = skcipher_request_cast(areq); 1398 struct caam_request *req_ctx = to_caam_req(areq); 1399 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1400 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1401 struct skcipher_edesc *edesc = req_ctx->edesc; 1402 int ecode = 0; 1403 int ivsize = crypto_skcipher_ivsize(skcipher); 1404 1405 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1406 1407 if (unlikely(status)) 1408 ecode = caam_qi2_strstatus(ctx->dev, status); 1409 1410 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1411 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1412 edesc->src_nents > 1 ? 100 : ivsize, 1); 1413 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1414 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1415 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1416 1417 skcipher_unmap(ctx->dev, edesc, req); 1418 1419 /* 1420 * The crypto API expects us to set the IV (req->iv) to the last 1421 * ciphertext block (CBC mode) or last counter (CTR mode). 1422 * This is used e.g. by the CTS mode. 1423 */ 1424 if (!ecode) 1425 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1426 ivsize); 1427 1428 qi_cache_free(edesc); 1429 skcipher_request_complete(req, ecode); 1430 } 1431 1432 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1433 { 1434 struct crypto_async_request *areq = cbk_ctx; 1435 struct skcipher_request *req = skcipher_request_cast(areq); 1436 struct caam_request *req_ctx = to_caam_req(areq); 1437 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1438 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1439 struct skcipher_edesc *edesc = req_ctx->edesc; 1440 int ecode = 0; 1441 int ivsize = crypto_skcipher_ivsize(skcipher); 1442 1443 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1444 1445 if (unlikely(status)) 1446 ecode = caam_qi2_strstatus(ctx->dev, status); 1447 1448 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1449 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1450 edesc->src_nents > 1 ? 100 : ivsize, 1); 1451 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1452 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1453 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1454 1455 skcipher_unmap(ctx->dev, edesc, req); 1456 1457 /* 1458 * The crypto API expects us to set the IV (req->iv) to the last 1459 * ciphertext block (CBC mode) or last counter (CTR mode). 1460 * This is used e.g. by the CTS mode. 1461 */ 1462 if (!ecode) 1463 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1464 ivsize); 1465 1466 qi_cache_free(edesc); 1467 skcipher_request_complete(req, ecode); 1468 } 1469 1470 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1471 { 1472 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1473 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1474 1475 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1476 } 1477 1478 static int skcipher_encrypt(struct skcipher_request *req) 1479 { 1480 struct skcipher_edesc *edesc; 1481 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1482 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1483 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1484 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1485 int ret; 1486 1487 /* 1488 * XTS is expected to return an error even for input length = 0 1489 * Note that the case input length < block size will be caught during 1490 * HW offloading and return an error. 1491 */ 1492 if (!req->cryptlen && !ctx->fallback) 1493 return 0; 1494 1495 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1496 ctx->xts_key_fallback)) { 1497 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1498 skcipher_request_set_callback(&caam_req->fallback_req, 1499 req->base.flags, 1500 req->base.complete, 1501 req->base.data); 1502 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1503 req->dst, req->cryptlen, req->iv); 1504 1505 return crypto_skcipher_encrypt(&caam_req->fallback_req); 1506 } 1507 1508 /* allocate extended descriptor */ 1509 edesc = skcipher_edesc_alloc(req); 1510 if (IS_ERR(edesc)) 1511 return PTR_ERR(edesc); 1512 1513 caam_req->flc = &ctx->flc[ENCRYPT]; 1514 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1515 caam_req->cbk = skcipher_encrypt_done; 1516 caam_req->ctx = &req->base; 1517 caam_req->edesc = edesc; 1518 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1519 if (ret != -EINPROGRESS && 1520 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1521 skcipher_unmap(ctx->dev, edesc, req); 1522 qi_cache_free(edesc); 1523 } 1524 1525 return ret; 1526 } 1527 1528 static int skcipher_decrypt(struct skcipher_request *req) 1529 { 1530 struct skcipher_edesc *edesc; 1531 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1532 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1533 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1534 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1535 int ret; 1536 1537 /* 1538 * XTS is expected to return an error even for input length = 0 1539 * Note that the case input length < block size will be caught during 1540 * HW offloading and return an error. 1541 */ 1542 if (!req->cryptlen && !ctx->fallback) 1543 return 0; 1544 1545 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1546 ctx->xts_key_fallback)) { 1547 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1548 skcipher_request_set_callback(&caam_req->fallback_req, 1549 req->base.flags, 1550 req->base.complete, 1551 req->base.data); 1552 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1553 req->dst, req->cryptlen, req->iv); 1554 1555 return crypto_skcipher_decrypt(&caam_req->fallback_req); 1556 } 1557 1558 /* allocate extended descriptor */ 1559 edesc = skcipher_edesc_alloc(req); 1560 if (IS_ERR(edesc)) 1561 return PTR_ERR(edesc); 1562 1563 caam_req->flc = &ctx->flc[DECRYPT]; 1564 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1565 caam_req->cbk = skcipher_decrypt_done; 1566 caam_req->ctx = &req->base; 1567 caam_req->edesc = edesc; 1568 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1569 if (ret != -EINPROGRESS && 1570 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1571 skcipher_unmap(ctx->dev, edesc, req); 1572 qi_cache_free(edesc); 1573 } 1574 1575 return ret; 1576 } 1577 1578 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1579 bool uses_dkp) 1580 { 1581 dma_addr_t dma_addr; 1582 int i; 1583 1584 /* copy descriptor header template value */ 1585 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1586 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1587 1588 ctx->dev = caam->dev; 1589 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1590 1591 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1592 offsetof(struct caam_ctx, flc_dma), 1593 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1594 if (dma_mapping_error(ctx->dev, dma_addr)) { 1595 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1596 return -ENOMEM; 1597 } 1598 1599 for (i = 0; i < NUM_OP; i++) 1600 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1601 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1602 1603 return 0; 1604 } 1605 1606 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1607 { 1608 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1609 struct caam_skcipher_alg *caam_alg = 1610 container_of(alg, typeof(*caam_alg), skcipher); 1611 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1612 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 1613 int ret = 0; 1614 1615 if (alg_aai == OP_ALG_AAI_XTS) { 1616 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1617 struct crypto_skcipher *fallback; 1618 1619 fallback = crypto_alloc_skcipher(tfm_name, 0, 1620 CRYPTO_ALG_NEED_FALLBACK); 1621 if (IS_ERR(fallback)) { 1622 dev_err(caam_alg->caam.dev, 1623 "Failed to allocate %s fallback: %ld\n", 1624 tfm_name, PTR_ERR(fallback)); 1625 return PTR_ERR(fallback); 1626 } 1627 1628 ctx->fallback = fallback; 1629 crypto_skcipher_set_reqsize_dma( 1630 tfm, sizeof(struct caam_request) + 1631 crypto_skcipher_reqsize(fallback)); 1632 } else { 1633 crypto_skcipher_set_reqsize_dma(tfm, 1634 sizeof(struct caam_request)); 1635 } 1636 1637 ret = caam_cra_init(ctx, &caam_alg->caam, false); 1638 if (ret && ctx->fallback) 1639 crypto_free_skcipher(ctx->fallback); 1640 1641 return ret; 1642 } 1643 1644 static int caam_cra_init_aead(struct crypto_aead *tfm) 1645 { 1646 struct aead_alg *alg = crypto_aead_alg(tfm); 1647 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1648 aead); 1649 1650 crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request)); 1651 return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam, 1652 !caam_alg->caam.nodkp); 1653 } 1654 1655 static void caam_exit_common(struct caam_ctx *ctx) 1656 { 1657 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1658 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1659 DMA_ATTR_SKIP_CPU_SYNC); 1660 } 1661 1662 static void caam_cra_exit(struct crypto_skcipher *tfm) 1663 { 1664 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1665 1666 if (ctx->fallback) 1667 crypto_free_skcipher(ctx->fallback); 1668 caam_exit_common(ctx); 1669 } 1670 1671 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1672 { 1673 caam_exit_common(crypto_aead_ctx_dma(tfm)); 1674 } 1675 1676 static struct caam_skcipher_alg driver_algs[] = { 1677 { 1678 .skcipher = { 1679 .base = { 1680 .cra_name = "cbc(aes)", 1681 .cra_driver_name = "cbc-aes-caam-qi2", 1682 .cra_blocksize = AES_BLOCK_SIZE, 1683 }, 1684 .setkey = aes_skcipher_setkey, 1685 .encrypt = skcipher_encrypt, 1686 .decrypt = skcipher_decrypt, 1687 .min_keysize = AES_MIN_KEY_SIZE, 1688 .max_keysize = AES_MAX_KEY_SIZE, 1689 .ivsize = AES_BLOCK_SIZE, 1690 }, 1691 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1692 }, 1693 { 1694 .skcipher = { 1695 .base = { 1696 .cra_name = "cbc(des3_ede)", 1697 .cra_driver_name = "cbc-3des-caam-qi2", 1698 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1699 }, 1700 .setkey = des3_skcipher_setkey, 1701 .encrypt = skcipher_encrypt, 1702 .decrypt = skcipher_decrypt, 1703 .min_keysize = DES3_EDE_KEY_SIZE, 1704 .max_keysize = DES3_EDE_KEY_SIZE, 1705 .ivsize = DES3_EDE_BLOCK_SIZE, 1706 }, 1707 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1708 }, 1709 { 1710 .skcipher = { 1711 .base = { 1712 .cra_name = "cbc(des)", 1713 .cra_driver_name = "cbc-des-caam-qi2", 1714 .cra_blocksize = DES_BLOCK_SIZE, 1715 }, 1716 .setkey = des_skcipher_setkey, 1717 .encrypt = skcipher_encrypt, 1718 .decrypt = skcipher_decrypt, 1719 .min_keysize = DES_KEY_SIZE, 1720 .max_keysize = DES_KEY_SIZE, 1721 .ivsize = DES_BLOCK_SIZE, 1722 }, 1723 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1724 }, 1725 { 1726 .skcipher = { 1727 .base = { 1728 .cra_name = "ctr(aes)", 1729 .cra_driver_name = "ctr-aes-caam-qi2", 1730 .cra_blocksize = 1, 1731 }, 1732 .setkey = ctr_skcipher_setkey, 1733 .encrypt = skcipher_encrypt, 1734 .decrypt = skcipher_decrypt, 1735 .min_keysize = AES_MIN_KEY_SIZE, 1736 .max_keysize = AES_MAX_KEY_SIZE, 1737 .ivsize = AES_BLOCK_SIZE, 1738 .chunksize = AES_BLOCK_SIZE, 1739 }, 1740 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1741 OP_ALG_AAI_CTR_MOD128, 1742 }, 1743 { 1744 .skcipher = { 1745 .base = { 1746 .cra_name = "rfc3686(ctr(aes))", 1747 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1748 .cra_blocksize = 1, 1749 }, 1750 .setkey = rfc3686_skcipher_setkey, 1751 .encrypt = skcipher_encrypt, 1752 .decrypt = skcipher_decrypt, 1753 .min_keysize = AES_MIN_KEY_SIZE + 1754 CTR_RFC3686_NONCE_SIZE, 1755 .max_keysize = AES_MAX_KEY_SIZE + 1756 CTR_RFC3686_NONCE_SIZE, 1757 .ivsize = CTR_RFC3686_IV_SIZE, 1758 .chunksize = AES_BLOCK_SIZE, 1759 }, 1760 .caam = { 1761 .class1_alg_type = OP_ALG_ALGSEL_AES | 1762 OP_ALG_AAI_CTR_MOD128, 1763 .rfc3686 = true, 1764 }, 1765 }, 1766 { 1767 .skcipher = { 1768 .base = { 1769 .cra_name = "xts(aes)", 1770 .cra_driver_name = "xts-aes-caam-qi2", 1771 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1772 .cra_blocksize = AES_BLOCK_SIZE, 1773 }, 1774 .setkey = xts_skcipher_setkey, 1775 .encrypt = skcipher_encrypt, 1776 .decrypt = skcipher_decrypt, 1777 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1778 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1779 .ivsize = AES_BLOCK_SIZE, 1780 }, 1781 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1782 }, 1783 { 1784 .skcipher = { 1785 .base = { 1786 .cra_name = "chacha20", 1787 .cra_driver_name = "chacha20-caam-qi2", 1788 .cra_blocksize = 1, 1789 }, 1790 .setkey = chacha20_skcipher_setkey, 1791 .encrypt = skcipher_encrypt, 1792 .decrypt = skcipher_decrypt, 1793 .min_keysize = CHACHA_KEY_SIZE, 1794 .max_keysize = CHACHA_KEY_SIZE, 1795 .ivsize = CHACHA_IV_SIZE, 1796 }, 1797 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1798 }, 1799 }; 1800 1801 static struct caam_aead_alg driver_aeads[] = { 1802 { 1803 .aead = { 1804 .base = { 1805 .cra_name = "rfc4106(gcm(aes))", 1806 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1807 .cra_blocksize = 1, 1808 }, 1809 .setkey = rfc4106_setkey, 1810 .setauthsize = rfc4106_setauthsize, 1811 .encrypt = ipsec_gcm_encrypt, 1812 .decrypt = ipsec_gcm_decrypt, 1813 .ivsize = 8, 1814 .maxauthsize = AES_BLOCK_SIZE, 1815 }, 1816 .caam = { 1817 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1818 .nodkp = true, 1819 }, 1820 }, 1821 { 1822 .aead = { 1823 .base = { 1824 .cra_name = "rfc4543(gcm(aes))", 1825 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1826 .cra_blocksize = 1, 1827 }, 1828 .setkey = rfc4543_setkey, 1829 .setauthsize = rfc4543_setauthsize, 1830 .encrypt = ipsec_gcm_encrypt, 1831 .decrypt = ipsec_gcm_decrypt, 1832 .ivsize = 8, 1833 .maxauthsize = AES_BLOCK_SIZE, 1834 }, 1835 .caam = { 1836 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1837 .nodkp = true, 1838 }, 1839 }, 1840 /* Galois Counter Mode */ 1841 { 1842 .aead = { 1843 .base = { 1844 .cra_name = "gcm(aes)", 1845 .cra_driver_name = "gcm-aes-caam-qi2", 1846 .cra_blocksize = 1, 1847 }, 1848 .setkey = gcm_setkey, 1849 .setauthsize = gcm_setauthsize, 1850 .encrypt = aead_encrypt, 1851 .decrypt = aead_decrypt, 1852 .ivsize = 12, 1853 .maxauthsize = AES_BLOCK_SIZE, 1854 }, 1855 .caam = { 1856 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1857 .nodkp = true, 1858 } 1859 }, 1860 /* single-pass ipsec_esp descriptor */ 1861 { 1862 .aead = { 1863 .base = { 1864 .cra_name = "authenc(hmac(md5),cbc(aes))", 1865 .cra_driver_name = "authenc-hmac-md5-" 1866 "cbc-aes-caam-qi2", 1867 .cra_blocksize = AES_BLOCK_SIZE, 1868 }, 1869 .setkey = aead_setkey, 1870 .setauthsize = aead_setauthsize, 1871 .encrypt = aead_encrypt, 1872 .decrypt = aead_decrypt, 1873 .ivsize = AES_BLOCK_SIZE, 1874 .maxauthsize = MD5_DIGEST_SIZE, 1875 }, 1876 .caam = { 1877 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1878 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1879 OP_ALG_AAI_HMAC_PRECOMP, 1880 } 1881 }, 1882 { 1883 .aead = { 1884 .base = { 1885 .cra_name = "echainiv(authenc(hmac(md5)," 1886 "cbc(aes)))", 1887 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1888 "cbc-aes-caam-qi2", 1889 .cra_blocksize = AES_BLOCK_SIZE, 1890 }, 1891 .setkey = aead_setkey, 1892 .setauthsize = aead_setauthsize, 1893 .encrypt = aead_encrypt, 1894 .decrypt = aead_decrypt, 1895 .ivsize = AES_BLOCK_SIZE, 1896 .maxauthsize = MD5_DIGEST_SIZE, 1897 }, 1898 .caam = { 1899 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1900 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1901 OP_ALG_AAI_HMAC_PRECOMP, 1902 .geniv = true, 1903 } 1904 }, 1905 { 1906 .aead = { 1907 .base = { 1908 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1909 .cra_driver_name = "authenc-hmac-sha1-" 1910 "cbc-aes-caam-qi2", 1911 .cra_blocksize = AES_BLOCK_SIZE, 1912 }, 1913 .setkey = aead_setkey, 1914 .setauthsize = aead_setauthsize, 1915 .encrypt = aead_encrypt, 1916 .decrypt = aead_decrypt, 1917 .ivsize = AES_BLOCK_SIZE, 1918 .maxauthsize = SHA1_DIGEST_SIZE, 1919 }, 1920 .caam = { 1921 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1922 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1923 OP_ALG_AAI_HMAC_PRECOMP, 1924 } 1925 }, 1926 { 1927 .aead = { 1928 .base = { 1929 .cra_name = "echainiv(authenc(hmac(sha1)," 1930 "cbc(aes)))", 1931 .cra_driver_name = "echainiv-authenc-" 1932 "hmac-sha1-cbc-aes-caam-qi2", 1933 .cra_blocksize = AES_BLOCK_SIZE, 1934 }, 1935 .setkey = aead_setkey, 1936 .setauthsize = aead_setauthsize, 1937 .encrypt = aead_encrypt, 1938 .decrypt = aead_decrypt, 1939 .ivsize = AES_BLOCK_SIZE, 1940 .maxauthsize = SHA1_DIGEST_SIZE, 1941 }, 1942 .caam = { 1943 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1944 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1945 OP_ALG_AAI_HMAC_PRECOMP, 1946 .geniv = true, 1947 }, 1948 }, 1949 { 1950 .aead = { 1951 .base = { 1952 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1953 .cra_driver_name = "authenc-hmac-sha224-" 1954 "cbc-aes-caam-qi2", 1955 .cra_blocksize = AES_BLOCK_SIZE, 1956 }, 1957 .setkey = aead_setkey, 1958 .setauthsize = aead_setauthsize, 1959 .encrypt = aead_encrypt, 1960 .decrypt = aead_decrypt, 1961 .ivsize = AES_BLOCK_SIZE, 1962 .maxauthsize = SHA224_DIGEST_SIZE, 1963 }, 1964 .caam = { 1965 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1966 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1967 OP_ALG_AAI_HMAC_PRECOMP, 1968 } 1969 }, 1970 { 1971 .aead = { 1972 .base = { 1973 .cra_name = "echainiv(authenc(hmac(sha224)," 1974 "cbc(aes)))", 1975 .cra_driver_name = "echainiv-authenc-" 1976 "hmac-sha224-cbc-aes-caam-qi2", 1977 .cra_blocksize = AES_BLOCK_SIZE, 1978 }, 1979 .setkey = aead_setkey, 1980 .setauthsize = aead_setauthsize, 1981 .encrypt = aead_encrypt, 1982 .decrypt = aead_decrypt, 1983 .ivsize = AES_BLOCK_SIZE, 1984 .maxauthsize = SHA224_DIGEST_SIZE, 1985 }, 1986 .caam = { 1987 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1988 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1989 OP_ALG_AAI_HMAC_PRECOMP, 1990 .geniv = true, 1991 } 1992 }, 1993 { 1994 .aead = { 1995 .base = { 1996 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1997 .cra_driver_name = "authenc-hmac-sha256-" 1998 "cbc-aes-caam-qi2", 1999 .cra_blocksize = AES_BLOCK_SIZE, 2000 }, 2001 .setkey = aead_setkey, 2002 .setauthsize = aead_setauthsize, 2003 .encrypt = aead_encrypt, 2004 .decrypt = aead_decrypt, 2005 .ivsize = AES_BLOCK_SIZE, 2006 .maxauthsize = SHA256_DIGEST_SIZE, 2007 }, 2008 .caam = { 2009 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2010 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2011 OP_ALG_AAI_HMAC_PRECOMP, 2012 } 2013 }, 2014 { 2015 .aead = { 2016 .base = { 2017 .cra_name = "echainiv(authenc(hmac(sha256)," 2018 "cbc(aes)))", 2019 .cra_driver_name = "echainiv-authenc-" 2020 "hmac-sha256-cbc-aes-" 2021 "caam-qi2", 2022 .cra_blocksize = AES_BLOCK_SIZE, 2023 }, 2024 .setkey = aead_setkey, 2025 .setauthsize = aead_setauthsize, 2026 .encrypt = aead_encrypt, 2027 .decrypt = aead_decrypt, 2028 .ivsize = AES_BLOCK_SIZE, 2029 .maxauthsize = SHA256_DIGEST_SIZE, 2030 }, 2031 .caam = { 2032 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2033 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2034 OP_ALG_AAI_HMAC_PRECOMP, 2035 .geniv = true, 2036 } 2037 }, 2038 { 2039 .aead = { 2040 .base = { 2041 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2042 .cra_driver_name = "authenc-hmac-sha384-" 2043 "cbc-aes-caam-qi2", 2044 .cra_blocksize = AES_BLOCK_SIZE, 2045 }, 2046 .setkey = aead_setkey, 2047 .setauthsize = aead_setauthsize, 2048 .encrypt = aead_encrypt, 2049 .decrypt = aead_decrypt, 2050 .ivsize = AES_BLOCK_SIZE, 2051 .maxauthsize = SHA384_DIGEST_SIZE, 2052 }, 2053 .caam = { 2054 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2055 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2056 OP_ALG_AAI_HMAC_PRECOMP, 2057 } 2058 }, 2059 { 2060 .aead = { 2061 .base = { 2062 .cra_name = "echainiv(authenc(hmac(sha384)," 2063 "cbc(aes)))", 2064 .cra_driver_name = "echainiv-authenc-" 2065 "hmac-sha384-cbc-aes-" 2066 "caam-qi2", 2067 .cra_blocksize = AES_BLOCK_SIZE, 2068 }, 2069 .setkey = aead_setkey, 2070 .setauthsize = aead_setauthsize, 2071 .encrypt = aead_encrypt, 2072 .decrypt = aead_decrypt, 2073 .ivsize = AES_BLOCK_SIZE, 2074 .maxauthsize = SHA384_DIGEST_SIZE, 2075 }, 2076 .caam = { 2077 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2078 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2079 OP_ALG_AAI_HMAC_PRECOMP, 2080 .geniv = true, 2081 } 2082 }, 2083 { 2084 .aead = { 2085 .base = { 2086 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2087 .cra_driver_name = "authenc-hmac-sha512-" 2088 "cbc-aes-caam-qi2", 2089 .cra_blocksize = AES_BLOCK_SIZE, 2090 }, 2091 .setkey = aead_setkey, 2092 .setauthsize = aead_setauthsize, 2093 .encrypt = aead_encrypt, 2094 .decrypt = aead_decrypt, 2095 .ivsize = AES_BLOCK_SIZE, 2096 .maxauthsize = SHA512_DIGEST_SIZE, 2097 }, 2098 .caam = { 2099 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2100 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2101 OP_ALG_AAI_HMAC_PRECOMP, 2102 } 2103 }, 2104 { 2105 .aead = { 2106 .base = { 2107 .cra_name = "echainiv(authenc(hmac(sha512)," 2108 "cbc(aes)))", 2109 .cra_driver_name = "echainiv-authenc-" 2110 "hmac-sha512-cbc-aes-" 2111 "caam-qi2", 2112 .cra_blocksize = AES_BLOCK_SIZE, 2113 }, 2114 .setkey = aead_setkey, 2115 .setauthsize = aead_setauthsize, 2116 .encrypt = aead_encrypt, 2117 .decrypt = aead_decrypt, 2118 .ivsize = AES_BLOCK_SIZE, 2119 .maxauthsize = SHA512_DIGEST_SIZE, 2120 }, 2121 .caam = { 2122 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2123 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2124 OP_ALG_AAI_HMAC_PRECOMP, 2125 .geniv = true, 2126 } 2127 }, 2128 { 2129 .aead = { 2130 .base = { 2131 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2132 .cra_driver_name = "authenc-hmac-md5-" 2133 "cbc-des3_ede-caam-qi2", 2134 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2135 }, 2136 .setkey = des3_aead_setkey, 2137 .setauthsize = aead_setauthsize, 2138 .encrypt = aead_encrypt, 2139 .decrypt = aead_decrypt, 2140 .ivsize = DES3_EDE_BLOCK_SIZE, 2141 .maxauthsize = MD5_DIGEST_SIZE, 2142 }, 2143 .caam = { 2144 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2145 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2146 OP_ALG_AAI_HMAC_PRECOMP, 2147 } 2148 }, 2149 { 2150 .aead = { 2151 .base = { 2152 .cra_name = "echainiv(authenc(hmac(md5)," 2153 "cbc(des3_ede)))", 2154 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2155 "cbc-des3_ede-caam-qi2", 2156 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2157 }, 2158 .setkey = des3_aead_setkey, 2159 .setauthsize = aead_setauthsize, 2160 .encrypt = aead_encrypt, 2161 .decrypt = aead_decrypt, 2162 .ivsize = DES3_EDE_BLOCK_SIZE, 2163 .maxauthsize = MD5_DIGEST_SIZE, 2164 }, 2165 .caam = { 2166 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2167 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2168 OP_ALG_AAI_HMAC_PRECOMP, 2169 .geniv = true, 2170 } 2171 }, 2172 { 2173 .aead = { 2174 .base = { 2175 .cra_name = "authenc(hmac(sha1)," 2176 "cbc(des3_ede))", 2177 .cra_driver_name = "authenc-hmac-sha1-" 2178 "cbc-des3_ede-caam-qi2", 2179 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2180 }, 2181 .setkey = des3_aead_setkey, 2182 .setauthsize = aead_setauthsize, 2183 .encrypt = aead_encrypt, 2184 .decrypt = aead_decrypt, 2185 .ivsize = DES3_EDE_BLOCK_SIZE, 2186 .maxauthsize = SHA1_DIGEST_SIZE, 2187 }, 2188 .caam = { 2189 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2190 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2191 OP_ALG_AAI_HMAC_PRECOMP, 2192 }, 2193 }, 2194 { 2195 .aead = { 2196 .base = { 2197 .cra_name = "echainiv(authenc(hmac(sha1)," 2198 "cbc(des3_ede)))", 2199 .cra_driver_name = "echainiv-authenc-" 2200 "hmac-sha1-" 2201 "cbc-des3_ede-caam-qi2", 2202 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2203 }, 2204 .setkey = des3_aead_setkey, 2205 .setauthsize = aead_setauthsize, 2206 .encrypt = aead_encrypt, 2207 .decrypt = aead_decrypt, 2208 .ivsize = DES3_EDE_BLOCK_SIZE, 2209 .maxauthsize = SHA1_DIGEST_SIZE, 2210 }, 2211 .caam = { 2212 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2213 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2214 OP_ALG_AAI_HMAC_PRECOMP, 2215 .geniv = true, 2216 } 2217 }, 2218 { 2219 .aead = { 2220 .base = { 2221 .cra_name = "authenc(hmac(sha224)," 2222 "cbc(des3_ede))", 2223 .cra_driver_name = "authenc-hmac-sha224-" 2224 "cbc-des3_ede-caam-qi2", 2225 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2226 }, 2227 .setkey = des3_aead_setkey, 2228 .setauthsize = aead_setauthsize, 2229 .encrypt = aead_encrypt, 2230 .decrypt = aead_decrypt, 2231 .ivsize = DES3_EDE_BLOCK_SIZE, 2232 .maxauthsize = SHA224_DIGEST_SIZE, 2233 }, 2234 .caam = { 2235 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2236 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2237 OP_ALG_AAI_HMAC_PRECOMP, 2238 }, 2239 }, 2240 { 2241 .aead = { 2242 .base = { 2243 .cra_name = "echainiv(authenc(hmac(sha224)," 2244 "cbc(des3_ede)))", 2245 .cra_driver_name = "echainiv-authenc-" 2246 "hmac-sha224-" 2247 "cbc-des3_ede-caam-qi2", 2248 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2249 }, 2250 .setkey = des3_aead_setkey, 2251 .setauthsize = aead_setauthsize, 2252 .encrypt = aead_encrypt, 2253 .decrypt = aead_decrypt, 2254 .ivsize = DES3_EDE_BLOCK_SIZE, 2255 .maxauthsize = SHA224_DIGEST_SIZE, 2256 }, 2257 .caam = { 2258 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2259 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2260 OP_ALG_AAI_HMAC_PRECOMP, 2261 .geniv = true, 2262 } 2263 }, 2264 { 2265 .aead = { 2266 .base = { 2267 .cra_name = "authenc(hmac(sha256)," 2268 "cbc(des3_ede))", 2269 .cra_driver_name = "authenc-hmac-sha256-" 2270 "cbc-des3_ede-caam-qi2", 2271 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2272 }, 2273 .setkey = des3_aead_setkey, 2274 .setauthsize = aead_setauthsize, 2275 .encrypt = aead_encrypt, 2276 .decrypt = aead_decrypt, 2277 .ivsize = DES3_EDE_BLOCK_SIZE, 2278 .maxauthsize = SHA256_DIGEST_SIZE, 2279 }, 2280 .caam = { 2281 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2282 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2283 OP_ALG_AAI_HMAC_PRECOMP, 2284 }, 2285 }, 2286 { 2287 .aead = { 2288 .base = { 2289 .cra_name = "echainiv(authenc(hmac(sha256)," 2290 "cbc(des3_ede)))", 2291 .cra_driver_name = "echainiv-authenc-" 2292 "hmac-sha256-" 2293 "cbc-des3_ede-caam-qi2", 2294 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2295 }, 2296 .setkey = des3_aead_setkey, 2297 .setauthsize = aead_setauthsize, 2298 .encrypt = aead_encrypt, 2299 .decrypt = aead_decrypt, 2300 .ivsize = DES3_EDE_BLOCK_SIZE, 2301 .maxauthsize = SHA256_DIGEST_SIZE, 2302 }, 2303 .caam = { 2304 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2305 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2306 OP_ALG_AAI_HMAC_PRECOMP, 2307 .geniv = true, 2308 } 2309 }, 2310 { 2311 .aead = { 2312 .base = { 2313 .cra_name = "authenc(hmac(sha384)," 2314 "cbc(des3_ede))", 2315 .cra_driver_name = "authenc-hmac-sha384-" 2316 "cbc-des3_ede-caam-qi2", 2317 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2318 }, 2319 .setkey = des3_aead_setkey, 2320 .setauthsize = aead_setauthsize, 2321 .encrypt = aead_encrypt, 2322 .decrypt = aead_decrypt, 2323 .ivsize = DES3_EDE_BLOCK_SIZE, 2324 .maxauthsize = SHA384_DIGEST_SIZE, 2325 }, 2326 .caam = { 2327 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2328 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2329 OP_ALG_AAI_HMAC_PRECOMP, 2330 }, 2331 }, 2332 { 2333 .aead = { 2334 .base = { 2335 .cra_name = "echainiv(authenc(hmac(sha384)," 2336 "cbc(des3_ede)))", 2337 .cra_driver_name = "echainiv-authenc-" 2338 "hmac-sha384-" 2339 "cbc-des3_ede-caam-qi2", 2340 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2341 }, 2342 .setkey = des3_aead_setkey, 2343 .setauthsize = aead_setauthsize, 2344 .encrypt = aead_encrypt, 2345 .decrypt = aead_decrypt, 2346 .ivsize = DES3_EDE_BLOCK_SIZE, 2347 .maxauthsize = SHA384_DIGEST_SIZE, 2348 }, 2349 .caam = { 2350 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2351 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2352 OP_ALG_AAI_HMAC_PRECOMP, 2353 .geniv = true, 2354 } 2355 }, 2356 { 2357 .aead = { 2358 .base = { 2359 .cra_name = "authenc(hmac(sha512)," 2360 "cbc(des3_ede))", 2361 .cra_driver_name = "authenc-hmac-sha512-" 2362 "cbc-des3_ede-caam-qi2", 2363 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2364 }, 2365 .setkey = des3_aead_setkey, 2366 .setauthsize = aead_setauthsize, 2367 .encrypt = aead_encrypt, 2368 .decrypt = aead_decrypt, 2369 .ivsize = DES3_EDE_BLOCK_SIZE, 2370 .maxauthsize = SHA512_DIGEST_SIZE, 2371 }, 2372 .caam = { 2373 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2374 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2375 OP_ALG_AAI_HMAC_PRECOMP, 2376 }, 2377 }, 2378 { 2379 .aead = { 2380 .base = { 2381 .cra_name = "echainiv(authenc(hmac(sha512)," 2382 "cbc(des3_ede)))", 2383 .cra_driver_name = "echainiv-authenc-" 2384 "hmac-sha512-" 2385 "cbc-des3_ede-caam-qi2", 2386 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2387 }, 2388 .setkey = des3_aead_setkey, 2389 .setauthsize = aead_setauthsize, 2390 .encrypt = aead_encrypt, 2391 .decrypt = aead_decrypt, 2392 .ivsize = DES3_EDE_BLOCK_SIZE, 2393 .maxauthsize = SHA512_DIGEST_SIZE, 2394 }, 2395 .caam = { 2396 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2397 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2398 OP_ALG_AAI_HMAC_PRECOMP, 2399 .geniv = true, 2400 } 2401 }, 2402 { 2403 .aead = { 2404 .base = { 2405 .cra_name = "authenc(hmac(md5),cbc(des))", 2406 .cra_driver_name = "authenc-hmac-md5-" 2407 "cbc-des-caam-qi2", 2408 .cra_blocksize = DES_BLOCK_SIZE, 2409 }, 2410 .setkey = aead_setkey, 2411 .setauthsize = aead_setauthsize, 2412 .encrypt = aead_encrypt, 2413 .decrypt = aead_decrypt, 2414 .ivsize = DES_BLOCK_SIZE, 2415 .maxauthsize = MD5_DIGEST_SIZE, 2416 }, 2417 .caam = { 2418 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2419 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2420 OP_ALG_AAI_HMAC_PRECOMP, 2421 }, 2422 }, 2423 { 2424 .aead = { 2425 .base = { 2426 .cra_name = "echainiv(authenc(hmac(md5)," 2427 "cbc(des)))", 2428 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2429 "cbc-des-caam-qi2", 2430 .cra_blocksize = DES_BLOCK_SIZE, 2431 }, 2432 .setkey = aead_setkey, 2433 .setauthsize = aead_setauthsize, 2434 .encrypt = aead_encrypt, 2435 .decrypt = aead_decrypt, 2436 .ivsize = DES_BLOCK_SIZE, 2437 .maxauthsize = MD5_DIGEST_SIZE, 2438 }, 2439 .caam = { 2440 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2441 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2442 OP_ALG_AAI_HMAC_PRECOMP, 2443 .geniv = true, 2444 } 2445 }, 2446 { 2447 .aead = { 2448 .base = { 2449 .cra_name = "authenc(hmac(sha1),cbc(des))", 2450 .cra_driver_name = "authenc-hmac-sha1-" 2451 "cbc-des-caam-qi2", 2452 .cra_blocksize = DES_BLOCK_SIZE, 2453 }, 2454 .setkey = aead_setkey, 2455 .setauthsize = aead_setauthsize, 2456 .encrypt = aead_encrypt, 2457 .decrypt = aead_decrypt, 2458 .ivsize = DES_BLOCK_SIZE, 2459 .maxauthsize = SHA1_DIGEST_SIZE, 2460 }, 2461 .caam = { 2462 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2463 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2464 OP_ALG_AAI_HMAC_PRECOMP, 2465 }, 2466 }, 2467 { 2468 .aead = { 2469 .base = { 2470 .cra_name = "echainiv(authenc(hmac(sha1)," 2471 "cbc(des)))", 2472 .cra_driver_name = "echainiv-authenc-" 2473 "hmac-sha1-cbc-des-caam-qi2", 2474 .cra_blocksize = DES_BLOCK_SIZE, 2475 }, 2476 .setkey = aead_setkey, 2477 .setauthsize = aead_setauthsize, 2478 .encrypt = aead_encrypt, 2479 .decrypt = aead_decrypt, 2480 .ivsize = DES_BLOCK_SIZE, 2481 .maxauthsize = SHA1_DIGEST_SIZE, 2482 }, 2483 .caam = { 2484 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2485 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2486 OP_ALG_AAI_HMAC_PRECOMP, 2487 .geniv = true, 2488 } 2489 }, 2490 { 2491 .aead = { 2492 .base = { 2493 .cra_name = "authenc(hmac(sha224),cbc(des))", 2494 .cra_driver_name = "authenc-hmac-sha224-" 2495 "cbc-des-caam-qi2", 2496 .cra_blocksize = DES_BLOCK_SIZE, 2497 }, 2498 .setkey = aead_setkey, 2499 .setauthsize = aead_setauthsize, 2500 .encrypt = aead_encrypt, 2501 .decrypt = aead_decrypt, 2502 .ivsize = DES_BLOCK_SIZE, 2503 .maxauthsize = SHA224_DIGEST_SIZE, 2504 }, 2505 .caam = { 2506 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2507 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2508 OP_ALG_AAI_HMAC_PRECOMP, 2509 }, 2510 }, 2511 { 2512 .aead = { 2513 .base = { 2514 .cra_name = "echainiv(authenc(hmac(sha224)," 2515 "cbc(des)))", 2516 .cra_driver_name = "echainiv-authenc-" 2517 "hmac-sha224-cbc-des-" 2518 "caam-qi2", 2519 .cra_blocksize = DES_BLOCK_SIZE, 2520 }, 2521 .setkey = aead_setkey, 2522 .setauthsize = aead_setauthsize, 2523 .encrypt = aead_encrypt, 2524 .decrypt = aead_decrypt, 2525 .ivsize = DES_BLOCK_SIZE, 2526 .maxauthsize = SHA224_DIGEST_SIZE, 2527 }, 2528 .caam = { 2529 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2530 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2531 OP_ALG_AAI_HMAC_PRECOMP, 2532 .geniv = true, 2533 } 2534 }, 2535 { 2536 .aead = { 2537 .base = { 2538 .cra_name = "authenc(hmac(sha256),cbc(des))", 2539 .cra_driver_name = "authenc-hmac-sha256-" 2540 "cbc-des-caam-qi2", 2541 .cra_blocksize = DES_BLOCK_SIZE, 2542 }, 2543 .setkey = aead_setkey, 2544 .setauthsize = aead_setauthsize, 2545 .encrypt = aead_encrypt, 2546 .decrypt = aead_decrypt, 2547 .ivsize = DES_BLOCK_SIZE, 2548 .maxauthsize = SHA256_DIGEST_SIZE, 2549 }, 2550 .caam = { 2551 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2552 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2553 OP_ALG_AAI_HMAC_PRECOMP, 2554 }, 2555 }, 2556 { 2557 .aead = { 2558 .base = { 2559 .cra_name = "echainiv(authenc(hmac(sha256)," 2560 "cbc(des)))", 2561 .cra_driver_name = "echainiv-authenc-" 2562 "hmac-sha256-cbc-des-" 2563 "caam-qi2", 2564 .cra_blocksize = DES_BLOCK_SIZE, 2565 }, 2566 .setkey = aead_setkey, 2567 .setauthsize = aead_setauthsize, 2568 .encrypt = aead_encrypt, 2569 .decrypt = aead_decrypt, 2570 .ivsize = DES_BLOCK_SIZE, 2571 .maxauthsize = SHA256_DIGEST_SIZE, 2572 }, 2573 .caam = { 2574 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2575 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2576 OP_ALG_AAI_HMAC_PRECOMP, 2577 .geniv = true, 2578 }, 2579 }, 2580 { 2581 .aead = { 2582 .base = { 2583 .cra_name = "authenc(hmac(sha384),cbc(des))", 2584 .cra_driver_name = "authenc-hmac-sha384-" 2585 "cbc-des-caam-qi2", 2586 .cra_blocksize = DES_BLOCK_SIZE, 2587 }, 2588 .setkey = aead_setkey, 2589 .setauthsize = aead_setauthsize, 2590 .encrypt = aead_encrypt, 2591 .decrypt = aead_decrypt, 2592 .ivsize = DES_BLOCK_SIZE, 2593 .maxauthsize = SHA384_DIGEST_SIZE, 2594 }, 2595 .caam = { 2596 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2597 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2598 OP_ALG_AAI_HMAC_PRECOMP, 2599 }, 2600 }, 2601 { 2602 .aead = { 2603 .base = { 2604 .cra_name = "echainiv(authenc(hmac(sha384)," 2605 "cbc(des)))", 2606 .cra_driver_name = "echainiv-authenc-" 2607 "hmac-sha384-cbc-des-" 2608 "caam-qi2", 2609 .cra_blocksize = DES_BLOCK_SIZE, 2610 }, 2611 .setkey = aead_setkey, 2612 .setauthsize = aead_setauthsize, 2613 .encrypt = aead_encrypt, 2614 .decrypt = aead_decrypt, 2615 .ivsize = DES_BLOCK_SIZE, 2616 .maxauthsize = SHA384_DIGEST_SIZE, 2617 }, 2618 .caam = { 2619 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2620 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2621 OP_ALG_AAI_HMAC_PRECOMP, 2622 .geniv = true, 2623 } 2624 }, 2625 { 2626 .aead = { 2627 .base = { 2628 .cra_name = "authenc(hmac(sha512),cbc(des))", 2629 .cra_driver_name = "authenc-hmac-sha512-" 2630 "cbc-des-caam-qi2", 2631 .cra_blocksize = DES_BLOCK_SIZE, 2632 }, 2633 .setkey = aead_setkey, 2634 .setauthsize = aead_setauthsize, 2635 .encrypt = aead_encrypt, 2636 .decrypt = aead_decrypt, 2637 .ivsize = DES_BLOCK_SIZE, 2638 .maxauthsize = SHA512_DIGEST_SIZE, 2639 }, 2640 .caam = { 2641 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2642 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2643 OP_ALG_AAI_HMAC_PRECOMP, 2644 } 2645 }, 2646 { 2647 .aead = { 2648 .base = { 2649 .cra_name = "echainiv(authenc(hmac(sha512)," 2650 "cbc(des)))", 2651 .cra_driver_name = "echainiv-authenc-" 2652 "hmac-sha512-cbc-des-" 2653 "caam-qi2", 2654 .cra_blocksize = DES_BLOCK_SIZE, 2655 }, 2656 .setkey = aead_setkey, 2657 .setauthsize = aead_setauthsize, 2658 .encrypt = aead_encrypt, 2659 .decrypt = aead_decrypt, 2660 .ivsize = DES_BLOCK_SIZE, 2661 .maxauthsize = SHA512_DIGEST_SIZE, 2662 }, 2663 .caam = { 2664 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2665 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2666 OP_ALG_AAI_HMAC_PRECOMP, 2667 .geniv = true, 2668 } 2669 }, 2670 { 2671 .aead = { 2672 .base = { 2673 .cra_name = "authenc(hmac(md5)," 2674 "rfc3686(ctr(aes)))", 2675 .cra_driver_name = "authenc-hmac-md5-" 2676 "rfc3686-ctr-aes-caam-qi2", 2677 .cra_blocksize = 1, 2678 }, 2679 .setkey = aead_setkey, 2680 .setauthsize = aead_setauthsize, 2681 .encrypt = aead_encrypt, 2682 .decrypt = aead_decrypt, 2683 .ivsize = CTR_RFC3686_IV_SIZE, 2684 .maxauthsize = MD5_DIGEST_SIZE, 2685 }, 2686 .caam = { 2687 .class1_alg_type = OP_ALG_ALGSEL_AES | 2688 OP_ALG_AAI_CTR_MOD128, 2689 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2690 OP_ALG_AAI_HMAC_PRECOMP, 2691 .rfc3686 = true, 2692 }, 2693 }, 2694 { 2695 .aead = { 2696 .base = { 2697 .cra_name = "seqiv(authenc(" 2698 "hmac(md5),rfc3686(ctr(aes))))", 2699 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2700 "rfc3686-ctr-aes-caam-qi2", 2701 .cra_blocksize = 1, 2702 }, 2703 .setkey = aead_setkey, 2704 .setauthsize = aead_setauthsize, 2705 .encrypt = aead_encrypt, 2706 .decrypt = aead_decrypt, 2707 .ivsize = CTR_RFC3686_IV_SIZE, 2708 .maxauthsize = MD5_DIGEST_SIZE, 2709 }, 2710 .caam = { 2711 .class1_alg_type = OP_ALG_ALGSEL_AES | 2712 OP_ALG_AAI_CTR_MOD128, 2713 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2714 OP_ALG_AAI_HMAC_PRECOMP, 2715 .rfc3686 = true, 2716 .geniv = true, 2717 }, 2718 }, 2719 { 2720 .aead = { 2721 .base = { 2722 .cra_name = "authenc(hmac(sha1)," 2723 "rfc3686(ctr(aes)))", 2724 .cra_driver_name = "authenc-hmac-sha1-" 2725 "rfc3686-ctr-aes-caam-qi2", 2726 .cra_blocksize = 1, 2727 }, 2728 .setkey = aead_setkey, 2729 .setauthsize = aead_setauthsize, 2730 .encrypt = aead_encrypt, 2731 .decrypt = aead_decrypt, 2732 .ivsize = CTR_RFC3686_IV_SIZE, 2733 .maxauthsize = SHA1_DIGEST_SIZE, 2734 }, 2735 .caam = { 2736 .class1_alg_type = OP_ALG_ALGSEL_AES | 2737 OP_ALG_AAI_CTR_MOD128, 2738 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2739 OP_ALG_AAI_HMAC_PRECOMP, 2740 .rfc3686 = true, 2741 }, 2742 }, 2743 { 2744 .aead = { 2745 .base = { 2746 .cra_name = "seqiv(authenc(" 2747 "hmac(sha1),rfc3686(ctr(aes))))", 2748 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2749 "rfc3686-ctr-aes-caam-qi2", 2750 .cra_blocksize = 1, 2751 }, 2752 .setkey = aead_setkey, 2753 .setauthsize = aead_setauthsize, 2754 .encrypt = aead_encrypt, 2755 .decrypt = aead_decrypt, 2756 .ivsize = CTR_RFC3686_IV_SIZE, 2757 .maxauthsize = SHA1_DIGEST_SIZE, 2758 }, 2759 .caam = { 2760 .class1_alg_type = OP_ALG_ALGSEL_AES | 2761 OP_ALG_AAI_CTR_MOD128, 2762 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2763 OP_ALG_AAI_HMAC_PRECOMP, 2764 .rfc3686 = true, 2765 .geniv = true, 2766 }, 2767 }, 2768 { 2769 .aead = { 2770 .base = { 2771 .cra_name = "authenc(hmac(sha224)," 2772 "rfc3686(ctr(aes)))", 2773 .cra_driver_name = "authenc-hmac-sha224-" 2774 "rfc3686-ctr-aes-caam-qi2", 2775 .cra_blocksize = 1, 2776 }, 2777 .setkey = aead_setkey, 2778 .setauthsize = aead_setauthsize, 2779 .encrypt = aead_encrypt, 2780 .decrypt = aead_decrypt, 2781 .ivsize = CTR_RFC3686_IV_SIZE, 2782 .maxauthsize = SHA224_DIGEST_SIZE, 2783 }, 2784 .caam = { 2785 .class1_alg_type = OP_ALG_ALGSEL_AES | 2786 OP_ALG_AAI_CTR_MOD128, 2787 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2788 OP_ALG_AAI_HMAC_PRECOMP, 2789 .rfc3686 = true, 2790 }, 2791 }, 2792 { 2793 .aead = { 2794 .base = { 2795 .cra_name = "seqiv(authenc(" 2796 "hmac(sha224),rfc3686(ctr(aes))))", 2797 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2798 "rfc3686-ctr-aes-caam-qi2", 2799 .cra_blocksize = 1, 2800 }, 2801 .setkey = aead_setkey, 2802 .setauthsize = aead_setauthsize, 2803 .encrypt = aead_encrypt, 2804 .decrypt = aead_decrypt, 2805 .ivsize = CTR_RFC3686_IV_SIZE, 2806 .maxauthsize = SHA224_DIGEST_SIZE, 2807 }, 2808 .caam = { 2809 .class1_alg_type = OP_ALG_ALGSEL_AES | 2810 OP_ALG_AAI_CTR_MOD128, 2811 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2812 OP_ALG_AAI_HMAC_PRECOMP, 2813 .rfc3686 = true, 2814 .geniv = true, 2815 }, 2816 }, 2817 { 2818 .aead = { 2819 .base = { 2820 .cra_name = "authenc(hmac(sha256)," 2821 "rfc3686(ctr(aes)))", 2822 .cra_driver_name = "authenc-hmac-sha256-" 2823 "rfc3686-ctr-aes-caam-qi2", 2824 .cra_blocksize = 1, 2825 }, 2826 .setkey = aead_setkey, 2827 .setauthsize = aead_setauthsize, 2828 .encrypt = aead_encrypt, 2829 .decrypt = aead_decrypt, 2830 .ivsize = CTR_RFC3686_IV_SIZE, 2831 .maxauthsize = SHA256_DIGEST_SIZE, 2832 }, 2833 .caam = { 2834 .class1_alg_type = OP_ALG_ALGSEL_AES | 2835 OP_ALG_AAI_CTR_MOD128, 2836 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2837 OP_ALG_AAI_HMAC_PRECOMP, 2838 .rfc3686 = true, 2839 }, 2840 }, 2841 { 2842 .aead = { 2843 .base = { 2844 .cra_name = "seqiv(authenc(hmac(sha256)," 2845 "rfc3686(ctr(aes))))", 2846 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2847 "rfc3686-ctr-aes-caam-qi2", 2848 .cra_blocksize = 1, 2849 }, 2850 .setkey = aead_setkey, 2851 .setauthsize = aead_setauthsize, 2852 .encrypt = aead_encrypt, 2853 .decrypt = aead_decrypt, 2854 .ivsize = CTR_RFC3686_IV_SIZE, 2855 .maxauthsize = SHA256_DIGEST_SIZE, 2856 }, 2857 .caam = { 2858 .class1_alg_type = OP_ALG_ALGSEL_AES | 2859 OP_ALG_AAI_CTR_MOD128, 2860 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2861 OP_ALG_AAI_HMAC_PRECOMP, 2862 .rfc3686 = true, 2863 .geniv = true, 2864 }, 2865 }, 2866 { 2867 .aead = { 2868 .base = { 2869 .cra_name = "authenc(hmac(sha384)," 2870 "rfc3686(ctr(aes)))", 2871 .cra_driver_name = "authenc-hmac-sha384-" 2872 "rfc3686-ctr-aes-caam-qi2", 2873 .cra_blocksize = 1, 2874 }, 2875 .setkey = aead_setkey, 2876 .setauthsize = aead_setauthsize, 2877 .encrypt = aead_encrypt, 2878 .decrypt = aead_decrypt, 2879 .ivsize = CTR_RFC3686_IV_SIZE, 2880 .maxauthsize = SHA384_DIGEST_SIZE, 2881 }, 2882 .caam = { 2883 .class1_alg_type = OP_ALG_ALGSEL_AES | 2884 OP_ALG_AAI_CTR_MOD128, 2885 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2886 OP_ALG_AAI_HMAC_PRECOMP, 2887 .rfc3686 = true, 2888 }, 2889 }, 2890 { 2891 .aead = { 2892 .base = { 2893 .cra_name = "seqiv(authenc(hmac(sha384)," 2894 "rfc3686(ctr(aes))))", 2895 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2896 "rfc3686-ctr-aes-caam-qi2", 2897 .cra_blocksize = 1, 2898 }, 2899 .setkey = aead_setkey, 2900 .setauthsize = aead_setauthsize, 2901 .encrypt = aead_encrypt, 2902 .decrypt = aead_decrypt, 2903 .ivsize = CTR_RFC3686_IV_SIZE, 2904 .maxauthsize = SHA384_DIGEST_SIZE, 2905 }, 2906 .caam = { 2907 .class1_alg_type = OP_ALG_ALGSEL_AES | 2908 OP_ALG_AAI_CTR_MOD128, 2909 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2910 OP_ALG_AAI_HMAC_PRECOMP, 2911 .rfc3686 = true, 2912 .geniv = true, 2913 }, 2914 }, 2915 { 2916 .aead = { 2917 .base = { 2918 .cra_name = "rfc7539(chacha20,poly1305)", 2919 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2920 "caam-qi2", 2921 .cra_blocksize = 1, 2922 }, 2923 .setkey = chachapoly_setkey, 2924 .setauthsize = chachapoly_setauthsize, 2925 .encrypt = aead_encrypt, 2926 .decrypt = aead_decrypt, 2927 .ivsize = CHACHAPOLY_IV_SIZE, 2928 .maxauthsize = POLY1305_DIGEST_SIZE, 2929 }, 2930 .caam = { 2931 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2932 OP_ALG_AAI_AEAD, 2933 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2934 OP_ALG_AAI_AEAD, 2935 .nodkp = true, 2936 }, 2937 }, 2938 { 2939 .aead = { 2940 .base = { 2941 .cra_name = "rfc7539esp(chacha20,poly1305)", 2942 .cra_driver_name = "rfc7539esp-chacha20-" 2943 "poly1305-caam-qi2", 2944 .cra_blocksize = 1, 2945 }, 2946 .setkey = chachapoly_setkey, 2947 .setauthsize = chachapoly_setauthsize, 2948 .encrypt = aead_encrypt, 2949 .decrypt = aead_decrypt, 2950 .ivsize = 8, 2951 .maxauthsize = POLY1305_DIGEST_SIZE, 2952 }, 2953 .caam = { 2954 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2955 OP_ALG_AAI_AEAD, 2956 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2957 OP_ALG_AAI_AEAD, 2958 .nodkp = true, 2959 }, 2960 }, 2961 { 2962 .aead = { 2963 .base = { 2964 .cra_name = "authenc(hmac(sha512)," 2965 "rfc3686(ctr(aes)))", 2966 .cra_driver_name = "authenc-hmac-sha512-" 2967 "rfc3686-ctr-aes-caam-qi2", 2968 .cra_blocksize = 1, 2969 }, 2970 .setkey = aead_setkey, 2971 .setauthsize = aead_setauthsize, 2972 .encrypt = aead_encrypt, 2973 .decrypt = aead_decrypt, 2974 .ivsize = CTR_RFC3686_IV_SIZE, 2975 .maxauthsize = SHA512_DIGEST_SIZE, 2976 }, 2977 .caam = { 2978 .class1_alg_type = OP_ALG_ALGSEL_AES | 2979 OP_ALG_AAI_CTR_MOD128, 2980 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2981 OP_ALG_AAI_HMAC_PRECOMP, 2982 .rfc3686 = true, 2983 }, 2984 }, 2985 { 2986 .aead = { 2987 .base = { 2988 .cra_name = "seqiv(authenc(hmac(sha512)," 2989 "rfc3686(ctr(aes))))", 2990 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2991 "rfc3686-ctr-aes-caam-qi2", 2992 .cra_blocksize = 1, 2993 }, 2994 .setkey = aead_setkey, 2995 .setauthsize = aead_setauthsize, 2996 .encrypt = aead_encrypt, 2997 .decrypt = aead_decrypt, 2998 .ivsize = CTR_RFC3686_IV_SIZE, 2999 .maxauthsize = SHA512_DIGEST_SIZE, 3000 }, 3001 .caam = { 3002 .class1_alg_type = OP_ALG_ALGSEL_AES | 3003 OP_ALG_AAI_CTR_MOD128, 3004 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3005 OP_ALG_AAI_HMAC_PRECOMP, 3006 .rfc3686 = true, 3007 .geniv = true, 3008 }, 3009 }, 3010 }; 3011 3012 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3013 { 3014 struct skcipher_alg *alg = &t_alg->skcipher; 3015 3016 alg->base.cra_module = THIS_MODULE; 3017 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3018 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3019 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3020 CRYPTO_ALG_KERN_DRIVER_ONLY); 3021 3022 alg->init = caam_cra_init_skcipher; 3023 alg->exit = caam_cra_exit; 3024 } 3025 3026 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3027 { 3028 struct aead_alg *alg = &t_alg->aead; 3029 3030 alg->base.cra_module = THIS_MODULE; 3031 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3032 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3033 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3034 CRYPTO_ALG_KERN_DRIVER_ONLY; 3035 3036 alg->init = caam_cra_init_aead; 3037 alg->exit = caam_cra_exit_aead; 3038 } 3039 3040 /* max hash key is max split key size */ 3041 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 3042 3043 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 3044 3045 /* caam context sizes for hashes: running digest + 8 */ 3046 #define HASH_MSG_LEN 8 3047 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 3048 3049 enum hash_optype { 3050 UPDATE = 0, 3051 UPDATE_FIRST, 3052 FINALIZE, 3053 DIGEST, 3054 HASH_NUM_OP 3055 }; 3056 3057 /** 3058 * struct caam_hash_ctx - ahash per-session context 3059 * @flc: Flow Contexts array 3060 * @key: authentication key 3061 * @flc_dma: I/O virtual addresses of the Flow Contexts 3062 * @dev: dpseci device 3063 * @ctx_len: size of Context Register 3064 * @adata: hashing algorithm details 3065 */ 3066 struct caam_hash_ctx { 3067 struct caam_flc flc[HASH_NUM_OP]; 3068 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3069 dma_addr_t flc_dma[HASH_NUM_OP]; 3070 struct device *dev; 3071 int ctx_len; 3072 struct alginfo adata; 3073 }; 3074 3075 /* ahash state */ 3076 struct caam_hash_state { 3077 struct caam_request caam_req; 3078 dma_addr_t buf_dma; 3079 dma_addr_t ctx_dma; 3080 int ctx_dma_len; 3081 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3082 int buflen; 3083 int next_buflen; 3084 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 3085 int (*update)(struct ahash_request *req); 3086 int (*final)(struct ahash_request *req); 3087 int (*finup)(struct ahash_request *req); 3088 }; 3089 3090 struct caam_export_state { 3091 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 3092 u8 caam_ctx[MAX_CTX_LEN]; 3093 int buflen; 3094 int (*update)(struct ahash_request *req); 3095 int (*final)(struct ahash_request *req); 3096 int (*finup)(struct ahash_request *req); 3097 }; 3098 3099 /* Map current buffer in state (if length > 0) and put it in link table */ 3100 static inline int buf_map_to_qm_sg(struct device *dev, 3101 struct dpaa2_sg_entry *qm_sg, 3102 struct caam_hash_state *state) 3103 { 3104 int buflen = state->buflen; 3105 3106 if (!buflen) 3107 return 0; 3108 3109 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3110 DMA_TO_DEVICE); 3111 if (dma_mapping_error(dev, state->buf_dma)) { 3112 dev_err(dev, "unable to map buf\n"); 3113 state->buf_dma = 0; 3114 return -ENOMEM; 3115 } 3116 3117 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3118 3119 return 0; 3120 } 3121 3122 /* Map state->caam_ctx, and add it to link table */ 3123 static inline int ctx_map_to_qm_sg(struct device *dev, 3124 struct caam_hash_state *state, int ctx_len, 3125 struct dpaa2_sg_entry *qm_sg, u32 flag) 3126 { 3127 state->ctx_dma_len = ctx_len; 3128 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3129 if (dma_mapping_error(dev, state->ctx_dma)) { 3130 dev_err(dev, "unable to map ctx\n"); 3131 state->ctx_dma = 0; 3132 return -ENOMEM; 3133 } 3134 3135 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3136 3137 return 0; 3138 } 3139 3140 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3141 { 3142 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3143 int digestsize = crypto_ahash_digestsize(ahash); 3144 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3145 struct caam_flc *flc; 3146 u32 *desc; 3147 3148 /* ahash_update shared descriptor */ 3149 flc = &ctx->flc[UPDATE]; 3150 desc = flc->sh_desc; 3151 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3152 ctx->ctx_len, true, priv->sec_attr.era); 3153 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3154 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3155 desc_bytes(desc), DMA_BIDIRECTIONAL); 3156 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3157 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3158 1); 3159 3160 /* ahash_update_first shared descriptor */ 3161 flc = &ctx->flc[UPDATE_FIRST]; 3162 desc = flc->sh_desc; 3163 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3164 ctx->ctx_len, false, priv->sec_attr.era); 3165 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3166 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3167 desc_bytes(desc), DMA_BIDIRECTIONAL); 3168 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3169 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3170 1); 3171 3172 /* ahash_final shared descriptor */ 3173 flc = &ctx->flc[FINALIZE]; 3174 desc = flc->sh_desc; 3175 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3176 ctx->ctx_len, true, priv->sec_attr.era); 3177 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3178 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3179 desc_bytes(desc), DMA_BIDIRECTIONAL); 3180 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3181 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3182 1); 3183 3184 /* ahash_digest shared descriptor */ 3185 flc = &ctx->flc[DIGEST]; 3186 desc = flc->sh_desc; 3187 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3188 ctx->ctx_len, false, priv->sec_attr.era); 3189 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3190 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3191 desc_bytes(desc), DMA_BIDIRECTIONAL); 3192 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3193 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3194 1); 3195 3196 return 0; 3197 } 3198 3199 struct split_key_sh_result { 3200 struct completion completion; 3201 int err; 3202 struct device *dev; 3203 }; 3204 3205 static void split_key_sh_done(void *cbk_ctx, u32 err) 3206 { 3207 struct split_key_sh_result *res = cbk_ctx; 3208 3209 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3210 3211 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3212 complete(&res->completion); 3213 } 3214 3215 /* Digest hash size if it is too large */ 3216 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3217 u32 digestsize) 3218 { 3219 struct caam_request *req_ctx; 3220 u32 *desc; 3221 struct split_key_sh_result result; 3222 dma_addr_t key_dma; 3223 struct caam_flc *flc; 3224 dma_addr_t flc_dma; 3225 int ret = -ENOMEM; 3226 struct dpaa2_fl_entry *in_fle, *out_fle; 3227 3228 req_ctx = kzalloc_obj(*req_ctx); 3229 if (!req_ctx) 3230 return -ENOMEM; 3231 3232 in_fle = &req_ctx->fd_flt[1]; 3233 out_fle = &req_ctx->fd_flt[0]; 3234 3235 flc = kzalloc_obj(*flc); 3236 if (!flc) 3237 goto err_flc; 3238 3239 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3240 if (dma_mapping_error(ctx->dev, key_dma)) { 3241 dev_err(ctx->dev, "unable to map key memory\n"); 3242 goto err_key_dma; 3243 } 3244 3245 desc = flc->sh_desc; 3246 3247 init_sh_desc(desc, 0); 3248 3249 /* descriptor to perform unkeyed hash on key_in */ 3250 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3251 OP_ALG_AS_INITFINAL); 3252 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3253 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3254 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3255 LDST_SRCDST_BYTE_CONTEXT); 3256 3257 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3258 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3259 desc_bytes(desc), DMA_TO_DEVICE); 3260 if (dma_mapping_error(ctx->dev, flc_dma)) { 3261 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3262 goto err_flc_dma; 3263 } 3264 3265 dpaa2_fl_set_final(in_fle, true); 3266 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3267 dpaa2_fl_set_addr(in_fle, key_dma); 3268 dpaa2_fl_set_len(in_fle, *keylen); 3269 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3270 dpaa2_fl_set_addr(out_fle, key_dma); 3271 dpaa2_fl_set_len(out_fle, digestsize); 3272 3273 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3274 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3275 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3276 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3277 1); 3278 3279 result.err = 0; 3280 init_completion(&result.completion); 3281 result.dev = ctx->dev; 3282 3283 req_ctx->flc = flc; 3284 req_ctx->flc_dma = flc_dma; 3285 req_ctx->cbk = split_key_sh_done; 3286 req_ctx->ctx = &result; 3287 3288 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3289 if (ret == -EINPROGRESS) { 3290 /* in progress */ 3291 wait_for_completion(&result.completion); 3292 ret = result.err; 3293 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3294 DUMP_PREFIX_ADDRESS, 16, 4, key, 3295 digestsize, 1); 3296 } 3297 3298 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3299 DMA_TO_DEVICE); 3300 err_flc_dma: 3301 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3302 err_key_dma: 3303 kfree(flc); 3304 err_flc: 3305 kfree(req_ctx); 3306 3307 *keylen = digestsize; 3308 3309 return ret; 3310 } 3311 3312 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3313 unsigned int keylen) 3314 { 3315 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3316 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3317 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3318 int ret; 3319 u8 *hashed_key = NULL; 3320 3321 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3322 3323 if (keylen > blocksize) { 3324 unsigned int aligned_len = 3325 ALIGN(keylen, dma_get_cache_alignment()); 3326 3327 if (aligned_len < keylen) 3328 return -EOVERFLOW; 3329 3330 hashed_key = kmemdup(key, aligned_len, GFP_KERNEL); 3331 if (!hashed_key) 3332 return -ENOMEM; 3333 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3334 if (ret) 3335 goto bad_free_key; 3336 key = hashed_key; 3337 } 3338 3339 ctx->adata.keylen = keylen; 3340 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3341 OP_ALG_ALGSEL_MASK); 3342 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3343 goto bad_free_key; 3344 3345 ctx->adata.key_virt = key; 3346 ctx->adata.key_inline = true; 3347 3348 /* 3349 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3350 * in invalid opcodes (last bytes of user key) in the resulting 3351 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3352 * addresses are needed. 3353 */ 3354 if (keylen > ctx->adata.keylen_pad) { 3355 memcpy(ctx->key, key, keylen); 3356 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3357 ctx->adata.keylen_pad, 3358 DMA_TO_DEVICE); 3359 } 3360 3361 ret = ahash_set_sh_desc(ahash); 3362 kfree(hashed_key); 3363 return ret; 3364 bad_free_key: 3365 kfree(hashed_key); 3366 return -EINVAL; 3367 } 3368 3369 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3370 struct ahash_request *req) 3371 { 3372 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3373 3374 if (edesc->src_nents) 3375 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3376 3377 if (edesc->qm_sg_bytes) 3378 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3379 DMA_TO_DEVICE); 3380 3381 if (state->buf_dma) { 3382 dma_unmap_single(dev, state->buf_dma, state->buflen, 3383 DMA_TO_DEVICE); 3384 state->buf_dma = 0; 3385 } 3386 } 3387 3388 static inline void ahash_unmap_ctx(struct device *dev, 3389 struct ahash_edesc *edesc, 3390 struct ahash_request *req, u32 flag) 3391 { 3392 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3393 3394 if (state->ctx_dma) { 3395 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3396 state->ctx_dma = 0; 3397 } 3398 ahash_unmap(dev, edesc, req); 3399 } 3400 3401 static void ahash_done(void *cbk_ctx, u32 status) 3402 { 3403 struct crypto_async_request *areq = cbk_ctx; 3404 struct ahash_request *req = ahash_request_cast(areq); 3405 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3406 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3407 struct ahash_edesc *edesc = state->caam_req.edesc; 3408 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3409 int digestsize = crypto_ahash_digestsize(ahash); 3410 int ecode = 0; 3411 3412 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3413 3414 if (unlikely(status)) 3415 ecode = caam_qi2_strstatus(ctx->dev, status); 3416 3417 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3418 memcpy(req->result, state->caam_ctx, digestsize); 3419 qi_cache_free(edesc); 3420 3421 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3422 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3423 ctx->ctx_len, 1); 3424 3425 ahash_request_complete(req, ecode); 3426 } 3427 3428 static void ahash_done_bi(void *cbk_ctx, u32 status) 3429 { 3430 struct crypto_async_request *areq = cbk_ctx; 3431 struct ahash_request *req = ahash_request_cast(areq); 3432 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3433 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3434 struct ahash_edesc *edesc = state->caam_req.edesc; 3435 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3436 int ecode = 0; 3437 3438 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3439 3440 if (unlikely(status)) 3441 ecode = caam_qi2_strstatus(ctx->dev, status); 3442 3443 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3444 qi_cache_free(edesc); 3445 3446 scatterwalk_map_and_copy(state->buf, req->src, 3447 req->nbytes - state->next_buflen, 3448 state->next_buflen, 0); 3449 state->buflen = state->next_buflen; 3450 3451 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3452 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3453 state->buflen, 1); 3454 3455 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3456 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3457 ctx->ctx_len, 1); 3458 if (req->result) 3459 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3460 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3461 crypto_ahash_digestsize(ahash), 1); 3462 3463 ahash_request_complete(req, ecode); 3464 } 3465 3466 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3467 { 3468 struct crypto_async_request *areq = cbk_ctx; 3469 struct ahash_request *req = ahash_request_cast(areq); 3470 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3471 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3472 struct ahash_edesc *edesc = state->caam_req.edesc; 3473 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3474 int digestsize = crypto_ahash_digestsize(ahash); 3475 int ecode = 0; 3476 3477 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3478 3479 if (unlikely(status)) 3480 ecode = caam_qi2_strstatus(ctx->dev, status); 3481 3482 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3483 memcpy(req->result, state->caam_ctx, digestsize); 3484 qi_cache_free(edesc); 3485 3486 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3487 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3488 ctx->ctx_len, 1); 3489 3490 ahash_request_complete(req, ecode); 3491 } 3492 3493 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3494 { 3495 struct crypto_async_request *areq = cbk_ctx; 3496 struct ahash_request *req = ahash_request_cast(areq); 3497 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3498 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3499 struct ahash_edesc *edesc = state->caam_req.edesc; 3500 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3501 int ecode = 0; 3502 3503 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3504 3505 if (unlikely(status)) 3506 ecode = caam_qi2_strstatus(ctx->dev, status); 3507 3508 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3509 qi_cache_free(edesc); 3510 3511 scatterwalk_map_and_copy(state->buf, req->src, 3512 req->nbytes - state->next_buflen, 3513 state->next_buflen, 0); 3514 state->buflen = state->next_buflen; 3515 3516 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3517 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3518 state->buflen, 1); 3519 3520 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3521 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3522 ctx->ctx_len, 1); 3523 if (req->result) 3524 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3525 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3526 crypto_ahash_digestsize(ahash), 1); 3527 3528 ahash_request_complete(req, ecode); 3529 } 3530 3531 static int ahash_update_ctx(struct ahash_request *req) 3532 { 3533 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3534 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3535 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3536 struct caam_request *req_ctx = &state->caam_req; 3537 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3538 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3539 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3540 GFP_KERNEL : GFP_ATOMIC; 3541 u8 *buf = state->buf; 3542 int *buflen = &state->buflen; 3543 int *next_buflen = &state->next_buflen; 3544 int in_len = *buflen + req->nbytes, to_hash; 3545 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3546 struct ahash_edesc *edesc; 3547 int ret = 0; 3548 3549 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3550 to_hash = in_len - *next_buflen; 3551 3552 if (to_hash) { 3553 struct dpaa2_sg_entry *sg_table; 3554 int src_len = req->nbytes - *next_buflen; 3555 3556 src_nents = sg_nents_for_len(req->src, src_len); 3557 if (src_nents < 0) { 3558 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3559 return src_nents; 3560 } 3561 3562 if (src_nents) { 3563 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3564 DMA_TO_DEVICE); 3565 if (!mapped_nents) { 3566 dev_err(ctx->dev, "unable to DMA map source\n"); 3567 return -ENOMEM; 3568 } 3569 } else { 3570 mapped_nents = 0; 3571 } 3572 3573 /* allocate space for base edesc and link tables */ 3574 edesc = qi_cache_zalloc(flags); 3575 if (!edesc) { 3576 dma_unmap_sg(ctx->dev, req->src, src_nents, 3577 DMA_TO_DEVICE); 3578 return -ENOMEM; 3579 } 3580 3581 edesc->src_nents = src_nents; 3582 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3583 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3584 sizeof(*sg_table); 3585 sg_table = &edesc->sgt[0]; 3586 3587 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3588 DMA_BIDIRECTIONAL); 3589 if (ret) 3590 goto unmap_ctx; 3591 3592 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3593 if (ret) 3594 goto unmap_ctx; 3595 3596 if (mapped_nents) { 3597 sg_to_qm_sg_last(req->src, src_len, 3598 sg_table + qm_sg_src_index, 0); 3599 } else { 3600 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3601 true); 3602 } 3603 3604 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3605 qm_sg_bytes, DMA_TO_DEVICE); 3606 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3607 dev_err(ctx->dev, "unable to map S/G table\n"); 3608 ret = -ENOMEM; 3609 goto unmap_ctx; 3610 } 3611 edesc->qm_sg_bytes = qm_sg_bytes; 3612 3613 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3614 dpaa2_fl_set_final(in_fle, true); 3615 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3616 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3617 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3618 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3619 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3620 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3621 3622 req_ctx->flc = &ctx->flc[UPDATE]; 3623 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3624 req_ctx->cbk = ahash_done_bi; 3625 req_ctx->ctx = &req->base; 3626 req_ctx->edesc = edesc; 3627 3628 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3629 if (ret != -EINPROGRESS && 3630 !(ret == -EBUSY && 3631 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3632 goto unmap_ctx; 3633 } else if (*next_buflen) { 3634 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3635 req->nbytes, 0); 3636 *buflen = *next_buflen; 3637 3638 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3639 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3640 *buflen, 1); 3641 } 3642 3643 return ret; 3644 unmap_ctx: 3645 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3646 qi_cache_free(edesc); 3647 return ret; 3648 } 3649 3650 static int ahash_final_ctx(struct ahash_request *req) 3651 { 3652 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3653 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3654 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3655 struct caam_request *req_ctx = &state->caam_req; 3656 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3657 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3658 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3659 GFP_KERNEL : GFP_ATOMIC; 3660 int buflen = state->buflen; 3661 int qm_sg_bytes; 3662 int digestsize = crypto_ahash_digestsize(ahash); 3663 struct ahash_edesc *edesc; 3664 struct dpaa2_sg_entry *sg_table; 3665 int ret; 3666 3667 /* allocate space for base edesc and link tables */ 3668 edesc = qi_cache_zalloc(flags); 3669 if (!edesc) 3670 return -ENOMEM; 3671 3672 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3673 sg_table = &edesc->sgt[0]; 3674 3675 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3676 DMA_BIDIRECTIONAL); 3677 if (ret) 3678 goto unmap_ctx; 3679 3680 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3681 if (ret) 3682 goto unmap_ctx; 3683 3684 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3685 3686 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3687 DMA_TO_DEVICE); 3688 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3689 dev_err(ctx->dev, "unable to map S/G table\n"); 3690 ret = -ENOMEM; 3691 goto unmap_ctx; 3692 } 3693 edesc->qm_sg_bytes = qm_sg_bytes; 3694 3695 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3696 dpaa2_fl_set_final(in_fle, true); 3697 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3698 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3699 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3700 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3701 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3702 dpaa2_fl_set_len(out_fle, digestsize); 3703 3704 req_ctx->flc = &ctx->flc[FINALIZE]; 3705 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3706 req_ctx->cbk = ahash_done_ctx_src; 3707 req_ctx->ctx = &req->base; 3708 req_ctx->edesc = edesc; 3709 3710 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3711 if (ret == -EINPROGRESS || 3712 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3713 return ret; 3714 3715 unmap_ctx: 3716 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3717 qi_cache_free(edesc); 3718 return ret; 3719 } 3720 3721 static int ahash_finup_ctx(struct ahash_request *req) 3722 { 3723 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3724 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3725 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3726 struct caam_request *req_ctx = &state->caam_req; 3727 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3728 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3729 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3730 GFP_KERNEL : GFP_ATOMIC; 3731 int buflen = state->buflen; 3732 int qm_sg_bytes, qm_sg_src_index; 3733 int src_nents, mapped_nents; 3734 int digestsize = crypto_ahash_digestsize(ahash); 3735 struct ahash_edesc *edesc; 3736 struct dpaa2_sg_entry *sg_table; 3737 int ret; 3738 3739 src_nents = sg_nents_for_len(req->src, req->nbytes); 3740 if (src_nents < 0) { 3741 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3742 return src_nents; 3743 } 3744 3745 if (src_nents) { 3746 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3747 DMA_TO_DEVICE); 3748 if (!mapped_nents) { 3749 dev_err(ctx->dev, "unable to DMA map source\n"); 3750 return -ENOMEM; 3751 } 3752 } else { 3753 mapped_nents = 0; 3754 } 3755 3756 /* allocate space for base edesc and link tables */ 3757 edesc = qi_cache_zalloc(flags); 3758 if (!edesc) { 3759 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3760 return -ENOMEM; 3761 } 3762 3763 edesc->src_nents = src_nents; 3764 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3765 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3766 sizeof(*sg_table); 3767 sg_table = &edesc->sgt[0]; 3768 3769 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3770 DMA_BIDIRECTIONAL); 3771 if (ret) 3772 goto unmap_ctx; 3773 3774 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3775 if (ret) 3776 goto unmap_ctx; 3777 3778 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3779 3780 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3781 DMA_TO_DEVICE); 3782 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3783 dev_err(ctx->dev, "unable to map S/G table\n"); 3784 ret = -ENOMEM; 3785 goto unmap_ctx; 3786 } 3787 edesc->qm_sg_bytes = qm_sg_bytes; 3788 3789 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3790 dpaa2_fl_set_final(in_fle, true); 3791 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3792 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3793 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3794 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3795 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3796 dpaa2_fl_set_len(out_fle, digestsize); 3797 3798 req_ctx->flc = &ctx->flc[FINALIZE]; 3799 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3800 req_ctx->cbk = ahash_done_ctx_src; 3801 req_ctx->ctx = &req->base; 3802 req_ctx->edesc = edesc; 3803 3804 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3805 if (ret == -EINPROGRESS || 3806 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3807 return ret; 3808 3809 unmap_ctx: 3810 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3811 qi_cache_free(edesc); 3812 return ret; 3813 } 3814 3815 static int ahash_digest(struct ahash_request *req) 3816 { 3817 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3818 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3819 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3820 struct caam_request *req_ctx = &state->caam_req; 3821 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3822 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3823 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3824 GFP_KERNEL : GFP_ATOMIC; 3825 int digestsize = crypto_ahash_digestsize(ahash); 3826 int src_nents, mapped_nents; 3827 struct ahash_edesc *edesc; 3828 int ret = -ENOMEM; 3829 3830 state->buf_dma = 0; 3831 3832 src_nents = sg_nents_for_len(req->src, req->nbytes); 3833 if (src_nents < 0) { 3834 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3835 return src_nents; 3836 } 3837 3838 if (src_nents) { 3839 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3840 DMA_TO_DEVICE); 3841 if (!mapped_nents) { 3842 dev_err(ctx->dev, "unable to map source for DMA\n"); 3843 return ret; 3844 } 3845 } else { 3846 mapped_nents = 0; 3847 } 3848 3849 /* allocate space for base edesc and link tables */ 3850 edesc = qi_cache_zalloc(flags); 3851 if (!edesc) { 3852 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3853 return ret; 3854 } 3855 3856 edesc->src_nents = src_nents; 3857 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3858 3859 if (mapped_nents > 1) { 3860 int qm_sg_bytes; 3861 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3862 3863 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3864 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3865 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3866 qm_sg_bytes, DMA_TO_DEVICE); 3867 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3868 dev_err(ctx->dev, "unable to map S/G table\n"); 3869 goto unmap; 3870 } 3871 edesc->qm_sg_bytes = qm_sg_bytes; 3872 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3873 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3874 } else { 3875 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3876 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3877 } 3878 3879 state->ctx_dma_len = digestsize; 3880 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3881 DMA_FROM_DEVICE); 3882 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3883 dev_err(ctx->dev, "unable to map ctx\n"); 3884 state->ctx_dma = 0; 3885 goto unmap; 3886 } 3887 3888 dpaa2_fl_set_final(in_fle, true); 3889 dpaa2_fl_set_len(in_fle, req->nbytes); 3890 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3891 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3892 dpaa2_fl_set_len(out_fle, digestsize); 3893 3894 req_ctx->flc = &ctx->flc[DIGEST]; 3895 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3896 req_ctx->cbk = ahash_done; 3897 req_ctx->ctx = &req->base; 3898 req_ctx->edesc = edesc; 3899 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3900 if (ret == -EINPROGRESS || 3901 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3902 return ret; 3903 3904 unmap: 3905 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3906 qi_cache_free(edesc); 3907 return ret; 3908 } 3909 3910 static int ahash_final_no_ctx(struct ahash_request *req) 3911 { 3912 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3913 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3914 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3915 struct caam_request *req_ctx = &state->caam_req; 3916 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3917 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3918 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3919 GFP_KERNEL : GFP_ATOMIC; 3920 u8 *buf = state->buf; 3921 int buflen = state->buflen; 3922 int digestsize = crypto_ahash_digestsize(ahash); 3923 struct ahash_edesc *edesc; 3924 int ret = -ENOMEM; 3925 3926 /* allocate space for base edesc and link tables */ 3927 edesc = qi_cache_zalloc(flags); 3928 if (!edesc) 3929 return ret; 3930 3931 if (buflen) { 3932 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3933 DMA_TO_DEVICE); 3934 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3935 dev_err(ctx->dev, "unable to map src\n"); 3936 goto unmap; 3937 } 3938 } 3939 3940 state->ctx_dma_len = digestsize; 3941 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3942 DMA_FROM_DEVICE); 3943 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3944 dev_err(ctx->dev, "unable to map ctx\n"); 3945 state->ctx_dma = 0; 3946 goto unmap; 3947 } 3948 3949 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3950 dpaa2_fl_set_final(in_fle, true); 3951 /* 3952 * crypto engine requires the input entry to be present when 3953 * "frame list" FD is used. 3954 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3955 * in_fle zeroized (except for "Final" flag) is the best option. 3956 */ 3957 if (buflen) { 3958 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3959 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3960 dpaa2_fl_set_len(in_fle, buflen); 3961 } 3962 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3963 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3964 dpaa2_fl_set_len(out_fle, digestsize); 3965 3966 req_ctx->flc = &ctx->flc[DIGEST]; 3967 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3968 req_ctx->cbk = ahash_done; 3969 req_ctx->ctx = &req->base; 3970 req_ctx->edesc = edesc; 3971 3972 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3973 if (ret == -EINPROGRESS || 3974 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3975 return ret; 3976 3977 unmap: 3978 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3979 qi_cache_free(edesc); 3980 return ret; 3981 } 3982 3983 static int ahash_update_no_ctx(struct ahash_request *req) 3984 { 3985 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3986 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3987 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3988 struct caam_request *req_ctx = &state->caam_req; 3989 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3990 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3991 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3992 GFP_KERNEL : GFP_ATOMIC; 3993 u8 *buf = state->buf; 3994 int *buflen = &state->buflen; 3995 int *next_buflen = &state->next_buflen; 3996 int in_len = *buflen + req->nbytes, to_hash; 3997 int qm_sg_bytes, src_nents, mapped_nents; 3998 struct ahash_edesc *edesc; 3999 int ret = 0; 4000 4001 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 4002 to_hash = in_len - *next_buflen; 4003 4004 if (to_hash) { 4005 struct dpaa2_sg_entry *sg_table; 4006 int src_len = req->nbytes - *next_buflen; 4007 4008 src_nents = sg_nents_for_len(req->src, src_len); 4009 if (src_nents < 0) { 4010 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4011 return src_nents; 4012 } 4013 4014 if (src_nents) { 4015 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4016 DMA_TO_DEVICE); 4017 if (!mapped_nents) { 4018 dev_err(ctx->dev, "unable to DMA map source\n"); 4019 return -ENOMEM; 4020 } 4021 } else { 4022 mapped_nents = 0; 4023 } 4024 4025 /* allocate space for base edesc and link tables */ 4026 edesc = qi_cache_zalloc(flags); 4027 if (!edesc) { 4028 dma_unmap_sg(ctx->dev, req->src, src_nents, 4029 DMA_TO_DEVICE); 4030 return -ENOMEM; 4031 } 4032 4033 edesc->src_nents = src_nents; 4034 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 4035 sizeof(*sg_table); 4036 sg_table = &edesc->sgt[0]; 4037 4038 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4039 if (ret) 4040 goto unmap_ctx; 4041 4042 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 4043 4044 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4045 qm_sg_bytes, DMA_TO_DEVICE); 4046 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4047 dev_err(ctx->dev, "unable to map S/G table\n"); 4048 ret = -ENOMEM; 4049 goto unmap_ctx; 4050 } 4051 edesc->qm_sg_bytes = qm_sg_bytes; 4052 4053 state->ctx_dma_len = ctx->ctx_len; 4054 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4055 ctx->ctx_len, DMA_FROM_DEVICE); 4056 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4057 dev_err(ctx->dev, "unable to map ctx\n"); 4058 state->ctx_dma = 0; 4059 ret = -ENOMEM; 4060 goto unmap_ctx; 4061 } 4062 4063 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4064 dpaa2_fl_set_final(in_fle, true); 4065 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4066 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4067 dpaa2_fl_set_len(in_fle, to_hash); 4068 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4069 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4070 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4071 4072 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4073 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4074 req_ctx->cbk = ahash_done_ctx_dst; 4075 req_ctx->ctx = &req->base; 4076 req_ctx->edesc = edesc; 4077 4078 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4079 if (ret != -EINPROGRESS && 4080 !(ret == -EBUSY && 4081 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4082 goto unmap_ctx; 4083 4084 state->update = ahash_update_ctx; 4085 state->finup = ahash_finup_ctx; 4086 state->final = ahash_final_ctx; 4087 } else if (*next_buflen) { 4088 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 4089 req->nbytes, 0); 4090 *buflen = *next_buflen; 4091 4092 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4093 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4094 *buflen, 1); 4095 } 4096 4097 return ret; 4098 unmap_ctx: 4099 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4100 qi_cache_free(edesc); 4101 return ret; 4102 } 4103 4104 static int ahash_finup_no_ctx(struct ahash_request *req) 4105 { 4106 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4107 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4108 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4109 struct caam_request *req_ctx = &state->caam_req; 4110 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4111 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4112 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4113 GFP_KERNEL : GFP_ATOMIC; 4114 int buflen = state->buflen; 4115 int qm_sg_bytes, src_nents, mapped_nents; 4116 int digestsize = crypto_ahash_digestsize(ahash); 4117 struct ahash_edesc *edesc; 4118 struct dpaa2_sg_entry *sg_table; 4119 int ret = -ENOMEM; 4120 4121 src_nents = sg_nents_for_len(req->src, req->nbytes); 4122 if (src_nents < 0) { 4123 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4124 return src_nents; 4125 } 4126 4127 if (src_nents) { 4128 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4129 DMA_TO_DEVICE); 4130 if (!mapped_nents) { 4131 dev_err(ctx->dev, "unable to DMA map source\n"); 4132 return ret; 4133 } 4134 } else { 4135 mapped_nents = 0; 4136 } 4137 4138 /* allocate space for base edesc and link tables */ 4139 edesc = qi_cache_zalloc(flags); 4140 if (!edesc) { 4141 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4142 return ret; 4143 } 4144 4145 edesc->src_nents = src_nents; 4146 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4147 sg_table = &edesc->sgt[0]; 4148 4149 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4150 if (ret) 4151 goto unmap; 4152 4153 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4154 4155 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4156 DMA_TO_DEVICE); 4157 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4158 dev_err(ctx->dev, "unable to map S/G table\n"); 4159 ret = -ENOMEM; 4160 goto unmap; 4161 } 4162 edesc->qm_sg_bytes = qm_sg_bytes; 4163 4164 state->ctx_dma_len = digestsize; 4165 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4166 DMA_FROM_DEVICE); 4167 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4168 dev_err(ctx->dev, "unable to map ctx\n"); 4169 state->ctx_dma = 0; 4170 ret = -ENOMEM; 4171 goto unmap; 4172 } 4173 4174 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4175 dpaa2_fl_set_final(in_fle, true); 4176 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4177 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4178 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4179 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4180 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4181 dpaa2_fl_set_len(out_fle, digestsize); 4182 4183 req_ctx->flc = &ctx->flc[DIGEST]; 4184 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4185 req_ctx->cbk = ahash_done; 4186 req_ctx->ctx = &req->base; 4187 req_ctx->edesc = edesc; 4188 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4189 if (ret != -EINPROGRESS && 4190 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4191 goto unmap; 4192 4193 return ret; 4194 unmap: 4195 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4196 qi_cache_free(edesc); 4197 return ret; 4198 } 4199 4200 static int ahash_update_first(struct ahash_request *req) 4201 { 4202 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4203 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4204 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4205 struct caam_request *req_ctx = &state->caam_req; 4206 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4207 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4208 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4209 GFP_KERNEL : GFP_ATOMIC; 4210 u8 *buf = state->buf; 4211 int *buflen = &state->buflen; 4212 int *next_buflen = &state->next_buflen; 4213 int to_hash; 4214 int src_nents, mapped_nents; 4215 struct ahash_edesc *edesc; 4216 int ret = 0; 4217 4218 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4219 1); 4220 to_hash = req->nbytes - *next_buflen; 4221 4222 if (to_hash) { 4223 struct dpaa2_sg_entry *sg_table; 4224 int src_len = req->nbytes - *next_buflen; 4225 4226 src_nents = sg_nents_for_len(req->src, src_len); 4227 if (src_nents < 0) { 4228 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4229 return src_nents; 4230 } 4231 4232 if (src_nents) { 4233 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4234 DMA_TO_DEVICE); 4235 if (!mapped_nents) { 4236 dev_err(ctx->dev, "unable to map source for DMA\n"); 4237 return -ENOMEM; 4238 } 4239 } else { 4240 mapped_nents = 0; 4241 } 4242 4243 /* allocate space for base edesc and link tables */ 4244 edesc = qi_cache_zalloc(flags); 4245 if (!edesc) { 4246 dma_unmap_sg(ctx->dev, req->src, src_nents, 4247 DMA_TO_DEVICE); 4248 return -ENOMEM; 4249 } 4250 4251 edesc->src_nents = src_nents; 4252 sg_table = &edesc->sgt[0]; 4253 4254 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4255 dpaa2_fl_set_final(in_fle, true); 4256 dpaa2_fl_set_len(in_fle, to_hash); 4257 4258 if (mapped_nents > 1) { 4259 int qm_sg_bytes; 4260 4261 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4262 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4263 sizeof(*sg_table); 4264 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4265 qm_sg_bytes, 4266 DMA_TO_DEVICE); 4267 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4268 dev_err(ctx->dev, "unable to map S/G table\n"); 4269 ret = -ENOMEM; 4270 goto unmap_ctx; 4271 } 4272 edesc->qm_sg_bytes = qm_sg_bytes; 4273 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4274 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4275 } else { 4276 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4277 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4278 } 4279 4280 state->ctx_dma_len = ctx->ctx_len; 4281 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4282 ctx->ctx_len, DMA_FROM_DEVICE); 4283 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4284 dev_err(ctx->dev, "unable to map ctx\n"); 4285 state->ctx_dma = 0; 4286 ret = -ENOMEM; 4287 goto unmap_ctx; 4288 } 4289 4290 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4291 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4292 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4293 4294 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4295 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4296 req_ctx->cbk = ahash_done_ctx_dst; 4297 req_ctx->ctx = &req->base; 4298 req_ctx->edesc = edesc; 4299 4300 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4301 if (ret != -EINPROGRESS && 4302 !(ret == -EBUSY && req->base.flags & 4303 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4304 goto unmap_ctx; 4305 4306 state->update = ahash_update_ctx; 4307 state->finup = ahash_finup_ctx; 4308 state->final = ahash_final_ctx; 4309 } else if (*next_buflen) { 4310 state->update = ahash_update_no_ctx; 4311 state->finup = ahash_finup_no_ctx; 4312 state->final = ahash_final_no_ctx; 4313 scatterwalk_map_and_copy(buf, req->src, 0, 4314 req->nbytes, 0); 4315 *buflen = *next_buflen; 4316 4317 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4318 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4319 *buflen, 1); 4320 } 4321 4322 return ret; 4323 unmap_ctx: 4324 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4325 qi_cache_free(edesc); 4326 return ret; 4327 } 4328 4329 static int ahash_finup_first(struct ahash_request *req) 4330 { 4331 return ahash_digest(req); 4332 } 4333 4334 static int ahash_init(struct ahash_request *req) 4335 { 4336 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4337 4338 state->update = ahash_update_first; 4339 state->finup = ahash_finup_first; 4340 state->final = ahash_final_no_ctx; 4341 4342 state->ctx_dma = 0; 4343 state->ctx_dma_len = 0; 4344 state->buf_dma = 0; 4345 state->buflen = 0; 4346 state->next_buflen = 0; 4347 4348 return 0; 4349 } 4350 4351 static int ahash_update(struct ahash_request *req) 4352 { 4353 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4354 4355 return state->update(req); 4356 } 4357 4358 static int ahash_finup(struct ahash_request *req) 4359 { 4360 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4361 4362 return state->finup(req); 4363 } 4364 4365 static int ahash_final(struct ahash_request *req) 4366 { 4367 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4368 4369 return state->final(req); 4370 } 4371 4372 static int ahash_export(struct ahash_request *req, void *out) 4373 { 4374 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4375 struct caam_export_state *export = out; 4376 u8 *buf = state->buf; 4377 int len = state->buflen; 4378 4379 memcpy(export->buf, buf, len); 4380 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4381 export->buflen = len; 4382 export->update = state->update; 4383 export->final = state->final; 4384 export->finup = state->finup; 4385 4386 return 0; 4387 } 4388 4389 static int ahash_import(struct ahash_request *req, const void *in) 4390 { 4391 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4392 const struct caam_export_state *export = in; 4393 4394 memset(state, 0, sizeof(*state)); 4395 memcpy(state->buf, export->buf, export->buflen); 4396 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4397 state->buflen = export->buflen; 4398 state->update = export->update; 4399 state->final = export->final; 4400 state->finup = export->finup; 4401 4402 return 0; 4403 } 4404 4405 struct caam_hash_template { 4406 char name[CRYPTO_MAX_ALG_NAME]; 4407 char driver_name[CRYPTO_MAX_ALG_NAME]; 4408 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4409 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4410 unsigned int blocksize; 4411 struct ahash_alg template_ahash; 4412 u32 alg_type; 4413 }; 4414 4415 /* ahash descriptors */ 4416 static struct caam_hash_template driver_hash[] = { 4417 { 4418 .name = "sha1", 4419 .driver_name = "sha1-caam-qi2", 4420 .hmac_name = "hmac(sha1)", 4421 .hmac_driver_name = "hmac-sha1-caam-qi2", 4422 .blocksize = SHA1_BLOCK_SIZE, 4423 .template_ahash = { 4424 .init = ahash_init, 4425 .update = ahash_update, 4426 .final = ahash_final, 4427 .finup = ahash_finup, 4428 .digest = ahash_digest, 4429 .export = ahash_export, 4430 .import = ahash_import, 4431 .setkey = ahash_setkey, 4432 .halg = { 4433 .digestsize = SHA1_DIGEST_SIZE, 4434 .statesize = sizeof(struct caam_export_state), 4435 }, 4436 }, 4437 .alg_type = OP_ALG_ALGSEL_SHA1, 4438 }, { 4439 .name = "sha224", 4440 .driver_name = "sha224-caam-qi2", 4441 .hmac_name = "hmac(sha224)", 4442 .hmac_driver_name = "hmac-sha224-caam-qi2", 4443 .blocksize = SHA224_BLOCK_SIZE, 4444 .template_ahash = { 4445 .init = ahash_init, 4446 .update = ahash_update, 4447 .final = ahash_final, 4448 .finup = ahash_finup, 4449 .digest = ahash_digest, 4450 .export = ahash_export, 4451 .import = ahash_import, 4452 .setkey = ahash_setkey, 4453 .halg = { 4454 .digestsize = SHA224_DIGEST_SIZE, 4455 .statesize = sizeof(struct caam_export_state), 4456 }, 4457 }, 4458 .alg_type = OP_ALG_ALGSEL_SHA224, 4459 }, { 4460 .name = "sha256", 4461 .driver_name = "sha256-caam-qi2", 4462 .hmac_name = "hmac(sha256)", 4463 .hmac_driver_name = "hmac-sha256-caam-qi2", 4464 .blocksize = SHA256_BLOCK_SIZE, 4465 .template_ahash = { 4466 .init = ahash_init, 4467 .update = ahash_update, 4468 .final = ahash_final, 4469 .finup = ahash_finup, 4470 .digest = ahash_digest, 4471 .export = ahash_export, 4472 .import = ahash_import, 4473 .setkey = ahash_setkey, 4474 .halg = { 4475 .digestsize = SHA256_DIGEST_SIZE, 4476 .statesize = sizeof(struct caam_export_state), 4477 }, 4478 }, 4479 .alg_type = OP_ALG_ALGSEL_SHA256, 4480 }, { 4481 .name = "sha384", 4482 .driver_name = "sha384-caam-qi2", 4483 .hmac_name = "hmac(sha384)", 4484 .hmac_driver_name = "hmac-sha384-caam-qi2", 4485 .blocksize = SHA384_BLOCK_SIZE, 4486 .template_ahash = { 4487 .init = ahash_init, 4488 .update = ahash_update, 4489 .final = ahash_final, 4490 .finup = ahash_finup, 4491 .digest = ahash_digest, 4492 .export = ahash_export, 4493 .import = ahash_import, 4494 .setkey = ahash_setkey, 4495 .halg = { 4496 .digestsize = SHA384_DIGEST_SIZE, 4497 .statesize = sizeof(struct caam_export_state), 4498 }, 4499 }, 4500 .alg_type = OP_ALG_ALGSEL_SHA384, 4501 }, { 4502 .name = "sha512", 4503 .driver_name = "sha512-caam-qi2", 4504 .hmac_name = "hmac(sha512)", 4505 .hmac_driver_name = "hmac-sha512-caam-qi2", 4506 .blocksize = SHA512_BLOCK_SIZE, 4507 .template_ahash = { 4508 .init = ahash_init, 4509 .update = ahash_update, 4510 .final = ahash_final, 4511 .finup = ahash_finup, 4512 .digest = ahash_digest, 4513 .export = ahash_export, 4514 .import = ahash_import, 4515 .setkey = ahash_setkey, 4516 .halg = { 4517 .digestsize = SHA512_DIGEST_SIZE, 4518 .statesize = sizeof(struct caam_export_state), 4519 }, 4520 }, 4521 .alg_type = OP_ALG_ALGSEL_SHA512, 4522 }, { 4523 .name = "md5", 4524 .driver_name = "md5-caam-qi2", 4525 .hmac_name = "hmac(md5)", 4526 .hmac_driver_name = "hmac-md5-caam-qi2", 4527 .blocksize = MD5_BLOCK_WORDS * 4, 4528 .template_ahash = { 4529 .init = ahash_init, 4530 .update = ahash_update, 4531 .final = ahash_final, 4532 .finup = ahash_finup, 4533 .digest = ahash_digest, 4534 .export = ahash_export, 4535 .import = ahash_import, 4536 .setkey = ahash_setkey, 4537 .halg = { 4538 .digestsize = MD5_DIGEST_SIZE, 4539 .statesize = sizeof(struct caam_export_state), 4540 }, 4541 }, 4542 .alg_type = OP_ALG_ALGSEL_MD5, 4543 } 4544 }; 4545 4546 struct caam_hash_alg { 4547 struct list_head entry; 4548 struct device *dev; 4549 int alg_type; 4550 bool is_hmac; 4551 struct ahash_alg ahash_alg; 4552 }; 4553 4554 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4555 { 4556 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4557 struct crypto_alg *base = tfm->__crt_alg; 4558 struct hash_alg_common *halg = 4559 container_of(base, struct hash_alg_common, base); 4560 struct ahash_alg *alg = 4561 container_of(halg, struct ahash_alg, halg); 4562 struct caam_hash_alg *caam_hash = 4563 container_of(alg, struct caam_hash_alg, ahash_alg); 4564 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4565 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4566 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4567 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4568 HASH_MSG_LEN + 32, 4569 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4570 HASH_MSG_LEN + 64, 4571 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4572 dma_addr_t dma_addr; 4573 int i; 4574 4575 ctx->dev = caam_hash->dev; 4576 4577 if (caam_hash->is_hmac) { 4578 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4579 ARRAY_SIZE(ctx->key), 4580 DMA_TO_DEVICE, 4581 DMA_ATTR_SKIP_CPU_SYNC); 4582 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4583 dev_err(ctx->dev, "unable to map key\n"); 4584 return -ENOMEM; 4585 } 4586 } 4587 4588 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4589 DMA_BIDIRECTIONAL, 4590 DMA_ATTR_SKIP_CPU_SYNC); 4591 if (dma_mapping_error(ctx->dev, dma_addr)) { 4592 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4593 if (ctx->adata.key_dma) 4594 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4595 ARRAY_SIZE(ctx->key), 4596 DMA_TO_DEVICE, 4597 DMA_ATTR_SKIP_CPU_SYNC); 4598 return -ENOMEM; 4599 } 4600 4601 for (i = 0; i < HASH_NUM_OP; i++) 4602 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4603 4604 /* copy descriptor header template value */ 4605 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4606 4607 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4608 OP_ALG_ALGSEL_SUBMASK) >> 4609 OP_ALG_ALGSEL_SHIFT]; 4610 4611 crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state)); 4612 4613 /* 4614 * For keyed hash algorithms shared descriptors 4615 * will be created later in setkey() callback 4616 */ 4617 return caam_hash->is_hmac ? 0 : ahash_set_sh_desc(ahash); 4618 } 4619 4620 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4621 { 4622 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4623 4624 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4625 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4626 if (ctx->adata.key_dma) 4627 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4628 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4629 DMA_ATTR_SKIP_CPU_SYNC); 4630 } 4631 4632 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4633 struct caam_hash_template *template, bool keyed) 4634 { 4635 struct caam_hash_alg *t_alg; 4636 struct ahash_alg *halg; 4637 struct crypto_alg *alg; 4638 4639 t_alg = kzalloc_obj(*t_alg); 4640 if (!t_alg) 4641 return ERR_PTR(-ENOMEM); 4642 4643 t_alg->ahash_alg = template->template_ahash; 4644 halg = &t_alg->ahash_alg; 4645 alg = &halg->halg.base; 4646 4647 if (keyed) { 4648 strscpy(alg->cra_name, template->hmac_name); 4649 strscpy(alg->cra_driver_name, template->hmac_driver_name); 4650 t_alg->is_hmac = true; 4651 } else { 4652 strscpy(alg->cra_name, template->name); 4653 strscpy(alg->cra_driver_name, template->driver_name); 4654 t_alg->ahash_alg.setkey = NULL; 4655 t_alg->is_hmac = false; 4656 } 4657 alg->cra_module = THIS_MODULE; 4658 alg->cra_init = caam_hash_cra_init; 4659 alg->cra_exit = caam_hash_cra_exit; 4660 alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding(); 4661 alg->cra_priority = CAAM_CRA_PRIORITY; 4662 alg->cra_blocksize = template->blocksize; 4663 alg->cra_alignmask = 0; 4664 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4665 4666 t_alg->alg_type = template->alg_type; 4667 t_alg->dev = dev; 4668 4669 return t_alg; 4670 } 4671 4672 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4673 { 4674 struct dpaa2_caam_priv_per_cpu *ppriv; 4675 4676 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4677 napi_schedule_irqoff(&ppriv->napi); 4678 } 4679 4680 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4681 { 4682 struct device *dev = priv->dev; 4683 struct dpaa2_io_notification_ctx *nctx; 4684 struct dpaa2_caam_priv_per_cpu *ppriv; 4685 int err, i = 0, cpu; 4686 4687 for_each_online_cpu(cpu) { 4688 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4689 ppriv->priv = priv; 4690 nctx = &ppriv->nctx; 4691 nctx->is_cdan = 0; 4692 nctx->id = ppriv->rsp_fqid; 4693 nctx->desired_cpu = cpu; 4694 nctx->cb = dpaa2_caam_fqdan_cb; 4695 4696 /* Register notification callbacks */ 4697 ppriv->dpio = dpaa2_io_service_select(cpu); 4698 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4699 if (unlikely(err)) { 4700 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4701 nctx->cb = NULL; 4702 /* 4703 * If no affine DPIO for this core, there's probably 4704 * none available for next cores either. Signal we want 4705 * to retry later, in case the DPIO devices weren't 4706 * probed yet. 4707 */ 4708 err = -EPROBE_DEFER; 4709 goto err; 4710 } 4711 4712 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4713 dev); 4714 if (unlikely(!ppriv->store)) { 4715 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4716 err = -ENOMEM; 4717 goto err; 4718 } 4719 4720 if (++i == priv->num_pairs) 4721 break; 4722 } 4723 4724 return 0; 4725 4726 err: 4727 for_each_online_cpu(cpu) { 4728 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4729 if (!ppriv->nctx.cb) 4730 break; 4731 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4732 } 4733 4734 for_each_online_cpu(cpu) { 4735 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4736 if (!ppriv->store) 4737 break; 4738 dpaa2_io_store_destroy(ppriv->store); 4739 } 4740 4741 return err; 4742 } 4743 4744 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4745 { 4746 struct dpaa2_caam_priv_per_cpu *ppriv; 4747 int i = 0, cpu; 4748 4749 for_each_online_cpu(cpu) { 4750 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4751 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4752 priv->dev); 4753 dpaa2_io_store_destroy(ppriv->store); 4754 4755 if (++i == priv->num_pairs) 4756 return; 4757 } 4758 } 4759 4760 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4761 { 4762 struct dpseci_rx_queue_cfg rx_queue_cfg; 4763 struct device *dev = priv->dev; 4764 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4765 struct dpaa2_caam_priv_per_cpu *ppriv; 4766 int err = 0, i = 0, cpu; 4767 4768 /* Configure Rx queues */ 4769 for_each_online_cpu(cpu) { 4770 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4771 4772 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4773 DPSECI_QUEUE_OPT_USER_CTX; 4774 rx_queue_cfg.order_preservation_en = 0; 4775 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4776 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4777 /* 4778 * Rx priority (WQ) doesn't really matter, since we use 4779 * pull mode, i.e. volatile dequeues from specific FQs 4780 */ 4781 rx_queue_cfg.dest_cfg.priority = 0; 4782 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4783 4784 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4785 &rx_queue_cfg); 4786 if (err) { 4787 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4788 err); 4789 return err; 4790 } 4791 4792 if (++i == priv->num_pairs) 4793 break; 4794 } 4795 4796 return err; 4797 } 4798 4799 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4800 { 4801 struct device *dev = priv->dev; 4802 4803 if (!priv->cscn_mem) 4804 return; 4805 4806 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4807 kfree(priv->cscn_mem); 4808 } 4809 4810 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4811 { 4812 struct device *dev = priv->dev; 4813 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4814 struct dpaa2_caam_priv_per_cpu *ppriv; 4815 int i, err; 4816 4817 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 4818 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 4819 if (err) 4820 dev_err(dev, "dpseci_reset() failed\n"); 4821 } 4822 4823 for_each_cpu(i, priv->clean_mask) { 4824 ppriv = per_cpu_ptr(priv->ppriv, i); 4825 free_netdev(ppriv->net_dev); 4826 } 4827 free_cpumask_var(priv->clean_mask); 4828 4829 dpaa2_dpseci_congestion_free(priv); 4830 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4831 } 4832 4833 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4834 const struct dpaa2_fd *fd) 4835 { 4836 struct caam_request *req; 4837 u32 fd_err; 4838 4839 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4840 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4841 return; 4842 } 4843 4844 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4845 if (unlikely(fd_err)) 4846 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4847 4848 /* 4849 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4850 * in FD[ERR] or FD[FRC]. 4851 */ 4852 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4853 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4854 DMA_BIDIRECTIONAL); 4855 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4856 } 4857 4858 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4859 { 4860 int err; 4861 4862 /* Retry while portal is busy */ 4863 do { 4864 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4865 ppriv->store); 4866 } while (err == -EBUSY); 4867 4868 if (unlikely(err)) 4869 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4870 4871 return err; 4872 } 4873 4874 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4875 { 4876 struct dpaa2_dq *dq; 4877 int cleaned = 0, is_last; 4878 4879 do { 4880 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4881 if (unlikely(!dq)) { 4882 if (unlikely(!is_last)) { 4883 dev_dbg(ppriv->priv->dev, 4884 "FQ %d returned no valid frames\n", 4885 ppriv->rsp_fqid); 4886 /* 4887 * MUST retry until we get some sort of 4888 * valid response token (be it "empty dequeue" 4889 * or a valid frame). 4890 */ 4891 continue; 4892 } 4893 break; 4894 } 4895 4896 /* Process FD */ 4897 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4898 cleaned++; 4899 } while (!is_last); 4900 4901 return cleaned; 4902 } 4903 4904 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4905 { 4906 struct dpaa2_caam_priv_per_cpu *ppriv; 4907 struct dpaa2_caam_priv *priv; 4908 int err, cleaned = 0, store_cleaned; 4909 4910 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4911 priv = ppriv->priv; 4912 4913 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4914 return 0; 4915 4916 do { 4917 store_cleaned = dpaa2_caam_store_consume(ppriv); 4918 cleaned += store_cleaned; 4919 4920 if (store_cleaned == 0 || 4921 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4922 break; 4923 4924 /* Try to dequeue some more */ 4925 err = dpaa2_caam_pull_fq(ppriv); 4926 if (unlikely(err)) 4927 break; 4928 } while (1); 4929 4930 if (cleaned < budget) { 4931 napi_complete_done(napi, cleaned); 4932 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4933 if (unlikely(err)) 4934 dev_err(priv->dev, "Notification rearm failed: %d\n", 4935 err); 4936 } 4937 4938 return cleaned; 4939 } 4940 4941 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4942 u16 token) 4943 { 4944 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4945 struct device *dev = priv->dev; 4946 unsigned int alignmask; 4947 int err; 4948 4949 /* 4950 * Congestion group feature supported starting with DPSECI API v5.1 4951 * and only when object has been created with this capability. 4952 */ 4953 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4954 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4955 return 0; 4956 4957 alignmask = DPAA2_CSCN_ALIGN - 1; 4958 alignmask |= dma_get_cache_alignment() - 1; 4959 priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1), 4960 GFP_KERNEL); 4961 if (!priv->cscn_mem) 4962 return -ENOMEM; 4963 4964 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem, 4965 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4966 if (dma_mapping_error(dev, priv->cscn_dma)) { 4967 dev_err(dev, "Error mapping CSCN memory area\n"); 4968 err = -ENOMEM; 4969 goto err_dma_map; 4970 } 4971 4972 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4973 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4974 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4975 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4976 cong_notif_cfg.message_iova = priv->cscn_dma; 4977 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4978 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4979 DPSECI_CGN_MODE_COHERENT_WRITE; 4980 4981 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4982 &cong_notif_cfg); 4983 if (err) { 4984 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4985 goto err_set_cong; 4986 } 4987 4988 return 0; 4989 4990 err_set_cong: 4991 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4992 err_dma_map: 4993 kfree(priv->cscn_mem); 4994 4995 return err; 4996 } 4997 4998 static void free_dpaa2_pcpu_netdev(struct dpaa2_caam_priv *priv, const cpumask_t *cpus) 4999 { 5000 struct dpaa2_caam_priv_per_cpu *ppriv; 5001 int i; 5002 5003 for_each_cpu(i, cpus) { 5004 ppriv = per_cpu_ptr(priv->ppriv, i); 5005 free_netdev(ppriv->net_dev); 5006 } 5007 } 5008 5009 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 5010 { 5011 struct device *dev = &ls_dev->dev; 5012 struct dpaa2_caam_priv *priv; 5013 struct dpaa2_caam_priv_per_cpu *ppriv; 5014 int err, cpu; 5015 u8 i; 5016 5017 err = -ENOMEM; 5018 priv = dev_get_drvdata(dev); 5019 5020 if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) 5021 goto err_cpumask; 5022 5023 priv->dev = dev; 5024 priv->dpsec_id = ls_dev->obj_desc.id; 5025 5026 /* Get a handle for the DPSECI this interface is associate with */ 5027 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 5028 if (err) { 5029 dev_err(dev, "dpseci_open() failed: %d\n", err); 5030 goto err_open; 5031 } 5032 5033 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 5034 &priv->minor_ver); 5035 if (err) { 5036 dev_err(dev, "dpseci_get_api_version() failed\n"); 5037 goto err_get_vers; 5038 } 5039 5040 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 5041 5042 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 5043 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 5044 if (err) { 5045 dev_err(dev, "dpseci_reset() failed\n"); 5046 goto err_get_vers; 5047 } 5048 } 5049 5050 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 5051 &priv->dpseci_attr); 5052 if (err) { 5053 dev_err(dev, "dpseci_get_attributes() failed\n"); 5054 goto err_get_vers; 5055 } 5056 5057 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 5058 &priv->sec_attr); 5059 if (err) { 5060 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 5061 goto err_get_vers; 5062 } 5063 5064 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 5065 if (err) { 5066 dev_err(dev, "setup_congestion() failed\n"); 5067 goto err_get_vers; 5068 } 5069 5070 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 5071 priv->dpseci_attr.num_tx_queues); 5072 if (priv->num_pairs > num_online_cpus()) { 5073 dev_warn(dev, "%d queues won't be used\n", 5074 priv->num_pairs - num_online_cpus()); 5075 priv->num_pairs = num_online_cpus(); 5076 } 5077 5078 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 5079 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5080 &priv->rx_queue_attr[i]); 5081 if (err) { 5082 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 5083 goto err_get_rx_queue; 5084 } 5085 } 5086 5087 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 5088 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5089 &priv->tx_queue_attr[i]); 5090 if (err) { 5091 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 5092 goto err_get_rx_queue; 5093 } 5094 } 5095 5096 i = 0; 5097 for_each_online_cpu(cpu) { 5098 u8 j; 5099 5100 j = i % priv->num_pairs; 5101 5102 ppriv = per_cpu_ptr(priv->ppriv, cpu); 5103 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 5104 5105 /* 5106 * Allow all cores to enqueue, while only some of them 5107 * will take part in dequeuing. 5108 */ 5109 if (++i > priv->num_pairs) 5110 continue; 5111 5112 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 5113 ppriv->prio = j; 5114 5115 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 5116 priv->rx_queue_attr[j].fqid, 5117 priv->tx_queue_attr[j].fqid); 5118 5119 ppriv->net_dev = alloc_netdev_dummy(0); 5120 if (!ppriv->net_dev) { 5121 err = -ENOMEM; 5122 goto err_alloc_netdev; 5123 } 5124 cpumask_set_cpu(cpu, priv->clean_mask); 5125 ppriv->net_dev->dev = *dev; 5126 5127 netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, 5128 dpaa2_dpseci_poll, 5129 DPAA2_CAAM_NAPI_WEIGHT); 5130 } 5131 5132 return 0; 5133 5134 err_alloc_netdev: 5135 free_dpaa2_pcpu_netdev(priv, priv->clean_mask); 5136 err_get_rx_queue: 5137 dpaa2_dpseci_congestion_free(priv); 5138 err_get_vers: 5139 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5140 err_open: 5141 free_cpumask_var(priv->clean_mask); 5142 err_cpumask: 5143 return err; 5144 } 5145 5146 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 5147 { 5148 struct device *dev = priv->dev; 5149 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5150 struct dpaa2_caam_priv_per_cpu *ppriv; 5151 int i; 5152 5153 for (i = 0; i < priv->num_pairs; i++) { 5154 ppriv = per_cpu_ptr(priv->ppriv, i); 5155 napi_enable(&ppriv->napi); 5156 } 5157 5158 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 5159 } 5160 5161 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 5162 { 5163 struct device *dev = priv->dev; 5164 struct dpaa2_caam_priv_per_cpu *ppriv; 5165 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5166 int i, err = 0, enabled; 5167 5168 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5169 if (err) { 5170 dev_err(dev, "dpseci_disable() failed\n"); 5171 return err; 5172 } 5173 5174 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5175 if (err) { 5176 dev_err(dev, "dpseci_is_enabled() failed\n"); 5177 return err; 5178 } 5179 5180 dev_dbg(dev, "disable: %s\n", str_false_true(enabled)); 5181 5182 for (i = 0; i < priv->num_pairs; i++) { 5183 ppriv = per_cpu_ptr(priv->ppriv, i); 5184 napi_disable(&ppriv->napi); 5185 netif_napi_del(&ppriv->napi); 5186 } 5187 5188 return 0; 5189 } 5190 5191 static struct list_head hash_list; 5192 5193 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5194 { 5195 struct device *dev; 5196 struct dpaa2_caam_priv *priv; 5197 int i, err = 0; 5198 bool registered = false; 5199 5200 /* 5201 * There is no way to get CAAM endianness - there is no direct register 5202 * space access and MC f/w does not provide this attribute. 5203 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5204 * property. 5205 */ 5206 caam_little_end = true; 5207 5208 caam_imx = false; 5209 5210 dev = &dpseci_dev->dev; 5211 5212 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5213 if (!priv) 5214 return -ENOMEM; 5215 5216 dev_set_drvdata(dev, priv); 5217 5218 priv->domain = iommu_get_domain_for_dev(dev); 5219 5220 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5221 0, 0, NULL); 5222 if (!qi_cache) { 5223 dev_err(dev, "Can't allocate SEC cache\n"); 5224 return -ENOMEM; 5225 } 5226 5227 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5228 if (err) { 5229 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5230 goto err_dma_mask; 5231 } 5232 5233 /* Obtain a MC portal */ 5234 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5235 if (err) { 5236 if (err == -ENXIO) 5237 err = -EPROBE_DEFER; 5238 else 5239 dev_err(dev, "MC portal allocation failed\n"); 5240 5241 goto err_dma_mask; 5242 } 5243 5244 priv->ppriv = alloc_percpu(*priv->ppriv); 5245 if (!priv->ppriv) { 5246 dev_err(dev, "alloc_percpu() failed\n"); 5247 err = -ENOMEM; 5248 goto err_alloc_ppriv; 5249 } 5250 5251 /* DPSECI initialization */ 5252 err = dpaa2_dpseci_setup(dpseci_dev); 5253 if (err) { 5254 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5255 goto err_dpseci_setup; 5256 } 5257 5258 /* DPIO */ 5259 err = dpaa2_dpseci_dpio_setup(priv); 5260 if (err) { 5261 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); 5262 goto err_dpio_setup; 5263 } 5264 5265 /* DPSECI binding to DPIO */ 5266 err = dpaa2_dpseci_bind(priv); 5267 if (err) { 5268 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5269 goto err_bind; 5270 } 5271 5272 /* DPSECI enable */ 5273 err = dpaa2_dpseci_enable(priv); 5274 if (err) { 5275 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5276 goto err_bind; 5277 } 5278 5279 dpaa2_dpseci_debugfs_init(priv); 5280 5281 /* register crypto algorithms the device supports */ 5282 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5283 struct caam_skcipher_alg *t_alg = driver_algs + i; 5284 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5285 5286 /* Skip DES algorithms if not supported by device */ 5287 if (!priv->sec_attr.des_acc_num && 5288 (alg_sel == OP_ALG_ALGSEL_3DES || 5289 alg_sel == OP_ALG_ALGSEL_DES)) 5290 continue; 5291 5292 /* Skip AES algorithms if not supported by device */ 5293 if (!priv->sec_attr.aes_acc_num && 5294 alg_sel == OP_ALG_ALGSEL_AES) 5295 continue; 5296 5297 /* Skip CHACHA20 algorithms if not supported by device */ 5298 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5299 !priv->sec_attr.ccha_acc_num) 5300 continue; 5301 5302 t_alg->caam.dev = dev; 5303 caam_skcipher_alg_init(t_alg); 5304 5305 err = crypto_register_skcipher(&t_alg->skcipher); 5306 if (err) { 5307 dev_warn(dev, "%s alg registration failed: %d\n", 5308 t_alg->skcipher.base.cra_driver_name, err); 5309 continue; 5310 } 5311 5312 t_alg->registered = true; 5313 registered = true; 5314 } 5315 5316 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5317 struct caam_aead_alg *t_alg = driver_aeads + i; 5318 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5319 OP_ALG_ALGSEL_MASK; 5320 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5321 OP_ALG_ALGSEL_MASK; 5322 5323 /* Skip DES algorithms if not supported by device */ 5324 if (!priv->sec_attr.des_acc_num && 5325 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5326 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5327 continue; 5328 5329 /* Skip AES algorithms if not supported by device */ 5330 if (!priv->sec_attr.aes_acc_num && 5331 c1_alg_sel == OP_ALG_ALGSEL_AES) 5332 continue; 5333 5334 /* Skip CHACHA20 algorithms if not supported by device */ 5335 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5336 !priv->sec_attr.ccha_acc_num) 5337 continue; 5338 5339 /* Skip POLY1305 algorithms if not supported by device */ 5340 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5341 !priv->sec_attr.ptha_acc_num) 5342 continue; 5343 5344 /* 5345 * Skip algorithms requiring message digests 5346 * if MD not supported by device. 5347 */ 5348 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5349 !priv->sec_attr.md_acc_num) 5350 continue; 5351 5352 t_alg->caam.dev = dev; 5353 caam_aead_alg_init(t_alg); 5354 5355 err = crypto_register_aead(&t_alg->aead); 5356 if (err) { 5357 dev_warn(dev, "%s alg registration failed: %d\n", 5358 t_alg->aead.base.cra_driver_name, err); 5359 continue; 5360 } 5361 5362 t_alg->registered = true; 5363 registered = true; 5364 } 5365 if (registered) 5366 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5367 5368 /* register hash algorithms the device supports */ 5369 INIT_LIST_HEAD(&hash_list); 5370 5371 /* 5372 * Skip registration of any hashing algorithms if MD block 5373 * is not present. 5374 */ 5375 if (!priv->sec_attr.md_acc_num) 5376 return 0; 5377 5378 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5379 struct caam_hash_alg *t_alg; 5380 struct caam_hash_template *alg = driver_hash + i; 5381 5382 /* register hmac version */ 5383 t_alg = caam_hash_alloc(dev, alg, true); 5384 if (IS_ERR(t_alg)) { 5385 err = PTR_ERR(t_alg); 5386 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5387 alg->hmac_driver_name, err); 5388 continue; 5389 } 5390 5391 err = crypto_register_ahash(&t_alg->ahash_alg); 5392 if (err) { 5393 dev_warn(dev, "%s alg registration failed: %d\n", 5394 t_alg->ahash_alg.halg.base.cra_driver_name, 5395 err); 5396 kfree(t_alg); 5397 } else { 5398 list_add_tail(&t_alg->entry, &hash_list); 5399 } 5400 5401 /* register unkeyed version */ 5402 t_alg = caam_hash_alloc(dev, alg, false); 5403 if (IS_ERR(t_alg)) { 5404 err = PTR_ERR(t_alg); 5405 dev_warn(dev, "%s alg allocation failed: %d\n", 5406 alg->driver_name, err); 5407 continue; 5408 } 5409 5410 err = crypto_register_ahash(&t_alg->ahash_alg); 5411 if (err) { 5412 dev_warn(dev, "%s alg registration failed: %d\n", 5413 t_alg->ahash_alg.halg.base.cra_driver_name, 5414 err); 5415 kfree(t_alg); 5416 } else { 5417 list_add_tail(&t_alg->entry, &hash_list); 5418 } 5419 } 5420 if (!list_empty(&hash_list)) 5421 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5422 5423 return err; 5424 5425 err_bind: 5426 dpaa2_dpseci_dpio_free(priv); 5427 err_dpio_setup: 5428 dpaa2_dpseci_free(priv); 5429 err_dpseci_setup: 5430 free_percpu(priv->ppriv); 5431 err_alloc_ppriv: 5432 fsl_mc_portal_free(priv->mc_io); 5433 err_dma_mask: 5434 kmem_cache_destroy(qi_cache); 5435 5436 return err; 5437 } 5438 5439 static void __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5440 { 5441 struct device *dev; 5442 struct dpaa2_caam_priv *priv; 5443 int i; 5444 5445 dev = &ls_dev->dev; 5446 priv = dev_get_drvdata(dev); 5447 5448 dpaa2_dpseci_debugfs_exit(priv); 5449 5450 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5451 struct caam_aead_alg *t_alg = driver_aeads + i; 5452 5453 if (t_alg->registered) 5454 crypto_unregister_aead(&t_alg->aead); 5455 } 5456 5457 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5458 struct caam_skcipher_alg *t_alg = driver_algs + i; 5459 5460 if (t_alg->registered) 5461 crypto_unregister_skcipher(&t_alg->skcipher); 5462 } 5463 5464 if (hash_list.next) { 5465 struct caam_hash_alg *t_hash_alg, *p; 5466 5467 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5468 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5469 list_del(&t_hash_alg->entry); 5470 kfree(t_hash_alg); 5471 } 5472 } 5473 5474 dpaa2_dpseci_disable(priv); 5475 dpaa2_dpseci_dpio_free(priv); 5476 dpaa2_dpseci_free(priv); 5477 free_percpu(priv->ppriv); 5478 fsl_mc_portal_free(priv->mc_io); 5479 kmem_cache_destroy(qi_cache); 5480 } 5481 5482 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5483 { 5484 struct dpaa2_fd fd; 5485 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5486 struct dpaa2_caam_priv_per_cpu *ppriv; 5487 int err = 0, i; 5488 5489 if (IS_ERR(req)) 5490 return PTR_ERR(req); 5491 5492 if (priv->cscn_mem) { 5493 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5494 DPAA2_CSCN_SIZE, 5495 DMA_FROM_DEVICE); 5496 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) { 5497 dev_dbg_ratelimited(dev, "Dropping request\n"); 5498 return -EBUSY; 5499 } 5500 } 5501 5502 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5503 5504 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5505 DMA_BIDIRECTIONAL); 5506 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5507 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5508 goto err_out; 5509 } 5510 5511 memset(&fd, 0, sizeof(fd)); 5512 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5513 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5514 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5515 dpaa2_fd_set_flc(&fd, req->flc_dma); 5516 5517 ppriv = raw_cpu_ptr(priv->ppriv); 5518 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5519 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5520 &fd); 5521 if (err != -EBUSY) 5522 break; 5523 5524 cpu_relax(); 5525 } 5526 5527 if (unlikely(err)) { 5528 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5529 goto err_out; 5530 } 5531 5532 return -EINPROGRESS; 5533 5534 err_out: 5535 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5536 DMA_BIDIRECTIONAL); 5537 return -EIO; 5538 } 5539 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5540 5541 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5542 { 5543 .vendor = FSL_MC_VENDOR_FREESCALE, 5544 .obj_type = "dpseci", 5545 }, 5546 { .vendor = 0x0 } 5547 }; 5548 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table); 5549 5550 static struct fsl_mc_driver dpaa2_caam_driver = { 5551 .driver = { 5552 .name = KBUILD_MODNAME, 5553 .owner = THIS_MODULE, 5554 }, 5555 .probe = dpaa2_caam_probe, 5556 .remove = dpaa2_caam_remove, 5557 .match_id_table = dpaa2_caam_match_id_table 5558 }; 5559 5560 MODULE_LICENSE("Dual BSD/GPL"); 5561 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5562 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5563 5564 module_fsl_mc_driver(dpaa2_caam_driver); 5565