1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include "dpseci-debugfs.h" 19 #include <linux/dma-mapping.h> 20 #include <linux/fsl/mc.h> 21 #include <linux/kernel.h> 22 #include <soc/fsl/dpaa2-io.h> 23 #include <soc/fsl/dpaa2-fd.h> 24 #include <crypto/xts.h> 25 #include <asm/unaligned.h> 26 27 #define CAAM_CRA_PRIORITY 2000 28 29 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 30 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 31 SHA512_DIGEST_SIZE * 2) 32 33 /* 34 * This is a cache of buffers, from which the users of CAAM QI driver 35 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 36 * NOTE: A more elegant solution would be to have some headroom in the frames 37 * being processed. This can be added by the dpaa2-eth driver. This would 38 * pose a problem for userspace application processing which cannot 39 * know of this limitation. So for now, this will work. 40 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 41 */ 42 static struct kmem_cache *qi_cache; 43 44 struct caam_alg_entry { 45 struct device *dev; 46 int class1_alg_type; 47 int class2_alg_type; 48 bool rfc3686; 49 bool geniv; 50 bool nodkp; 51 }; 52 53 struct caam_aead_alg { 54 struct aead_alg aead; 55 struct caam_alg_entry caam; 56 bool registered; 57 }; 58 59 struct caam_skcipher_alg { 60 struct skcipher_alg skcipher; 61 struct caam_alg_entry caam; 62 bool registered; 63 }; 64 65 /** 66 * struct caam_ctx - per-session context 67 * @flc: Flow Contexts array 68 * @key: [authentication key], encryption key 69 * @flc_dma: I/O virtual addresses of the Flow Contexts 70 * @key_dma: I/O virtual address of the key 71 * @dir: DMA direction for mapping key and Flow Contexts 72 * @dev: dpseci device 73 * @adata: authentication algorithm details 74 * @cdata: encryption algorithm details 75 * @authsize: authentication tag (a.k.a. ICV / MAC) size 76 * @xts_key_fallback: true if fallback tfm needs to be used due 77 * to unsupported xts key lengths 78 * @fallback: xts fallback tfm 79 */ 80 struct caam_ctx { 81 struct caam_flc flc[NUM_OP]; 82 u8 key[CAAM_MAX_KEY_SIZE]; 83 dma_addr_t flc_dma[NUM_OP]; 84 dma_addr_t key_dma; 85 enum dma_data_direction dir; 86 struct device *dev; 87 struct alginfo adata; 88 struct alginfo cdata; 89 unsigned int authsize; 90 bool xts_key_fallback; 91 struct crypto_skcipher *fallback; 92 }; 93 94 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 95 dma_addr_t iova_addr) 96 { 97 phys_addr_t phys_addr; 98 99 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 100 iova_addr; 101 102 return phys_to_virt(phys_addr); 103 } 104 105 /* 106 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 107 * 108 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 109 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 110 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 111 * hosting 16 SG entries. 112 * 113 * @flags - flags that would be used for the equivalent kmalloc(..) call 114 * 115 * Returns a pointer to a retrieved buffer on success or NULL on failure. 116 */ 117 static inline void *qi_cache_zalloc(gfp_t flags) 118 { 119 return kmem_cache_zalloc(qi_cache, flags); 120 } 121 122 /* 123 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 124 * 125 * @obj - buffer previously allocated by qi_cache_zalloc 126 * 127 * No checking is being done, the call is a passthrough call to 128 * kmem_cache_free(...) 129 */ 130 static inline void qi_cache_free(void *obj) 131 { 132 kmem_cache_free(qi_cache, obj); 133 } 134 135 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 136 { 137 switch (crypto_tfm_alg_type(areq->tfm)) { 138 case CRYPTO_ALG_TYPE_SKCIPHER: 139 return skcipher_request_ctx_dma(skcipher_request_cast(areq)); 140 case CRYPTO_ALG_TYPE_AEAD: 141 return aead_request_ctx_dma( 142 container_of(areq, struct aead_request, base)); 143 case CRYPTO_ALG_TYPE_AHASH: 144 return ahash_request_ctx_dma(ahash_request_cast(areq)); 145 default: 146 return ERR_PTR(-EINVAL); 147 } 148 } 149 150 static void caam_unmap(struct device *dev, struct scatterlist *src, 151 struct scatterlist *dst, int src_nents, 152 int dst_nents, dma_addr_t iv_dma, int ivsize, 153 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 154 int qm_sg_bytes) 155 { 156 if (dst != src) { 157 if (src_nents) 158 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 159 if (dst_nents) 160 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 161 } else { 162 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 163 } 164 165 if (iv_dma) 166 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 167 168 if (qm_sg_bytes) 169 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 170 } 171 172 static int aead_set_sh_desc(struct crypto_aead *aead) 173 { 174 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 175 typeof(*alg), aead); 176 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 177 unsigned int ivsize = crypto_aead_ivsize(aead); 178 struct device *dev = ctx->dev; 179 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 180 struct caam_flc *flc; 181 u32 *desc; 182 u32 ctx1_iv_off = 0; 183 u32 *nonce = NULL; 184 unsigned int data_len[2]; 185 u32 inl_mask; 186 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 187 OP_ALG_AAI_CTR_MOD128); 188 const bool is_rfc3686 = alg->caam.rfc3686; 189 190 if (!ctx->cdata.keylen || !ctx->authsize) 191 return 0; 192 193 /* 194 * AES-CTR needs to load IV in CONTEXT1 reg 195 * at an offset of 128bits (16bytes) 196 * CONTEXT1[255:128] = IV 197 */ 198 if (ctr_mode) 199 ctx1_iv_off = 16; 200 201 /* 202 * RFC3686 specific: 203 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 204 */ 205 if (is_rfc3686) { 206 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 207 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 208 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 209 } 210 211 /* 212 * In case |user key| > |derived key|, using DKP<imm,imm> would result 213 * in invalid opcodes (last bytes of user key) in the resulting 214 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 215 * addresses are needed. 216 */ 217 ctx->adata.key_virt = ctx->key; 218 ctx->adata.key_dma = ctx->key_dma; 219 220 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 221 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 222 223 data_len[0] = ctx->adata.keylen_pad; 224 data_len[1] = ctx->cdata.keylen; 225 226 /* aead_encrypt shared descriptor */ 227 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 228 DESC_QI_AEAD_ENC_LEN) + 229 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 230 DESC_JOB_IO_LEN, data_len, &inl_mask, 231 ARRAY_SIZE(data_len)) < 0) 232 return -EINVAL; 233 234 ctx->adata.key_inline = !!(inl_mask & 1); 235 ctx->cdata.key_inline = !!(inl_mask & 2); 236 237 flc = &ctx->flc[ENCRYPT]; 238 desc = flc->sh_desc; 239 240 if (alg->caam.geniv) 241 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 242 ivsize, ctx->authsize, is_rfc3686, 243 nonce, ctx1_iv_off, true, 244 priv->sec_attr.era); 245 else 246 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 247 ivsize, ctx->authsize, is_rfc3686, nonce, 248 ctx1_iv_off, true, priv->sec_attr.era); 249 250 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 251 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 252 sizeof(flc->flc) + desc_bytes(desc), 253 ctx->dir); 254 255 /* aead_decrypt shared descriptor */ 256 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 257 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 258 DESC_JOB_IO_LEN, data_len, &inl_mask, 259 ARRAY_SIZE(data_len)) < 0) 260 return -EINVAL; 261 262 ctx->adata.key_inline = !!(inl_mask & 1); 263 ctx->cdata.key_inline = !!(inl_mask & 2); 264 265 flc = &ctx->flc[DECRYPT]; 266 desc = flc->sh_desc; 267 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 268 ivsize, ctx->authsize, alg->caam.geniv, 269 is_rfc3686, nonce, ctx1_iv_off, true, 270 priv->sec_attr.era); 271 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 272 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 273 sizeof(flc->flc) + desc_bytes(desc), 274 ctx->dir); 275 276 return 0; 277 } 278 279 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 280 { 281 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 282 283 ctx->authsize = authsize; 284 aead_set_sh_desc(authenc); 285 286 return 0; 287 } 288 289 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 290 unsigned int keylen) 291 { 292 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 293 struct device *dev = ctx->dev; 294 struct crypto_authenc_keys keys; 295 296 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 297 goto badkey; 298 299 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 300 keys.authkeylen + keys.enckeylen, keys.enckeylen, 301 keys.authkeylen); 302 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 303 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 304 305 ctx->adata.keylen = keys.authkeylen; 306 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 307 OP_ALG_ALGSEL_MASK); 308 309 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 310 goto badkey; 311 312 memcpy(ctx->key, keys.authkey, keys.authkeylen); 313 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 314 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 315 keys.enckeylen, ctx->dir); 316 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 317 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 318 ctx->adata.keylen_pad + keys.enckeylen, 1); 319 320 ctx->cdata.keylen = keys.enckeylen; 321 322 memzero_explicit(&keys, sizeof(keys)); 323 return aead_set_sh_desc(aead); 324 badkey: 325 memzero_explicit(&keys, sizeof(keys)); 326 return -EINVAL; 327 } 328 329 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 330 unsigned int keylen) 331 { 332 struct crypto_authenc_keys keys; 333 int err; 334 335 err = crypto_authenc_extractkeys(&keys, key, keylen); 336 if (unlikely(err)) 337 goto out; 338 339 err = -EINVAL; 340 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 341 goto out; 342 343 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 344 aead_setkey(aead, key, keylen); 345 346 out: 347 memzero_explicit(&keys, sizeof(keys)); 348 return err; 349 } 350 351 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 352 bool encrypt) 353 { 354 struct crypto_aead *aead = crypto_aead_reqtfm(req); 355 struct caam_request *req_ctx = aead_request_ctx_dma(req); 356 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 357 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 358 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 359 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 360 typeof(*alg), aead); 361 struct device *dev = ctx->dev; 362 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 363 GFP_KERNEL : GFP_ATOMIC; 364 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 365 int src_len, dst_len = 0; 366 struct aead_edesc *edesc; 367 dma_addr_t qm_sg_dma, iv_dma = 0; 368 int ivsize = 0; 369 unsigned int authsize = ctx->authsize; 370 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 371 int in_len, out_len; 372 struct dpaa2_sg_entry *sg_table; 373 374 /* allocate space for base edesc, link tables and IV */ 375 edesc = qi_cache_zalloc(flags); 376 if (unlikely(!edesc)) { 377 dev_err(dev, "could not allocate extended descriptor\n"); 378 return ERR_PTR(-ENOMEM); 379 } 380 381 if (unlikely(req->dst != req->src)) { 382 src_len = req->assoclen + req->cryptlen; 383 dst_len = src_len + (encrypt ? authsize : (-authsize)); 384 385 src_nents = sg_nents_for_len(req->src, src_len); 386 if (unlikely(src_nents < 0)) { 387 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 388 src_len); 389 qi_cache_free(edesc); 390 return ERR_PTR(src_nents); 391 } 392 393 dst_nents = sg_nents_for_len(req->dst, dst_len); 394 if (unlikely(dst_nents < 0)) { 395 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 396 dst_len); 397 qi_cache_free(edesc); 398 return ERR_PTR(dst_nents); 399 } 400 401 if (src_nents) { 402 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 403 DMA_TO_DEVICE); 404 if (unlikely(!mapped_src_nents)) { 405 dev_err(dev, "unable to map source\n"); 406 qi_cache_free(edesc); 407 return ERR_PTR(-ENOMEM); 408 } 409 } else { 410 mapped_src_nents = 0; 411 } 412 413 if (dst_nents) { 414 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 415 DMA_FROM_DEVICE); 416 if (unlikely(!mapped_dst_nents)) { 417 dev_err(dev, "unable to map destination\n"); 418 dma_unmap_sg(dev, req->src, src_nents, 419 DMA_TO_DEVICE); 420 qi_cache_free(edesc); 421 return ERR_PTR(-ENOMEM); 422 } 423 } else { 424 mapped_dst_nents = 0; 425 } 426 } else { 427 src_len = req->assoclen + req->cryptlen + 428 (encrypt ? authsize : 0); 429 430 src_nents = sg_nents_for_len(req->src, src_len); 431 if (unlikely(src_nents < 0)) { 432 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 433 src_len); 434 qi_cache_free(edesc); 435 return ERR_PTR(src_nents); 436 } 437 438 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 439 DMA_BIDIRECTIONAL); 440 if (unlikely(!mapped_src_nents)) { 441 dev_err(dev, "unable to map source\n"); 442 qi_cache_free(edesc); 443 return ERR_PTR(-ENOMEM); 444 } 445 } 446 447 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 448 ivsize = crypto_aead_ivsize(aead); 449 450 /* 451 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 452 * Input is not contiguous. 453 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 454 * the end of the table by allocating more S/G entries. Logic: 455 * if (src != dst && output S/G) 456 * pad output S/G, if needed 457 * else if (src == dst && S/G) 458 * overlapping S/Gs; pad one of them 459 * else if (input S/G) ... 460 * pad input S/G, if needed 461 */ 462 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 463 if (mapped_dst_nents > 1) 464 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 465 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 466 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 467 1 + !!ivsize + 468 pad_sg_nents(mapped_src_nents)); 469 else 470 qm_sg_nents = pad_sg_nents(qm_sg_nents); 471 472 sg_table = &edesc->sgt[0]; 473 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 474 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 475 CAAM_QI_MEMCACHE_SIZE)) { 476 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 477 qm_sg_nents, ivsize); 478 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 479 0, DMA_NONE, 0, 0); 480 qi_cache_free(edesc); 481 return ERR_PTR(-ENOMEM); 482 } 483 484 if (ivsize) { 485 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 486 487 /* Make sure IV is located in a DMAable area */ 488 memcpy(iv, req->iv, ivsize); 489 490 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 491 if (dma_mapping_error(dev, iv_dma)) { 492 dev_err(dev, "unable to map IV\n"); 493 caam_unmap(dev, req->src, req->dst, src_nents, 494 dst_nents, 0, 0, DMA_NONE, 0, 0); 495 qi_cache_free(edesc); 496 return ERR_PTR(-ENOMEM); 497 } 498 } 499 500 edesc->src_nents = src_nents; 501 edesc->dst_nents = dst_nents; 502 edesc->iv_dma = iv_dma; 503 504 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 505 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 506 /* 507 * The associated data comes already with the IV but we need 508 * to skip it when we authenticate or encrypt... 509 */ 510 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 511 else 512 edesc->assoclen = cpu_to_caam32(req->assoclen); 513 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 514 DMA_TO_DEVICE); 515 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 516 dev_err(dev, "unable to map assoclen\n"); 517 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 518 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 519 qi_cache_free(edesc); 520 return ERR_PTR(-ENOMEM); 521 } 522 523 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 524 qm_sg_index++; 525 if (ivsize) { 526 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 527 qm_sg_index++; 528 } 529 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 530 qm_sg_index += mapped_src_nents; 531 532 if (mapped_dst_nents > 1) 533 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 534 535 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 536 if (dma_mapping_error(dev, qm_sg_dma)) { 537 dev_err(dev, "unable to map S/G table\n"); 538 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 539 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 540 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 541 qi_cache_free(edesc); 542 return ERR_PTR(-ENOMEM); 543 } 544 545 edesc->qm_sg_dma = qm_sg_dma; 546 edesc->qm_sg_bytes = qm_sg_bytes; 547 548 out_len = req->assoclen + req->cryptlen + 549 (encrypt ? ctx->authsize : (-ctx->authsize)); 550 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 551 552 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 553 dpaa2_fl_set_final(in_fle, true); 554 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 555 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 556 dpaa2_fl_set_len(in_fle, in_len); 557 558 if (req->dst == req->src) { 559 if (mapped_src_nents == 1) { 560 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 561 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 562 } else { 563 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 564 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 565 (1 + !!ivsize) * sizeof(*sg_table)); 566 } 567 } else if (!mapped_dst_nents) { 568 /* 569 * crypto engine requires the output entry to be present when 570 * "frame list" FD is used. 571 * Since engine does not support FMT=2'b11 (unused entry type), 572 * leaving out_fle zeroized is the best option. 573 */ 574 goto skip_out_fle; 575 } else if (mapped_dst_nents == 1) { 576 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 577 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 578 } else { 579 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 580 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 581 sizeof(*sg_table)); 582 } 583 584 dpaa2_fl_set_len(out_fle, out_len); 585 586 skip_out_fle: 587 return edesc; 588 } 589 590 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 591 { 592 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 593 unsigned int ivsize = crypto_aead_ivsize(aead); 594 struct device *dev = ctx->dev; 595 struct caam_flc *flc; 596 u32 *desc; 597 598 if (!ctx->cdata.keylen || !ctx->authsize) 599 return 0; 600 601 flc = &ctx->flc[ENCRYPT]; 602 desc = flc->sh_desc; 603 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 604 ctx->authsize, true, true); 605 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 606 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 607 sizeof(flc->flc) + desc_bytes(desc), 608 ctx->dir); 609 610 flc = &ctx->flc[DECRYPT]; 611 desc = flc->sh_desc; 612 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 613 ctx->authsize, false, true); 614 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 615 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 616 sizeof(flc->flc) + desc_bytes(desc), 617 ctx->dir); 618 619 return 0; 620 } 621 622 static int chachapoly_setauthsize(struct crypto_aead *aead, 623 unsigned int authsize) 624 { 625 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 626 627 if (authsize != POLY1305_DIGEST_SIZE) 628 return -EINVAL; 629 630 ctx->authsize = authsize; 631 return chachapoly_set_sh_desc(aead); 632 } 633 634 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 635 unsigned int keylen) 636 { 637 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 638 unsigned int ivsize = crypto_aead_ivsize(aead); 639 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 640 641 if (keylen != CHACHA_KEY_SIZE + saltlen) 642 return -EINVAL; 643 644 ctx->cdata.key_virt = key; 645 ctx->cdata.keylen = keylen - saltlen; 646 647 return chachapoly_set_sh_desc(aead); 648 } 649 650 static int gcm_set_sh_desc(struct crypto_aead *aead) 651 { 652 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 653 struct device *dev = ctx->dev; 654 unsigned int ivsize = crypto_aead_ivsize(aead); 655 struct caam_flc *flc; 656 u32 *desc; 657 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 658 ctx->cdata.keylen; 659 660 if (!ctx->cdata.keylen || !ctx->authsize) 661 return 0; 662 663 /* 664 * AES GCM encrypt shared descriptor 665 * Job Descriptor and Shared Descriptor 666 * must fit into the 64-word Descriptor h/w Buffer 667 */ 668 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 669 ctx->cdata.key_inline = true; 670 ctx->cdata.key_virt = ctx->key; 671 } else { 672 ctx->cdata.key_inline = false; 673 ctx->cdata.key_dma = ctx->key_dma; 674 } 675 676 flc = &ctx->flc[ENCRYPT]; 677 desc = flc->sh_desc; 678 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 679 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 680 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 681 sizeof(flc->flc) + desc_bytes(desc), 682 ctx->dir); 683 684 /* 685 * Job Descriptor and Shared Descriptors 686 * must all fit into the 64-word Descriptor h/w Buffer 687 */ 688 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 689 ctx->cdata.key_inline = true; 690 ctx->cdata.key_virt = ctx->key; 691 } else { 692 ctx->cdata.key_inline = false; 693 ctx->cdata.key_dma = ctx->key_dma; 694 } 695 696 flc = &ctx->flc[DECRYPT]; 697 desc = flc->sh_desc; 698 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 699 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 700 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 701 sizeof(flc->flc) + desc_bytes(desc), 702 ctx->dir); 703 704 return 0; 705 } 706 707 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 708 { 709 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 710 int err; 711 712 err = crypto_gcm_check_authsize(authsize); 713 if (err) 714 return err; 715 716 ctx->authsize = authsize; 717 gcm_set_sh_desc(authenc); 718 719 return 0; 720 } 721 722 static int gcm_setkey(struct crypto_aead *aead, 723 const u8 *key, unsigned int keylen) 724 { 725 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 726 struct device *dev = ctx->dev; 727 int ret; 728 729 ret = aes_check_keylen(keylen); 730 if (ret) 731 return ret; 732 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 733 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 734 735 memcpy(ctx->key, key, keylen); 736 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 737 ctx->cdata.keylen = keylen; 738 739 return gcm_set_sh_desc(aead); 740 } 741 742 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 743 { 744 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 745 struct device *dev = ctx->dev; 746 unsigned int ivsize = crypto_aead_ivsize(aead); 747 struct caam_flc *flc; 748 u32 *desc; 749 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 750 ctx->cdata.keylen; 751 752 if (!ctx->cdata.keylen || !ctx->authsize) 753 return 0; 754 755 ctx->cdata.key_virt = ctx->key; 756 757 /* 758 * RFC4106 encrypt shared descriptor 759 * Job Descriptor and Shared Descriptor 760 * must fit into the 64-word Descriptor h/w Buffer 761 */ 762 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 763 ctx->cdata.key_inline = true; 764 } else { 765 ctx->cdata.key_inline = false; 766 ctx->cdata.key_dma = ctx->key_dma; 767 } 768 769 flc = &ctx->flc[ENCRYPT]; 770 desc = flc->sh_desc; 771 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 772 true); 773 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 774 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 775 sizeof(flc->flc) + desc_bytes(desc), 776 ctx->dir); 777 778 /* 779 * Job Descriptor and Shared Descriptors 780 * must all fit into the 64-word Descriptor h/w Buffer 781 */ 782 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 783 ctx->cdata.key_inline = true; 784 } else { 785 ctx->cdata.key_inline = false; 786 ctx->cdata.key_dma = ctx->key_dma; 787 } 788 789 flc = &ctx->flc[DECRYPT]; 790 desc = flc->sh_desc; 791 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 792 true); 793 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 794 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 795 sizeof(flc->flc) + desc_bytes(desc), 796 ctx->dir); 797 798 return 0; 799 } 800 801 static int rfc4106_setauthsize(struct crypto_aead *authenc, 802 unsigned int authsize) 803 { 804 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 805 int err; 806 807 err = crypto_rfc4106_check_authsize(authsize); 808 if (err) 809 return err; 810 811 ctx->authsize = authsize; 812 rfc4106_set_sh_desc(authenc); 813 814 return 0; 815 } 816 817 static int rfc4106_setkey(struct crypto_aead *aead, 818 const u8 *key, unsigned int keylen) 819 { 820 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 821 struct device *dev = ctx->dev; 822 int ret; 823 824 ret = aes_check_keylen(keylen - 4); 825 if (ret) 826 return ret; 827 828 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 829 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 830 831 memcpy(ctx->key, key, keylen); 832 /* 833 * The last four bytes of the key material are used as the salt value 834 * in the nonce. Update the AES key length. 835 */ 836 ctx->cdata.keylen = keylen - 4; 837 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 838 ctx->dir); 839 840 return rfc4106_set_sh_desc(aead); 841 } 842 843 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 844 { 845 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 846 struct device *dev = ctx->dev; 847 unsigned int ivsize = crypto_aead_ivsize(aead); 848 struct caam_flc *flc; 849 u32 *desc; 850 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 851 ctx->cdata.keylen; 852 853 if (!ctx->cdata.keylen || !ctx->authsize) 854 return 0; 855 856 ctx->cdata.key_virt = ctx->key; 857 858 /* 859 * RFC4543 encrypt shared descriptor 860 * Job Descriptor and Shared Descriptor 861 * must fit into the 64-word Descriptor h/w Buffer 862 */ 863 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 864 ctx->cdata.key_inline = true; 865 } else { 866 ctx->cdata.key_inline = false; 867 ctx->cdata.key_dma = ctx->key_dma; 868 } 869 870 flc = &ctx->flc[ENCRYPT]; 871 desc = flc->sh_desc; 872 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 873 true); 874 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 875 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 876 sizeof(flc->flc) + desc_bytes(desc), 877 ctx->dir); 878 879 /* 880 * Job Descriptor and Shared Descriptors 881 * must all fit into the 64-word Descriptor h/w Buffer 882 */ 883 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 884 ctx->cdata.key_inline = true; 885 } else { 886 ctx->cdata.key_inline = false; 887 ctx->cdata.key_dma = ctx->key_dma; 888 } 889 890 flc = &ctx->flc[DECRYPT]; 891 desc = flc->sh_desc; 892 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 893 true); 894 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 895 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 896 sizeof(flc->flc) + desc_bytes(desc), 897 ctx->dir); 898 899 return 0; 900 } 901 902 static int rfc4543_setauthsize(struct crypto_aead *authenc, 903 unsigned int authsize) 904 { 905 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 906 907 if (authsize != 16) 908 return -EINVAL; 909 910 ctx->authsize = authsize; 911 rfc4543_set_sh_desc(authenc); 912 913 return 0; 914 } 915 916 static int rfc4543_setkey(struct crypto_aead *aead, 917 const u8 *key, unsigned int keylen) 918 { 919 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 920 struct device *dev = ctx->dev; 921 int ret; 922 923 ret = aes_check_keylen(keylen - 4); 924 if (ret) 925 return ret; 926 927 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 928 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 929 930 memcpy(ctx->key, key, keylen); 931 /* 932 * The last four bytes of the key material are used as the salt value 933 * in the nonce. Update the AES key length. 934 */ 935 ctx->cdata.keylen = keylen - 4; 936 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 937 ctx->dir); 938 939 return rfc4543_set_sh_desc(aead); 940 } 941 942 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 943 unsigned int keylen, const u32 ctx1_iv_off) 944 { 945 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 946 struct caam_skcipher_alg *alg = 947 container_of(crypto_skcipher_alg(skcipher), 948 struct caam_skcipher_alg, skcipher); 949 struct device *dev = ctx->dev; 950 struct caam_flc *flc; 951 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 952 u32 *desc; 953 const bool is_rfc3686 = alg->caam.rfc3686; 954 955 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 956 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 957 958 ctx->cdata.keylen = keylen; 959 ctx->cdata.key_virt = key; 960 ctx->cdata.key_inline = true; 961 962 /* skcipher_encrypt shared descriptor */ 963 flc = &ctx->flc[ENCRYPT]; 964 desc = flc->sh_desc; 965 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 966 ctx1_iv_off); 967 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 968 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 969 sizeof(flc->flc) + desc_bytes(desc), 970 ctx->dir); 971 972 /* skcipher_decrypt shared descriptor */ 973 flc = &ctx->flc[DECRYPT]; 974 desc = flc->sh_desc; 975 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 976 ctx1_iv_off); 977 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 978 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 979 sizeof(flc->flc) + desc_bytes(desc), 980 ctx->dir); 981 982 return 0; 983 } 984 985 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 986 const u8 *key, unsigned int keylen) 987 { 988 int err; 989 990 err = aes_check_keylen(keylen); 991 if (err) 992 return err; 993 994 return skcipher_setkey(skcipher, key, keylen, 0); 995 } 996 997 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 998 const u8 *key, unsigned int keylen) 999 { 1000 u32 ctx1_iv_off; 1001 int err; 1002 1003 /* 1004 * RFC3686 specific: 1005 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1006 * | *key = {KEY, NONCE} 1007 */ 1008 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1009 keylen -= CTR_RFC3686_NONCE_SIZE; 1010 1011 err = aes_check_keylen(keylen); 1012 if (err) 1013 return err; 1014 1015 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1016 } 1017 1018 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1019 const u8 *key, unsigned int keylen) 1020 { 1021 u32 ctx1_iv_off; 1022 int err; 1023 1024 /* 1025 * AES-CTR needs to load IV in CONTEXT1 reg 1026 * at an offset of 128bits (16bytes) 1027 * CONTEXT1[255:128] = IV 1028 */ 1029 ctx1_iv_off = 16; 1030 1031 err = aes_check_keylen(keylen); 1032 if (err) 1033 return err; 1034 1035 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1036 } 1037 1038 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1039 const u8 *key, unsigned int keylen) 1040 { 1041 if (keylen != CHACHA_KEY_SIZE) 1042 return -EINVAL; 1043 1044 return skcipher_setkey(skcipher, key, keylen, 0); 1045 } 1046 1047 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1048 const u8 *key, unsigned int keylen) 1049 { 1050 return verify_skcipher_des_key(skcipher, key) ?: 1051 skcipher_setkey(skcipher, key, keylen, 0); 1052 } 1053 1054 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1055 const u8 *key, unsigned int keylen) 1056 { 1057 return verify_skcipher_des3_key(skcipher, key) ?: 1058 skcipher_setkey(skcipher, key, keylen, 0); 1059 } 1060 1061 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1062 unsigned int keylen) 1063 { 1064 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1065 struct device *dev = ctx->dev; 1066 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 1067 struct caam_flc *flc; 1068 u32 *desc; 1069 int err; 1070 1071 err = xts_verify_key(skcipher, key, keylen); 1072 if (err) { 1073 dev_dbg(dev, "key size mismatch\n"); 1074 return err; 1075 } 1076 1077 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 1078 ctx->xts_key_fallback = true; 1079 1080 if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { 1081 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 1082 if (err) 1083 return err; 1084 } 1085 1086 ctx->cdata.keylen = keylen; 1087 ctx->cdata.key_virt = key; 1088 ctx->cdata.key_inline = true; 1089 1090 /* xts_skcipher_encrypt shared descriptor */ 1091 flc = &ctx->flc[ENCRYPT]; 1092 desc = flc->sh_desc; 1093 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1094 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1095 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1096 sizeof(flc->flc) + desc_bytes(desc), 1097 ctx->dir); 1098 1099 /* xts_skcipher_decrypt shared descriptor */ 1100 flc = &ctx->flc[DECRYPT]; 1101 desc = flc->sh_desc; 1102 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1103 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1104 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1105 sizeof(flc->flc) + desc_bytes(desc), 1106 ctx->dir); 1107 1108 return 0; 1109 } 1110 1111 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1112 { 1113 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1114 struct caam_request *req_ctx = skcipher_request_ctx_dma(req); 1115 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1116 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1117 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1118 struct device *dev = ctx->dev; 1119 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1120 GFP_KERNEL : GFP_ATOMIC; 1121 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1122 struct skcipher_edesc *edesc; 1123 dma_addr_t iv_dma; 1124 u8 *iv; 1125 int ivsize = crypto_skcipher_ivsize(skcipher); 1126 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1127 struct dpaa2_sg_entry *sg_table; 1128 1129 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1130 if (unlikely(src_nents < 0)) { 1131 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1132 req->cryptlen); 1133 return ERR_PTR(src_nents); 1134 } 1135 1136 if (unlikely(req->dst != req->src)) { 1137 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1138 if (unlikely(dst_nents < 0)) { 1139 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1140 req->cryptlen); 1141 return ERR_PTR(dst_nents); 1142 } 1143 1144 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1145 DMA_TO_DEVICE); 1146 if (unlikely(!mapped_src_nents)) { 1147 dev_err(dev, "unable to map source\n"); 1148 return ERR_PTR(-ENOMEM); 1149 } 1150 1151 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1152 DMA_FROM_DEVICE); 1153 if (unlikely(!mapped_dst_nents)) { 1154 dev_err(dev, "unable to map destination\n"); 1155 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1156 return ERR_PTR(-ENOMEM); 1157 } 1158 } else { 1159 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1160 DMA_BIDIRECTIONAL); 1161 if (unlikely(!mapped_src_nents)) { 1162 dev_err(dev, "unable to map source\n"); 1163 return ERR_PTR(-ENOMEM); 1164 } 1165 } 1166 1167 qm_sg_ents = 1 + mapped_src_nents; 1168 dst_sg_idx = qm_sg_ents; 1169 1170 /* 1171 * Input, output HW S/G tables: [IV, src][dst, IV] 1172 * IV entries point to the same buffer 1173 * If src == dst, S/G entries are reused (S/G tables overlap) 1174 * 1175 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1176 * the end of the table by allocating more S/G entries. 1177 */ 1178 if (req->src != req->dst) 1179 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1180 else 1181 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1182 1183 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1184 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1185 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1186 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1187 qm_sg_ents, ivsize); 1188 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1189 0, DMA_NONE, 0, 0); 1190 return ERR_PTR(-ENOMEM); 1191 } 1192 1193 /* allocate space for base edesc, link tables and IV */ 1194 edesc = qi_cache_zalloc(flags); 1195 if (unlikely(!edesc)) { 1196 dev_err(dev, "could not allocate extended descriptor\n"); 1197 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1198 0, DMA_NONE, 0, 0); 1199 return ERR_PTR(-ENOMEM); 1200 } 1201 1202 /* Make sure IV is located in a DMAable area */ 1203 sg_table = &edesc->sgt[0]; 1204 iv = (u8 *)(sg_table + qm_sg_ents); 1205 memcpy(iv, req->iv, ivsize); 1206 1207 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1208 if (dma_mapping_error(dev, iv_dma)) { 1209 dev_err(dev, "unable to map IV\n"); 1210 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1211 0, DMA_NONE, 0, 0); 1212 qi_cache_free(edesc); 1213 return ERR_PTR(-ENOMEM); 1214 } 1215 1216 edesc->src_nents = src_nents; 1217 edesc->dst_nents = dst_nents; 1218 edesc->iv_dma = iv_dma; 1219 edesc->qm_sg_bytes = qm_sg_bytes; 1220 1221 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1222 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1223 1224 if (req->src != req->dst) 1225 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1226 1227 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1228 ivsize, 0); 1229 1230 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1231 DMA_TO_DEVICE); 1232 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1233 dev_err(dev, "unable to map S/G table\n"); 1234 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1235 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1236 qi_cache_free(edesc); 1237 return ERR_PTR(-ENOMEM); 1238 } 1239 1240 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1241 dpaa2_fl_set_final(in_fle, true); 1242 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1243 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1244 1245 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1246 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1247 1248 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1249 1250 if (req->src == req->dst) 1251 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1252 sizeof(*sg_table)); 1253 else 1254 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1255 sizeof(*sg_table)); 1256 1257 return edesc; 1258 } 1259 1260 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1261 struct aead_request *req) 1262 { 1263 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1264 int ivsize = crypto_aead_ivsize(aead); 1265 1266 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1267 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1268 edesc->qm_sg_bytes); 1269 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1270 } 1271 1272 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1273 struct skcipher_request *req) 1274 { 1275 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1276 int ivsize = crypto_skcipher_ivsize(skcipher); 1277 1278 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1279 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1280 edesc->qm_sg_bytes); 1281 } 1282 1283 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1284 { 1285 struct crypto_async_request *areq = cbk_ctx; 1286 struct aead_request *req = container_of(areq, struct aead_request, 1287 base); 1288 struct caam_request *req_ctx = to_caam_req(areq); 1289 struct aead_edesc *edesc = req_ctx->edesc; 1290 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1291 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1292 int ecode = 0; 1293 1294 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1295 1296 if (unlikely(status)) 1297 ecode = caam_qi2_strstatus(ctx->dev, status); 1298 1299 aead_unmap(ctx->dev, edesc, req); 1300 qi_cache_free(edesc); 1301 aead_request_complete(req, ecode); 1302 } 1303 1304 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1305 { 1306 struct crypto_async_request *areq = cbk_ctx; 1307 struct aead_request *req = container_of(areq, struct aead_request, 1308 base); 1309 struct caam_request *req_ctx = to_caam_req(areq); 1310 struct aead_edesc *edesc = req_ctx->edesc; 1311 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1312 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1313 int ecode = 0; 1314 1315 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1316 1317 if (unlikely(status)) 1318 ecode = caam_qi2_strstatus(ctx->dev, status); 1319 1320 aead_unmap(ctx->dev, edesc, req); 1321 qi_cache_free(edesc); 1322 aead_request_complete(req, ecode); 1323 } 1324 1325 static int aead_encrypt(struct aead_request *req) 1326 { 1327 struct aead_edesc *edesc; 1328 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1329 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1330 struct caam_request *caam_req = aead_request_ctx_dma(req); 1331 int ret; 1332 1333 /* allocate extended descriptor */ 1334 edesc = aead_edesc_alloc(req, true); 1335 if (IS_ERR(edesc)) 1336 return PTR_ERR(edesc); 1337 1338 caam_req->flc = &ctx->flc[ENCRYPT]; 1339 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1340 caam_req->cbk = aead_encrypt_done; 1341 caam_req->ctx = &req->base; 1342 caam_req->edesc = edesc; 1343 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1344 if (ret != -EINPROGRESS && 1345 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1346 aead_unmap(ctx->dev, edesc, req); 1347 qi_cache_free(edesc); 1348 } 1349 1350 return ret; 1351 } 1352 1353 static int aead_decrypt(struct aead_request *req) 1354 { 1355 struct aead_edesc *edesc; 1356 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1357 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1358 struct caam_request *caam_req = aead_request_ctx_dma(req); 1359 int ret; 1360 1361 /* allocate extended descriptor */ 1362 edesc = aead_edesc_alloc(req, false); 1363 if (IS_ERR(edesc)) 1364 return PTR_ERR(edesc); 1365 1366 caam_req->flc = &ctx->flc[DECRYPT]; 1367 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1368 caam_req->cbk = aead_decrypt_done; 1369 caam_req->ctx = &req->base; 1370 caam_req->edesc = edesc; 1371 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1372 if (ret != -EINPROGRESS && 1373 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1374 aead_unmap(ctx->dev, edesc, req); 1375 qi_cache_free(edesc); 1376 } 1377 1378 return ret; 1379 } 1380 1381 static int ipsec_gcm_encrypt(struct aead_request *req) 1382 { 1383 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1384 } 1385 1386 static int ipsec_gcm_decrypt(struct aead_request *req) 1387 { 1388 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1389 } 1390 1391 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1392 { 1393 struct crypto_async_request *areq = cbk_ctx; 1394 struct skcipher_request *req = skcipher_request_cast(areq); 1395 struct caam_request *req_ctx = to_caam_req(areq); 1396 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1397 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1398 struct skcipher_edesc *edesc = req_ctx->edesc; 1399 int ecode = 0; 1400 int ivsize = crypto_skcipher_ivsize(skcipher); 1401 1402 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1403 1404 if (unlikely(status)) 1405 ecode = caam_qi2_strstatus(ctx->dev, status); 1406 1407 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1408 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1409 edesc->src_nents > 1 ? 100 : ivsize, 1); 1410 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1411 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1412 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1413 1414 skcipher_unmap(ctx->dev, edesc, req); 1415 1416 /* 1417 * The crypto API expects us to set the IV (req->iv) to the last 1418 * ciphertext block (CBC mode) or last counter (CTR mode). 1419 * This is used e.g. by the CTS mode. 1420 */ 1421 if (!ecode) 1422 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1423 ivsize); 1424 1425 qi_cache_free(edesc); 1426 skcipher_request_complete(req, ecode); 1427 } 1428 1429 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1430 { 1431 struct crypto_async_request *areq = cbk_ctx; 1432 struct skcipher_request *req = skcipher_request_cast(areq); 1433 struct caam_request *req_ctx = to_caam_req(areq); 1434 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1435 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1436 struct skcipher_edesc *edesc = req_ctx->edesc; 1437 int ecode = 0; 1438 int ivsize = crypto_skcipher_ivsize(skcipher); 1439 1440 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1441 1442 if (unlikely(status)) 1443 ecode = caam_qi2_strstatus(ctx->dev, status); 1444 1445 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1446 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1447 edesc->src_nents > 1 ? 100 : ivsize, 1); 1448 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1449 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1450 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1451 1452 skcipher_unmap(ctx->dev, edesc, req); 1453 1454 /* 1455 * The crypto API expects us to set the IV (req->iv) to the last 1456 * ciphertext block (CBC mode) or last counter (CTR mode). 1457 * This is used e.g. by the CTS mode. 1458 */ 1459 if (!ecode) 1460 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1461 ivsize); 1462 1463 qi_cache_free(edesc); 1464 skcipher_request_complete(req, ecode); 1465 } 1466 1467 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1468 { 1469 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1470 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1471 1472 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1473 } 1474 1475 static int skcipher_encrypt(struct skcipher_request *req) 1476 { 1477 struct skcipher_edesc *edesc; 1478 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1479 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1480 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1481 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1482 int ret; 1483 1484 /* 1485 * XTS is expected to return an error even for input length = 0 1486 * Note that the case input length < block size will be caught during 1487 * HW offloading and return an error. 1488 */ 1489 if (!req->cryptlen && !ctx->fallback) 1490 return 0; 1491 1492 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1493 ctx->xts_key_fallback)) { 1494 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1495 skcipher_request_set_callback(&caam_req->fallback_req, 1496 req->base.flags, 1497 req->base.complete, 1498 req->base.data); 1499 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1500 req->dst, req->cryptlen, req->iv); 1501 1502 return crypto_skcipher_encrypt(&caam_req->fallback_req); 1503 } 1504 1505 /* allocate extended descriptor */ 1506 edesc = skcipher_edesc_alloc(req); 1507 if (IS_ERR(edesc)) 1508 return PTR_ERR(edesc); 1509 1510 caam_req->flc = &ctx->flc[ENCRYPT]; 1511 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1512 caam_req->cbk = skcipher_encrypt_done; 1513 caam_req->ctx = &req->base; 1514 caam_req->edesc = edesc; 1515 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1516 if (ret != -EINPROGRESS && 1517 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1518 skcipher_unmap(ctx->dev, edesc, req); 1519 qi_cache_free(edesc); 1520 } 1521 1522 return ret; 1523 } 1524 1525 static int skcipher_decrypt(struct skcipher_request *req) 1526 { 1527 struct skcipher_edesc *edesc; 1528 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1529 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1530 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1531 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1532 int ret; 1533 1534 /* 1535 * XTS is expected to return an error even for input length = 0 1536 * Note that the case input length < block size will be caught during 1537 * HW offloading and return an error. 1538 */ 1539 if (!req->cryptlen && !ctx->fallback) 1540 return 0; 1541 1542 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1543 ctx->xts_key_fallback)) { 1544 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1545 skcipher_request_set_callback(&caam_req->fallback_req, 1546 req->base.flags, 1547 req->base.complete, 1548 req->base.data); 1549 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1550 req->dst, req->cryptlen, req->iv); 1551 1552 return crypto_skcipher_decrypt(&caam_req->fallback_req); 1553 } 1554 1555 /* allocate extended descriptor */ 1556 edesc = skcipher_edesc_alloc(req); 1557 if (IS_ERR(edesc)) 1558 return PTR_ERR(edesc); 1559 1560 caam_req->flc = &ctx->flc[DECRYPT]; 1561 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1562 caam_req->cbk = skcipher_decrypt_done; 1563 caam_req->ctx = &req->base; 1564 caam_req->edesc = edesc; 1565 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1566 if (ret != -EINPROGRESS && 1567 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1568 skcipher_unmap(ctx->dev, edesc, req); 1569 qi_cache_free(edesc); 1570 } 1571 1572 return ret; 1573 } 1574 1575 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1576 bool uses_dkp) 1577 { 1578 dma_addr_t dma_addr; 1579 int i; 1580 1581 /* copy descriptor header template value */ 1582 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1583 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1584 1585 ctx->dev = caam->dev; 1586 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1587 1588 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1589 offsetof(struct caam_ctx, flc_dma), 1590 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1591 if (dma_mapping_error(ctx->dev, dma_addr)) { 1592 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1593 return -ENOMEM; 1594 } 1595 1596 for (i = 0; i < NUM_OP; i++) 1597 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1598 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1599 1600 return 0; 1601 } 1602 1603 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1604 { 1605 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1606 struct caam_skcipher_alg *caam_alg = 1607 container_of(alg, typeof(*caam_alg), skcipher); 1608 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1609 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 1610 int ret = 0; 1611 1612 if (alg_aai == OP_ALG_AAI_XTS) { 1613 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1614 struct crypto_skcipher *fallback; 1615 1616 fallback = crypto_alloc_skcipher(tfm_name, 0, 1617 CRYPTO_ALG_NEED_FALLBACK); 1618 if (IS_ERR(fallback)) { 1619 dev_err(caam_alg->caam.dev, 1620 "Failed to allocate %s fallback: %ld\n", 1621 tfm_name, PTR_ERR(fallback)); 1622 return PTR_ERR(fallback); 1623 } 1624 1625 ctx->fallback = fallback; 1626 crypto_skcipher_set_reqsize_dma( 1627 tfm, sizeof(struct caam_request) + 1628 crypto_skcipher_reqsize(fallback)); 1629 } else { 1630 crypto_skcipher_set_reqsize_dma(tfm, 1631 sizeof(struct caam_request)); 1632 } 1633 1634 ret = caam_cra_init(ctx, &caam_alg->caam, false); 1635 if (ret && ctx->fallback) 1636 crypto_free_skcipher(ctx->fallback); 1637 1638 return ret; 1639 } 1640 1641 static int caam_cra_init_aead(struct crypto_aead *tfm) 1642 { 1643 struct aead_alg *alg = crypto_aead_alg(tfm); 1644 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1645 aead); 1646 1647 crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request)); 1648 return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam, 1649 !caam_alg->caam.nodkp); 1650 } 1651 1652 static void caam_exit_common(struct caam_ctx *ctx) 1653 { 1654 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1655 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1656 DMA_ATTR_SKIP_CPU_SYNC); 1657 } 1658 1659 static void caam_cra_exit(struct crypto_skcipher *tfm) 1660 { 1661 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1662 1663 if (ctx->fallback) 1664 crypto_free_skcipher(ctx->fallback); 1665 caam_exit_common(ctx); 1666 } 1667 1668 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1669 { 1670 caam_exit_common(crypto_aead_ctx_dma(tfm)); 1671 } 1672 1673 static struct caam_skcipher_alg driver_algs[] = { 1674 { 1675 .skcipher = { 1676 .base = { 1677 .cra_name = "cbc(aes)", 1678 .cra_driver_name = "cbc-aes-caam-qi2", 1679 .cra_blocksize = AES_BLOCK_SIZE, 1680 }, 1681 .setkey = aes_skcipher_setkey, 1682 .encrypt = skcipher_encrypt, 1683 .decrypt = skcipher_decrypt, 1684 .min_keysize = AES_MIN_KEY_SIZE, 1685 .max_keysize = AES_MAX_KEY_SIZE, 1686 .ivsize = AES_BLOCK_SIZE, 1687 }, 1688 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1689 }, 1690 { 1691 .skcipher = { 1692 .base = { 1693 .cra_name = "cbc(des3_ede)", 1694 .cra_driver_name = "cbc-3des-caam-qi2", 1695 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1696 }, 1697 .setkey = des3_skcipher_setkey, 1698 .encrypt = skcipher_encrypt, 1699 .decrypt = skcipher_decrypt, 1700 .min_keysize = DES3_EDE_KEY_SIZE, 1701 .max_keysize = DES3_EDE_KEY_SIZE, 1702 .ivsize = DES3_EDE_BLOCK_SIZE, 1703 }, 1704 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1705 }, 1706 { 1707 .skcipher = { 1708 .base = { 1709 .cra_name = "cbc(des)", 1710 .cra_driver_name = "cbc-des-caam-qi2", 1711 .cra_blocksize = DES_BLOCK_SIZE, 1712 }, 1713 .setkey = des_skcipher_setkey, 1714 .encrypt = skcipher_encrypt, 1715 .decrypt = skcipher_decrypt, 1716 .min_keysize = DES_KEY_SIZE, 1717 .max_keysize = DES_KEY_SIZE, 1718 .ivsize = DES_BLOCK_SIZE, 1719 }, 1720 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1721 }, 1722 { 1723 .skcipher = { 1724 .base = { 1725 .cra_name = "ctr(aes)", 1726 .cra_driver_name = "ctr-aes-caam-qi2", 1727 .cra_blocksize = 1, 1728 }, 1729 .setkey = ctr_skcipher_setkey, 1730 .encrypt = skcipher_encrypt, 1731 .decrypt = skcipher_decrypt, 1732 .min_keysize = AES_MIN_KEY_SIZE, 1733 .max_keysize = AES_MAX_KEY_SIZE, 1734 .ivsize = AES_BLOCK_SIZE, 1735 .chunksize = AES_BLOCK_SIZE, 1736 }, 1737 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1738 OP_ALG_AAI_CTR_MOD128, 1739 }, 1740 { 1741 .skcipher = { 1742 .base = { 1743 .cra_name = "rfc3686(ctr(aes))", 1744 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1745 .cra_blocksize = 1, 1746 }, 1747 .setkey = rfc3686_skcipher_setkey, 1748 .encrypt = skcipher_encrypt, 1749 .decrypt = skcipher_decrypt, 1750 .min_keysize = AES_MIN_KEY_SIZE + 1751 CTR_RFC3686_NONCE_SIZE, 1752 .max_keysize = AES_MAX_KEY_SIZE + 1753 CTR_RFC3686_NONCE_SIZE, 1754 .ivsize = CTR_RFC3686_IV_SIZE, 1755 .chunksize = AES_BLOCK_SIZE, 1756 }, 1757 .caam = { 1758 .class1_alg_type = OP_ALG_ALGSEL_AES | 1759 OP_ALG_AAI_CTR_MOD128, 1760 .rfc3686 = true, 1761 }, 1762 }, 1763 { 1764 .skcipher = { 1765 .base = { 1766 .cra_name = "xts(aes)", 1767 .cra_driver_name = "xts-aes-caam-qi2", 1768 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1769 .cra_blocksize = AES_BLOCK_SIZE, 1770 }, 1771 .setkey = xts_skcipher_setkey, 1772 .encrypt = skcipher_encrypt, 1773 .decrypt = skcipher_decrypt, 1774 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1775 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1776 .ivsize = AES_BLOCK_SIZE, 1777 }, 1778 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1779 }, 1780 { 1781 .skcipher = { 1782 .base = { 1783 .cra_name = "chacha20", 1784 .cra_driver_name = "chacha20-caam-qi2", 1785 .cra_blocksize = 1, 1786 }, 1787 .setkey = chacha20_skcipher_setkey, 1788 .encrypt = skcipher_encrypt, 1789 .decrypt = skcipher_decrypt, 1790 .min_keysize = CHACHA_KEY_SIZE, 1791 .max_keysize = CHACHA_KEY_SIZE, 1792 .ivsize = CHACHA_IV_SIZE, 1793 }, 1794 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1795 }, 1796 }; 1797 1798 static struct caam_aead_alg driver_aeads[] = { 1799 { 1800 .aead = { 1801 .base = { 1802 .cra_name = "rfc4106(gcm(aes))", 1803 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1804 .cra_blocksize = 1, 1805 }, 1806 .setkey = rfc4106_setkey, 1807 .setauthsize = rfc4106_setauthsize, 1808 .encrypt = ipsec_gcm_encrypt, 1809 .decrypt = ipsec_gcm_decrypt, 1810 .ivsize = 8, 1811 .maxauthsize = AES_BLOCK_SIZE, 1812 }, 1813 .caam = { 1814 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1815 .nodkp = true, 1816 }, 1817 }, 1818 { 1819 .aead = { 1820 .base = { 1821 .cra_name = "rfc4543(gcm(aes))", 1822 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1823 .cra_blocksize = 1, 1824 }, 1825 .setkey = rfc4543_setkey, 1826 .setauthsize = rfc4543_setauthsize, 1827 .encrypt = ipsec_gcm_encrypt, 1828 .decrypt = ipsec_gcm_decrypt, 1829 .ivsize = 8, 1830 .maxauthsize = AES_BLOCK_SIZE, 1831 }, 1832 .caam = { 1833 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1834 .nodkp = true, 1835 }, 1836 }, 1837 /* Galois Counter Mode */ 1838 { 1839 .aead = { 1840 .base = { 1841 .cra_name = "gcm(aes)", 1842 .cra_driver_name = "gcm-aes-caam-qi2", 1843 .cra_blocksize = 1, 1844 }, 1845 .setkey = gcm_setkey, 1846 .setauthsize = gcm_setauthsize, 1847 .encrypt = aead_encrypt, 1848 .decrypt = aead_decrypt, 1849 .ivsize = 12, 1850 .maxauthsize = AES_BLOCK_SIZE, 1851 }, 1852 .caam = { 1853 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1854 .nodkp = true, 1855 } 1856 }, 1857 /* single-pass ipsec_esp descriptor */ 1858 { 1859 .aead = { 1860 .base = { 1861 .cra_name = "authenc(hmac(md5),cbc(aes))", 1862 .cra_driver_name = "authenc-hmac-md5-" 1863 "cbc-aes-caam-qi2", 1864 .cra_blocksize = AES_BLOCK_SIZE, 1865 }, 1866 .setkey = aead_setkey, 1867 .setauthsize = aead_setauthsize, 1868 .encrypt = aead_encrypt, 1869 .decrypt = aead_decrypt, 1870 .ivsize = AES_BLOCK_SIZE, 1871 .maxauthsize = MD5_DIGEST_SIZE, 1872 }, 1873 .caam = { 1874 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1875 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1876 OP_ALG_AAI_HMAC_PRECOMP, 1877 } 1878 }, 1879 { 1880 .aead = { 1881 .base = { 1882 .cra_name = "echainiv(authenc(hmac(md5)," 1883 "cbc(aes)))", 1884 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1885 "cbc-aes-caam-qi2", 1886 .cra_blocksize = AES_BLOCK_SIZE, 1887 }, 1888 .setkey = aead_setkey, 1889 .setauthsize = aead_setauthsize, 1890 .encrypt = aead_encrypt, 1891 .decrypt = aead_decrypt, 1892 .ivsize = AES_BLOCK_SIZE, 1893 .maxauthsize = MD5_DIGEST_SIZE, 1894 }, 1895 .caam = { 1896 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1897 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1898 OP_ALG_AAI_HMAC_PRECOMP, 1899 .geniv = true, 1900 } 1901 }, 1902 { 1903 .aead = { 1904 .base = { 1905 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1906 .cra_driver_name = "authenc-hmac-sha1-" 1907 "cbc-aes-caam-qi2", 1908 .cra_blocksize = AES_BLOCK_SIZE, 1909 }, 1910 .setkey = aead_setkey, 1911 .setauthsize = aead_setauthsize, 1912 .encrypt = aead_encrypt, 1913 .decrypt = aead_decrypt, 1914 .ivsize = AES_BLOCK_SIZE, 1915 .maxauthsize = SHA1_DIGEST_SIZE, 1916 }, 1917 .caam = { 1918 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1919 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1920 OP_ALG_AAI_HMAC_PRECOMP, 1921 } 1922 }, 1923 { 1924 .aead = { 1925 .base = { 1926 .cra_name = "echainiv(authenc(hmac(sha1)," 1927 "cbc(aes)))", 1928 .cra_driver_name = "echainiv-authenc-" 1929 "hmac-sha1-cbc-aes-caam-qi2", 1930 .cra_blocksize = AES_BLOCK_SIZE, 1931 }, 1932 .setkey = aead_setkey, 1933 .setauthsize = aead_setauthsize, 1934 .encrypt = aead_encrypt, 1935 .decrypt = aead_decrypt, 1936 .ivsize = AES_BLOCK_SIZE, 1937 .maxauthsize = SHA1_DIGEST_SIZE, 1938 }, 1939 .caam = { 1940 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1941 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1942 OP_ALG_AAI_HMAC_PRECOMP, 1943 .geniv = true, 1944 }, 1945 }, 1946 { 1947 .aead = { 1948 .base = { 1949 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1950 .cra_driver_name = "authenc-hmac-sha224-" 1951 "cbc-aes-caam-qi2", 1952 .cra_blocksize = AES_BLOCK_SIZE, 1953 }, 1954 .setkey = aead_setkey, 1955 .setauthsize = aead_setauthsize, 1956 .encrypt = aead_encrypt, 1957 .decrypt = aead_decrypt, 1958 .ivsize = AES_BLOCK_SIZE, 1959 .maxauthsize = SHA224_DIGEST_SIZE, 1960 }, 1961 .caam = { 1962 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1963 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1964 OP_ALG_AAI_HMAC_PRECOMP, 1965 } 1966 }, 1967 { 1968 .aead = { 1969 .base = { 1970 .cra_name = "echainiv(authenc(hmac(sha224)," 1971 "cbc(aes)))", 1972 .cra_driver_name = "echainiv-authenc-" 1973 "hmac-sha224-cbc-aes-caam-qi2", 1974 .cra_blocksize = AES_BLOCK_SIZE, 1975 }, 1976 .setkey = aead_setkey, 1977 .setauthsize = aead_setauthsize, 1978 .encrypt = aead_encrypt, 1979 .decrypt = aead_decrypt, 1980 .ivsize = AES_BLOCK_SIZE, 1981 .maxauthsize = SHA224_DIGEST_SIZE, 1982 }, 1983 .caam = { 1984 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1985 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1986 OP_ALG_AAI_HMAC_PRECOMP, 1987 .geniv = true, 1988 } 1989 }, 1990 { 1991 .aead = { 1992 .base = { 1993 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1994 .cra_driver_name = "authenc-hmac-sha256-" 1995 "cbc-aes-caam-qi2", 1996 .cra_blocksize = AES_BLOCK_SIZE, 1997 }, 1998 .setkey = aead_setkey, 1999 .setauthsize = aead_setauthsize, 2000 .encrypt = aead_encrypt, 2001 .decrypt = aead_decrypt, 2002 .ivsize = AES_BLOCK_SIZE, 2003 .maxauthsize = SHA256_DIGEST_SIZE, 2004 }, 2005 .caam = { 2006 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2007 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2008 OP_ALG_AAI_HMAC_PRECOMP, 2009 } 2010 }, 2011 { 2012 .aead = { 2013 .base = { 2014 .cra_name = "echainiv(authenc(hmac(sha256)," 2015 "cbc(aes)))", 2016 .cra_driver_name = "echainiv-authenc-" 2017 "hmac-sha256-cbc-aes-" 2018 "caam-qi2", 2019 .cra_blocksize = AES_BLOCK_SIZE, 2020 }, 2021 .setkey = aead_setkey, 2022 .setauthsize = aead_setauthsize, 2023 .encrypt = aead_encrypt, 2024 .decrypt = aead_decrypt, 2025 .ivsize = AES_BLOCK_SIZE, 2026 .maxauthsize = SHA256_DIGEST_SIZE, 2027 }, 2028 .caam = { 2029 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2030 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2031 OP_ALG_AAI_HMAC_PRECOMP, 2032 .geniv = true, 2033 } 2034 }, 2035 { 2036 .aead = { 2037 .base = { 2038 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2039 .cra_driver_name = "authenc-hmac-sha384-" 2040 "cbc-aes-caam-qi2", 2041 .cra_blocksize = AES_BLOCK_SIZE, 2042 }, 2043 .setkey = aead_setkey, 2044 .setauthsize = aead_setauthsize, 2045 .encrypt = aead_encrypt, 2046 .decrypt = aead_decrypt, 2047 .ivsize = AES_BLOCK_SIZE, 2048 .maxauthsize = SHA384_DIGEST_SIZE, 2049 }, 2050 .caam = { 2051 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2052 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2053 OP_ALG_AAI_HMAC_PRECOMP, 2054 } 2055 }, 2056 { 2057 .aead = { 2058 .base = { 2059 .cra_name = "echainiv(authenc(hmac(sha384)," 2060 "cbc(aes)))", 2061 .cra_driver_name = "echainiv-authenc-" 2062 "hmac-sha384-cbc-aes-" 2063 "caam-qi2", 2064 .cra_blocksize = AES_BLOCK_SIZE, 2065 }, 2066 .setkey = aead_setkey, 2067 .setauthsize = aead_setauthsize, 2068 .encrypt = aead_encrypt, 2069 .decrypt = aead_decrypt, 2070 .ivsize = AES_BLOCK_SIZE, 2071 .maxauthsize = SHA384_DIGEST_SIZE, 2072 }, 2073 .caam = { 2074 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2075 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2076 OP_ALG_AAI_HMAC_PRECOMP, 2077 .geniv = true, 2078 } 2079 }, 2080 { 2081 .aead = { 2082 .base = { 2083 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2084 .cra_driver_name = "authenc-hmac-sha512-" 2085 "cbc-aes-caam-qi2", 2086 .cra_blocksize = AES_BLOCK_SIZE, 2087 }, 2088 .setkey = aead_setkey, 2089 .setauthsize = aead_setauthsize, 2090 .encrypt = aead_encrypt, 2091 .decrypt = aead_decrypt, 2092 .ivsize = AES_BLOCK_SIZE, 2093 .maxauthsize = SHA512_DIGEST_SIZE, 2094 }, 2095 .caam = { 2096 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2097 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2098 OP_ALG_AAI_HMAC_PRECOMP, 2099 } 2100 }, 2101 { 2102 .aead = { 2103 .base = { 2104 .cra_name = "echainiv(authenc(hmac(sha512)," 2105 "cbc(aes)))", 2106 .cra_driver_name = "echainiv-authenc-" 2107 "hmac-sha512-cbc-aes-" 2108 "caam-qi2", 2109 .cra_blocksize = AES_BLOCK_SIZE, 2110 }, 2111 .setkey = aead_setkey, 2112 .setauthsize = aead_setauthsize, 2113 .encrypt = aead_encrypt, 2114 .decrypt = aead_decrypt, 2115 .ivsize = AES_BLOCK_SIZE, 2116 .maxauthsize = SHA512_DIGEST_SIZE, 2117 }, 2118 .caam = { 2119 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2120 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2121 OP_ALG_AAI_HMAC_PRECOMP, 2122 .geniv = true, 2123 } 2124 }, 2125 { 2126 .aead = { 2127 .base = { 2128 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2129 .cra_driver_name = "authenc-hmac-md5-" 2130 "cbc-des3_ede-caam-qi2", 2131 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2132 }, 2133 .setkey = des3_aead_setkey, 2134 .setauthsize = aead_setauthsize, 2135 .encrypt = aead_encrypt, 2136 .decrypt = aead_decrypt, 2137 .ivsize = DES3_EDE_BLOCK_SIZE, 2138 .maxauthsize = MD5_DIGEST_SIZE, 2139 }, 2140 .caam = { 2141 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2142 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2143 OP_ALG_AAI_HMAC_PRECOMP, 2144 } 2145 }, 2146 { 2147 .aead = { 2148 .base = { 2149 .cra_name = "echainiv(authenc(hmac(md5)," 2150 "cbc(des3_ede)))", 2151 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2152 "cbc-des3_ede-caam-qi2", 2153 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2154 }, 2155 .setkey = des3_aead_setkey, 2156 .setauthsize = aead_setauthsize, 2157 .encrypt = aead_encrypt, 2158 .decrypt = aead_decrypt, 2159 .ivsize = DES3_EDE_BLOCK_SIZE, 2160 .maxauthsize = MD5_DIGEST_SIZE, 2161 }, 2162 .caam = { 2163 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2164 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2165 OP_ALG_AAI_HMAC_PRECOMP, 2166 .geniv = true, 2167 } 2168 }, 2169 { 2170 .aead = { 2171 .base = { 2172 .cra_name = "authenc(hmac(sha1)," 2173 "cbc(des3_ede))", 2174 .cra_driver_name = "authenc-hmac-sha1-" 2175 "cbc-des3_ede-caam-qi2", 2176 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2177 }, 2178 .setkey = des3_aead_setkey, 2179 .setauthsize = aead_setauthsize, 2180 .encrypt = aead_encrypt, 2181 .decrypt = aead_decrypt, 2182 .ivsize = DES3_EDE_BLOCK_SIZE, 2183 .maxauthsize = SHA1_DIGEST_SIZE, 2184 }, 2185 .caam = { 2186 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2187 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2188 OP_ALG_AAI_HMAC_PRECOMP, 2189 }, 2190 }, 2191 { 2192 .aead = { 2193 .base = { 2194 .cra_name = "echainiv(authenc(hmac(sha1)," 2195 "cbc(des3_ede)))", 2196 .cra_driver_name = "echainiv-authenc-" 2197 "hmac-sha1-" 2198 "cbc-des3_ede-caam-qi2", 2199 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2200 }, 2201 .setkey = des3_aead_setkey, 2202 .setauthsize = aead_setauthsize, 2203 .encrypt = aead_encrypt, 2204 .decrypt = aead_decrypt, 2205 .ivsize = DES3_EDE_BLOCK_SIZE, 2206 .maxauthsize = SHA1_DIGEST_SIZE, 2207 }, 2208 .caam = { 2209 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2210 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2211 OP_ALG_AAI_HMAC_PRECOMP, 2212 .geniv = true, 2213 } 2214 }, 2215 { 2216 .aead = { 2217 .base = { 2218 .cra_name = "authenc(hmac(sha224)," 2219 "cbc(des3_ede))", 2220 .cra_driver_name = "authenc-hmac-sha224-" 2221 "cbc-des3_ede-caam-qi2", 2222 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2223 }, 2224 .setkey = des3_aead_setkey, 2225 .setauthsize = aead_setauthsize, 2226 .encrypt = aead_encrypt, 2227 .decrypt = aead_decrypt, 2228 .ivsize = DES3_EDE_BLOCK_SIZE, 2229 .maxauthsize = SHA224_DIGEST_SIZE, 2230 }, 2231 .caam = { 2232 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2233 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2234 OP_ALG_AAI_HMAC_PRECOMP, 2235 }, 2236 }, 2237 { 2238 .aead = { 2239 .base = { 2240 .cra_name = "echainiv(authenc(hmac(sha224)," 2241 "cbc(des3_ede)))", 2242 .cra_driver_name = "echainiv-authenc-" 2243 "hmac-sha224-" 2244 "cbc-des3_ede-caam-qi2", 2245 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2246 }, 2247 .setkey = des3_aead_setkey, 2248 .setauthsize = aead_setauthsize, 2249 .encrypt = aead_encrypt, 2250 .decrypt = aead_decrypt, 2251 .ivsize = DES3_EDE_BLOCK_SIZE, 2252 .maxauthsize = SHA224_DIGEST_SIZE, 2253 }, 2254 .caam = { 2255 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2256 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2257 OP_ALG_AAI_HMAC_PRECOMP, 2258 .geniv = true, 2259 } 2260 }, 2261 { 2262 .aead = { 2263 .base = { 2264 .cra_name = "authenc(hmac(sha256)," 2265 "cbc(des3_ede))", 2266 .cra_driver_name = "authenc-hmac-sha256-" 2267 "cbc-des3_ede-caam-qi2", 2268 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2269 }, 2270 .setkey = des3_aead_setkey, 2271 .setauthsize = aead_setauthsize, 2272 .encrypt = aead_encrypt, 2273 .decrypt = aead_decrypt, 2274 .ivsize = DES3_EDE_BLOCK_SIZE, 2275 .maxauthsize = SHA256_DIGEST_SIZE, 2276 }, 2277 .caam = { 2278 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2279 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2280 OP_ALG_AAI_HMAC_PRECOMP, 2281 }, 2282 }, 2283 { 2284 .aead = { 2285 .base = { 2286 .cra_name = "echainiv(authenc(hmac(sha256)," 2287 "cbc(des3_ede)))", 2288 .cra_driver_name = "echainiv-authenc-" 2289 "hmac-sha256-" 2290 "cbc-des3_ede-caam-qi2", 2291 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2292 }, 2293 .setkey = des3_aead_setkey, 2294 .setauthsize = aead_setauthsize, 2295 .encrypt = aead_encrypt, 2296 .decrypt = aead_decrypt, 2297 .ivsize = DES3_EDE_BLOCK_SIZE, 2298 .maxauthsize = SHA256_DIGEST_SIZE, 2299 }, 2300 .caam = { 2301 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2302 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2303 OP_ALG_AAI_HMAC_PRECOMP, 2304 .geniv = true, 2305 } 2306 }, 2307 { 2308 .aead = { 2309 .base = { 2310 .cra_name = "authenc(hmac(sha384)," 2311 "cbc(des3_ede))", 2312 .cra_driver_name = "authenc-hmac-sha384-" 2313 "cbc-des3_ede-caam-qi2", 2314 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2315 }, 2316 .setkey = des3_aead_setkey, 2317 .setauthsize = aead_setauthsize, 2318 .encrypt = aead_encrypt, 2319 .decrypt = aead_decrypt, 2320 .ivsize = DES3_EDE_BLOCK_SIZE, 2321 .maxauthsize = SHA384_DIGEST_SIZE, 2322 }, 2323 .caam = { 2324 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2325 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2326 OP_ALG_AAI_HMAC_PRECOMP, 2327 }, 2328 }, 2329 { 2330 .aead = { 2331 .base = { 2332 .cra_name = "echainiv(authenc(hmac(sha384)," 2333 "cbc(des3_ede)))", 2334 .cra_driver_name = "echainiv-authenc-" 2335 "hmac-sha384-" 2336 "cbc-des3_ede-caam-qi2", 2337 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2338 }, 2339 .setkey = des3_aead_setkey, 2340 .setauthsize = aead_setauthsize, 2341 .encrypt = aead_encrypt, 2342 .decrypt = aead_decrypt, 2343 .ivsize = DES3_EDE_BLOCK_SIZE, 2344 .maxauthsize = SHA384_DIGEST_SIZE, 2345 }, 2346 .caam = { 2347 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2348 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2349 OP_ALG_AAI_HMAC_PRECOMP, 2350 .geniv = true, 2351 } 2352 }, 2353 { 2354 .aead = { 2355 .base = { 2356 .cra_name = "authenc(hmac(sha512)," 2357 "cbc(des3_ede))", 2358 .cra_driver_name = "authenc-hmac-sha512-" 2359 "cbc-des3_ede-caam-qi2", 2360 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2361 }, 2362 .setkey = des3_aead_setkey, 2363 .setauthsize = aead_setauthsize, 2364 .encrypt = aead_encrypt, 2365 .decrypt = aead_decrypt, 2366 .ivsize = DES3_EDE_BLOCK_SIZE, 2367 .maxauthsize = SHA512_DIGEST_SIZE, 2368 }, 2369 .caam = { 2370 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2371 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2372 OP_ALG_AAI_HMAC_PRECOMP, 2373 }, 2374 }, 2375 { 2376 .aead = { 2377 .base = { 2378 .cra_name = "echainiv(authenc(hmac(sha512)," 2379 "cbc(des3_ede)))", 2380 .cra_driver_name = "echainiv-authenc-" 2381 "hmac-sha512-" 2382 "cbc-des3_ede-caam-qi2", 2383 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2384 }, 2385 .setkey = des3_aead_setkey, 2386 .setauthsize = aead_setauthsize, 2387 .encrypt = aead_encrypt, 2388 .decrypt = aead_decrypt, 2389 .ivsize = DES3_EDE_BLOCK_SIZE, 2390 .maxauthsize = SHA512_DIGEST_SIZE, 2391 }, 2392 .caam = { 2393 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2394 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2395 OP_ALG_AAI_HMAC_PRECOMP, 2396 .geniv = true, 2397 } 2398 }, 2399 { 2400 .aead = { 2401 .base = { 2402 .cra_name = "authenc(hmac(md5),cbc(des))", 2403 .cra_driver_name = "authenc-hmac-md5-" 2404 "cbc-des-caam-qi2", 2405 .cra_blocksize = DES_BLOCK_SIZE, 2406 }, 2407 .setkey = aead_setkey, 2408 .setauthsize = aead_setauthsize, 2409 .encrypt = aead_encrypt, 2410 .decrypt = aead_decrypt, 2411 .ivsize = DES_BLOCK_SIZE, 2412 .maxauthsize = MD5_DIGEST_SIZE, 2413 }, 2414 .caam = { 2415 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2416 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2417 OP_ALG_AAI_HMAC_PRECOMP, 2418 }, 2419 }, 2420 { 2421 .aead = { 2422 .base = { 2423 .cra_name = "echainiv(authenc(hmac(md5)," 2424 "cbc(des)))", 2425 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2426 "cbc-des-caam-qi2", 2427 .cra_blocksize = DES_BLOCK_SIZE, 2428 }, 2429 .setkey = aead_setkey, 2430 .setauthsize = aead_setauthsize, 2431 .encrypt = aead_encrypt, 2432 .decrypt = aead_decrypt, 2433 .ivsize = DES_BLOCK_SIZE, 2434 .maxauthsize = MD5_DIGEST_SIZE, 2435 }, 2436 .caam = { 2437 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2438 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2439 OP_ALG_AAI_HMAC_PRECOMP, 2440 .geniv = true, 2441 } 2442 }, 2443 { 2444 .aead = { 2445 .base = { 2446 .cra_name = "authenc(hmac(sha1),cbc(des))", 2447 .cra_driver_name = "authenc-hmac-sha1-" 2448 "cbc-des-caam-qi2", 2449 .cra_blocksize = DES_BLOCK_SIZE, 2450 }, 2451 .setkey = aead_setkey, 2452 .setauthsize = aead_setauthsize, 2453 .encrypt = aead_encrypt, 2454 .decrypt = aead_decrypt, 2455 .ivsize = DES_BLOCK_SIZE, 2456 .maxauthsize = SHA1_DIGEST_SIZE, 2457 }, 2458 .caam = { 2459 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2460 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2461 OP_ALG_AAI_HMAC_PRECOMP, 2462 }, 2463 }, 2464 { 2465 .aead = { 2466 .base = { 2467 .cra_name = "echainiv(authenc(hmac(sha1)," 2468 "cbc(des)))", 2469 .cra_driver_name = "echainiv-authenc-" 2470 "hmac-sha1-cbc-des-caam-qi2", 2471 .cra_blocksize = DES_BLOCK_SIZE, 2472 }, 2473 .setkey = aead_setkey, 2474 .setauthsize = aead_setauthsize, 2475 .encrypt = aead_encrypt, 2476 .decrypt = aead_decrypt, 2477 .ivsize = DES_BLOCK_SIZE, 2478 .maxauthsize = SHA1_DIGEST_SIZE, 2479 }, 2480 .caam = { 2481 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2482 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2483 OP_ALG_AAI_HMAC_PRECOMP, 2484 .geniv = true, 2485 } 2486 }, 2487 { 2488 .aead = { 2489 .base = { 2490 .cra_name = "authenc(hmac(sha224),cbc(des))", 2491 .cra_driver_name = "authenc-hmac-sha224-" 2492 "cbc-des-caam-qi2", 2493 .cra_blocksize = DES_BLOCK_SIZE, 2494 }, 2495 .setkey = aead_setkey, 2496 .setauthsize = aead_setauthsize, 2497 .encrypt = aead_encrypt, 2498 .decrypt = aead_decrypt, 2499 .ivsize = DES_BLOCK_SIZE, 2500 .maxauthsize = SHA224_DIGEST_SIZE, 2501 }, 2502 .caam = { 2503 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2504 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2505 OP_ALG_AAI_HMAC_PRECOMP, 2506 }, 2507 }, 2508 { 2509 .aead = { 2510 .base = { 2511 .cra_name = "echainiv(authenc(hmac(sha224)," 2512 "cbc(des)))", 2513 .cra_driver_name = "echainiv-authenc-" 2514 "hmac-sha224-cbc-des-" 2515 "caam-qi2", 2516 .cra_blocksize = DES_BLOCK_SIZE, 2517 }, 2518 .setkey = aead_setkey, 2519 .setauthsize = aead_setauthsize, 2520 .encrypt = aead_encrypt, 2521 .decrypt = aead_decrypt, 2522 .ivsize = DES_BLOCK_SIZE, 2523 .maxauthsize = SHA224_DIGEST_SIZE, 2524 }, 2525 .caam = { 2526 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2527 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2528 OP_ALG_AAI_HMAC_PRECOMP, 2529 .geniv = true, 2530 } 2531 }, 2532 { 2533 .aead = { 2534 .base = { 2535 .cra_name = "authenc(hmac(sha256),cbc(des))", 2536 .cra_driver_name = "authenc-hmac-sha256-" 2537 "cbc-des-caam-qi2", 2538 .cra_blocksize = DES_BLOCK_SIZE, 2539 }, 2540 .setkey = aead_setkey, 2541 .setauthsize = aead_setauthsize, 2542 .encrypt = aead_encrypt, 2543 .decrypt = aead_decrypt, 2544 .ivsize = DES_BLOCK_SIZE, 2545 .maxauthsize = SHA256_DIGEST_SIZE, 2546 }, 2547 .caam = { 2548 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2549 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2550 OP_ALG_AAI_HMAC_PRECOMP, 2551 }, 2552 }, 2553 { 2554 .aead = { 2555 .base = { 2556 .cra_name = "echainiv(authenc(hmac(sha256)," 2557 "cbc(des)))", 2558 .cra_driver_name = "echainiv-authenc-" 2559 "hmac-sha256-cbc-des-" 2560 "caam-qi2", 2561 .cra_blocksize = DES_BLOCK_SIZE, 2562 }, 2563 .setkey = aead_setkey, 2564 .setauthsize = aead_setauthsize, 2565 .encrypt = aead_encrypt, 2566 .decrypt = aead_decrypt, 2567 .ivsize = DES_BLOCK_SIZE, 2568 .maxauthsize = SHA256_DIGEST_SIZE, 2569 }, 2570 .caam = { 2571 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2572 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2573 OP_ALG_AAI_HMAC_PRECOMP, 2574 .geniv = true, 2575 }, 2576 }, 2577 { 2578 .aead = { 2579 .base = { 2580 .cra_name = "authenc(hmac(sha384),cbc(des))", 2581 .cra_driver_name = "authenc-hmac-sha384-" 2582 "cbc-des-caam-qi2", 2583 .cra_blocksize = DES_BLOCK_SIZE, 2584 }, 2585 .setkey = aead_setkey, 2586 .setauthsize = aead_setauthsize, 2587 .encrypt = aead_encrypt, 2588 .decrypt = aead_decrypt, 2589 .ivsize = DES_BLOCK_SIZE, 2590 .maxauthsize = SHA384_DIGEST_SIZE, 2591 }, 2592 .caam = { 2593 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2594 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2595 OP_ALG_AAI_HMAC_PRECOMP, 2596 }, 2597 }, 2598 { 2599 .aead = { 2600 .base = { 2601 .cra_name = "echainiv(authenc(hmac(sha384)," 2602 "cbc(des)))", 2603 .cra_driver_name = "echainiv-authenc-" 2604 "hmac-sha384-cbc-des-" 2605 "caam-qi2", 2606 .cra_blocksize = DES_BLOCK_SIZE, 2607 }, 2608 .setkey = aead_setkey, 2609 .setauthsize = aead_setauthsize, 2610 .encrypt = aead_encrypt, 2611 .decrypt = aead_decrypt, 2612 .ivsize = DES_BLOCK_SIZE, 2613 .maxauthsize = SHA384_DIGEST_SIZE, 2614 }, 2615 .caam = { 2616 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2617 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2618 OP_ALG_AAI_HMAC_PRECOMP, 2619 .geniv = true, 2620 } 2621 }, 2622 { 2623 .aead = { 2624 .base = { 2625 .cra_name = "authenc(hmac(sha512),cbc(des))", 2626 .cra_driver_name = "authenc-hmac-sha512-" 2627 "cbc-des-caam-qi2", 2628 .cra_blocksize = DES_BLOCK_SIZE, 2629 }, 2630 .setkey = aead_setkey, 2631 .setauthsize = aead_setauthsize, 2632 .encrypt = aead_encrypt, 2633 .decrypt = aead_decrypt, 2634 .ivsize = DES_BLOCK_SIZE, 2635 .maxauthsize = SHA512_DIGEST_SIZE, 2636 }, 2637 .caam = { 2638 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2639 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2640 OP_ALG_AAI_HMAC_PRECOMP, 2641 } 2642 }, 2643 { 2644 .aead = { 2645 .base = { 2646 .cra_name = "echainiv(authenc(hmac(sha512)," 2647 "cbc(des)))", 2648 .cra_driver_name = "echainiv-authenc-" 2649 "hmac-sha512-cbc-des-" 2650 "caam-qi2", 2651 .cra_blocksize = DES_BLOCK_SIZE, 2652 }, 2653 .setkey = aead_setkey, 2654 .setauthsize = aead_setauthsize, 2655 .encrypt = aead_encrypt, 2656 .decrypt = aead_decrypt, 2657 .ivsize = DES_BLOCK_SIZE, 2658 .maxauthsize = SHA512_DIGEST_SIZE, 2659 }, 2660 .caam = { 2661 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2662 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2663 OP_ALG_AAI_HMAC_PRECOMP, 2664 .geniv = true, 2665 } 2666 }, 2667 { 2668 .aead = { 2669 .base = { 2670 .cra_name = "authenc(hmac(md5)," 2671 "rfc3686(ctr(aes)))", 2672 .cra_driver_name = "authenc-hmac-md5-" 2673 "rfc3686-ctr-aes-caam-qi2", 2674 .cra_blocksize = 1, 2675 }, 2676 .setkey = aead_setkey, 2677 .setauthsize = aead_setauthsize, 2678 .encrypt = aead_encrypt, 2679 .decrypt = aead_decrypt, 2680 .ivsize = CTR_RFC3686_IV_SIZE, 2681 .maxauthsize = MD5_DIGEST_SIZE, 2682 }, 2683 .caam = { 2684 .class1_alg_type = OP_ALG_ALGSEL_AES | 2685 OP_ALG_AAI_CTR_MOD128, 2686 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2687 OP_ALG_AAI_HMAC_PRECOMP, 2688 .rfc3686 = true, 2689 }, 2690 }, 2691 { 2692 .aead = { 2693 .base = { 2694 .cra_name = "seqiv(authenc(" 2695 "hmac(md5),rfc3686(ctr(aes))))", 2696 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2697 "rfc3686-ctr-aes-caam-qi2", 2698 .cra_blocksize = 1, 2699 }, 2700 .setkey = aead_setkey, 2701 .setauthsize = aead_setauthsize, 2702 .encrypt = aead_encrypt, 2703 .decrypt = aead_decrypt, 2704 .ivsize = CTR_RFC3686_IV_SIZE, 2705 .maxauthsize = MD5_DIGEST_SIZE, 2706 }, 2707 .caam = { 2708 .class1_alg_type = OP_ALG_ALGSEL_AES | 2709 OP_ALG_AAI_CTR_MOD128, 2710 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2711 OP_ALG_AAI_HMAC_PRECOMP, 2712 .rfc3686 = true, 2713 .geniv = true, 2714 }, 2715 }, 2716 { 2717 .aead = { 2718 .base = { 2719 .cra_name = "authenc(hmac(sha1)," 2720 "rfc3686(ctr(aes)))", 2721 .cra_driver_name = "authenc-hmac-sha1-" 2722 "rfc3686-ctr-aes-caam-qi2", 2723 .cra_blocksize = 1, 2724 }, 2725 .setkey = aead_setkey, 2726 .setauthsize = aead_setauthsize, 2727 .encrypt = aead_encrypt, 2728 .decrypt = aead_decrypt, 2729 .ivsize = CTR_RFC3686_IV_SIZE, 2730 .maxauthsize = SHA1_DIGEST_SIZE, 2731 }, 2732 .caam = { 2733 .class1_alg_type = OP_ALG_ALGSEL_AES | 2734 OP_ALG_AAI_CTR_MOD128, 2735 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2736 OP_ALG_AAI_HMAC_PRECOMP, 2737 .rfc3686 = true, 2738 }, 2739 }, 2740 { 2741 .aead = { 2742 .base = { 2743 .cra_name = "seqiv(authenc(" 2744 "hmac(sha1),rfc3686(ctr(aes))))", 2745 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2746 "rfc3686-ctr-aes-caam-qi2", 2747 .cra_blocksize = 1, 2748 }, 2749 .setkey = aead_setkey, 2750 .setauthsize = aead_setauthsize, 2751 .encrypt = aead_encrypt, 2752 .decrypt = aead_decrypt, 2753 .ivsize = CTR_RFC3686_IV_SIZE, 2754 .maxauthsize = SHA1_DIGEST_SIZE, 2755 }, 2756 .caam = { 2757 .class1_alg_type = OP_ALG_ALGSEL_AES | 2758 OP_ALG_AAI_CTR_MOD128, 2759 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2760 OP_ALG_AAI_HMAC_PRECOMP, 2761 .rfc3686 = true, 2762 .geniv = true, 2763 }, 2764 }, 2765 { 2766 .aead = { 2767 .base = { 2768 .cra_name = "authenc(hmac(sha224)," 2769 "rfc3686(ctr(aes)))", 2770 .cra_driver_name = "authenc-hmac-sha224-" 2771 "rfc3686-ctr-aes-caam-qi2", 2772 .cra_blocksize = 1, 2773 }, 2774 .setkey = aead_setkey, 2775 .setauthsize = aead_setauthsize, 2776 .encrypt = aead_encrypt, 2777 .decrypt = aead_decrypt, 2778 .ivsize = CTR_RFC3686_IV_SIZE, 2779 .maxauthsize = SHA224_DIGEST_SIZE, 2780 }, 2781 .caam = { 2782 .class1_alg_type = OP_ALG_ALGSEL_AES | 2783 OP_ALG_AAI_CTR_MOD128, 2784 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2785 OP_ALG_AAI_HMAC_PRECOMP, 2786 .rfc3686 = true, 2787 }, 2788 }, 2789 { 2790 .aead = { 2791 .base = { 2792 .cra_name = "seqiv(authenc(" 2793 "hmac(sha224),rfc3686(ctr(aes))))", 2794 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2795 "rfc3686-ctr-aes-caam-qi2", 2796 .cra_blocksize = 1, 2797 }, 2798 .setkey = aead_setkey, 2799 .setauthsize = aead_setauthsize, 2800 .encrypt = aead_encrypt, 2801 .decrypt = aead_decrypt, 2802 .ivsize = CTR_RFC3686_IV_SIZE, 2803 .maxauthsize = SHA224_DIGEST_SIZE, 2804 }, 2805 .caam = { 2806 .class1_alg_type = OP_ALG_ALGSEL_AES | 2807 OP_ALG_AAI_CTR_MOD128, 2808 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2809 OP_ALG_AAI_HMAC_PRECOMP, 2810 .rfc3686 = true, 2811 .geniv = true, 2812 }, 2813 }, 2814 { 2815 .aead = { 2816 .base = { 2817 .cra_name = "authenc(hmac(sha256)," 2818 "rfc3686(ctr(aes)))", 2819 .cra_driver_name = "authenc-hmac-sha256-" 2820 "rfc3686-ctr-aes-caam-qi2", 2821 .cra_blocksize = 1, 2822 }, 2823 .setkey = aead_setkey, 2824 .setauthsize = aead_setauthsize, 2825 .encrypt = aead_encrypt, 2826 .decrypt = aead_decrypt, 2827 .ivsize = CTR_RFC3686_IV_SIZE, 2828 .maxauthsize = SHA256_DIGEST_SIZE, 2829 }, 2830 .caam = { 2831 .class1_alg_type = OP_ALG_ALGSEL_AES | 2832 OP_ALG_AAI_CTR_MOD128, 2833 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2834 OP_ALG_AAI_HMAC_PRECOMP, 2835 .rfc3686 = true, 2836 }, 2837 }, 2838 { 2839 .aead = { 2840 .base = { 2841 .cra_name = "seqiv(authenc(hmac(sha256)," 2842 "rfc3686(ctr(aes))))", 2843 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2844 "rfc3686-ctr-aes-caam-qi2", 2845 .cra_blocksize = 1, 2846 }, 2847 .setkey = aead_setkey, 2848 .setauthsize = aead_setauthsize, 2849 .encrypt = aead_encrypt, 2850 .decrypt = aead_decrypt, 2851 .ivsize = CTR_RFC3686_IV_SIZE, 2852 .maxauthsize = SHA256_DIGEST_SIZE, 2853 }, 2854 .caam = { 2855 .class1_alg_type = OP_ALG_ALGSEL_AES | 2856 OP_ALG_AAI_CTR_MOD128, 2857 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2858 OP_ALG_AAI_HMAC_PRECOMP, 2859 .rfc3686 = true, 2860 .geniv = true, 2861 }, 2862 }, 2863 { 2864 .aead = { 2865 .base = { 2866 .cra_name = "authenc(hmac(sha384)," 2867 "rfc3686(ctr(aes)))", 2868 .cra_driver_name = "authenc-hmac-sha384-" 2869 "rfc3686-ctr-aes-caam-qi2", 2870 .cra_blocksize = 1, 2871 }, 2872 .setkey = aead_setkey, 2873 .setauthsize = aead_setauthsize, 2874 .encrypt = aead_encrypt, 2875 .decrypt = aead_decrypt, 2876 .ivsize = CTR_RFC3686_IV_SIZE, 2877 .maxauthsize = SHA384_DIGEST_SIZE, 2878 }, 2879 .caam = { 2880 .class1_alg_type = OP_ALG_ALGSEL_AES | 2881 OP_ALG_AAI_CTR_MOD128, 2882 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2883 OP_ALG_AAI_HMAC_PRECOMP, 2884 .rfc3686 = true, 2885 }, 2886 }, 2887 { 2888 .aead = { 2889 .base = { 2890 .cra_name = "seqiv(authenc(hmac(sha384)," 2891 "rfc3686(ctr(aes))))", 2892 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2893 "rfc3686-ctr-aes-caam-qi2", 2894 .cra_blocksize = 1, 2895 }, 2896 .setkey = aead_setkey, 2897 .setauthsize = aead_setauthsize, 2898 .encrypt = aead_encrypt, 2899 .decrypt = aead_decrypt, 2900 .ivsize = CTR_RFC3686_IV_SIZE, 2901 .maxauthsize = SHA384_DIGEST_SIZE, 2902 }, 2903 .caam = { 2904 .class1_alg_type = OP_ALG_ALGSEL_AES | 2905 OP_ALG_AAI_CTR_MOD128, 2906 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2907 OP_ALG_AAI_HMAC_PRECOMP, 2908 .rfc3686 = true, 2909 .geniv = true, 2910 }, 2911 }, 2912 { 2913 .aead = { 2914 .base = { 2915 .cra_name = "rfc7539(chacha20,poly1305)", 2916 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2917 "caam-qi2", 2918 .cra_blocksize = 1, 2919 }, 2920 .setkey = chachapoly_setkey, 2921 .setauthsize = chachapoly_setauthsize, 2922 .encrypt = aead_encrypt, 2923 .decrypt = aead_decrypt, 2924 .ivsize = CHACHAPOLY_IV_SIZE, 2925 .maxauthsize = POLY1305_DIGEST_SIZE, 2926 }, 2927 .caam = { 2928 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2929 OP_ALG_AAI_AEAD, 2930 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2931 OP_ALG_AAI_AEAD, 2932 .nodkp = true, 2933 }, 2934 }, 2935 { 2936 .aead = { 2937 .base = { 2938 .cra_name = "rfc7539esp(chacha20,poly1305)", 2939 .cra_driver_name = "rfc7539esp-chacha20-" 2940 "poly1305-caam-qi2", 2941 .cra_blocksize = 1, 2942 }, 2943 .setkey = chachapoly_setkey, 2944 .setauthsize = chachapoly_setauthsize, 2945 .encrypt = aead_encrypt, 2946 .decrypt = aead_decrypt, 2947 .ivsize = 8, 2948 .maxauthsize = POLY1305_DIGEST_SIZE, 2949 }, 2950 .caam = { 2951 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2952 OP_ALG_AAI_AEAD, 2953 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2954 OP_ALG_AAI_AEAD, 2955 .nodkp = true, 2956 }, 2957 }, 2958 { 2959 .aead = { 2960 .base = { 2961 .cra_name = "authenc(hmac(sha512)," 2962 "rfc3686(ctr(aes)))", 2963 .cra_driver_name = "authenc-hmac-sha512-" 2964 "rfc3686-ctr-aes-caam-qi2", 2965 .cra_blocksize = 1, 2966 }, 2967 .setkey = aead_setkey, 2968 .setauthsize = aead_setauthsize, 2969 .encrypt = aead_encrypt, 2970 .decrypt = aead_decrypt, 2971 .ivsize = CTR_RFC3686_IV_SIZE, 2972 .maxauthsize = SHA512_DIGEST_SIZE, 2973 }, 2974 .caam = { 2975 .class1_alg_type = OP_ALG_ALGSEL_AES | 2976 OP_ALG_AAI_CTR_MOD128, 2977 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2978 OP_ALG_AAI_HMAC_PRECOMP, 2979 .rfc3686 = true, 2980 }, 2981 }, 2982 { 2983 .aead = { 2984 .base = { 2985 .cra_name = "seqiv(authenc(hmac(sha512)," 2986 "rfc3686(ctr(aes))))", 2987 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2988 "rfc3686-ctr-aes-caam-qi2", 2989 .cra_blocksize = 1, 2990 }, 2991 .setkey = aead_setkey, 2992 .setauthsize = aead_setauthsize, 2993 .encrypt = aead_encrypt, 2994 .decrypt = aead_decrypt, 2995 .ivsize = CTR_RFC3686_IV_SIZE, 2996 .maxauthsize = SHA512_DIGEST_SIZE, 2997 }, 2998 .caam = { 2999 .class1_alg_type = OP_ALG_ALGSEL_AES | 3000 OP_ALG_AAI_CTR_MOD128, 3001 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3002 OP_ALG_AAI_HMAC_PRECOMP, 3003 .rfc3686 = true, 3004 .geniv = true, 3005 }, 3006 }, 3007 }; 3008 3009 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3010 { 3011 struct skcipher_alg *alg = &t_alg->skcipher; 3012 3013 alg->base.cra_module = THIS_MODULE; 3014 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3015 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3016 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3017 CRYPTO_ALG_KERN_DRIVER_ONLY); 3018 3019 alg->init = caam_cra_init_skcipher; 3020 alg->exit = caam_cra_exit; 3021 } 3022 3023 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3024 { 3025 struct aead_alg *alg = &t_alg->aead; 3026 3027 alg->base.cra_module = THIS_MODULE; 3028 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3029 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3030 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3031 CRYPTO_ALG_KERN_DRIVER_ONLY; 3032 3033 alg->init = caam_cra_init_aead; 3034 alg->exit = caam_cra_exit_aead; 3035 } 3036 3037 /* max hash key is max split key size */ 3038 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 3039 3040 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 3041 3042 /* caam context sizes for hashes: running digest + 8 */ 3043 #define HASH_MSG_LEN 8 3044 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 3045 3046 enum hash_optype { 3047 UPDATE = 0, 3048 UPDATE_FIRST, 3049 FINALIZE, 3050 DIGEST, 3051 HASH_NUM_OP 3052 }; 3053 3054 /** 3055 * struct caam_hash_ctx - ahash per-session context 3056 * @flc: Flow Contexts array 3057 * @key: authentication key 3058 * @flc_dma: I/O virtual addresses of the Flow Contexts 3059 * @dev: dpseci device 3060 * @ctx_len: size of Context Register 3061 * @adata: hashing algorithm details 3062 */ 3063 struct caam_hash_ctx { 3064 struct caam_flc flc[HASH_NUM_OP]; 3065 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3066 dma_addr_t flc_dma[HASH_NUM_OP]; 3067 struct device *dev; 3068 int ctx_len; 3069 struct alginfo adata; 3070 }; 3071 3072 /* ahash state */ 3073 struct caam_hash_state { 3074 struct caam_request caam_req; 3075 dma_addr_t buf_dma; 3076 dma_addr_t ctx_dma; 3077 int ctx_dma_len; 3078 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3079 int buflen; 3080 int next_buflen; 3081 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 3082 int (*update)(struct ahash_request *req); 3083 int (*final)(struct ahash_request *req); 3084 int (*finup)(struct ahash_request *req); 3085 }; 3086 3087 struct caam_export_state { 3088 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 3089 u8 caam_ctx[MAX_CTX_LEN]; 3090 int buflen; 3091 int (*update)(struct ahash_request *req); 3092 int (*final)(struct ahash_request *req); 3093 int (*finup)(struct ahash_request *req); 3094 }; 3095 3096 /* Map current buffer in state (if length > 0) and put it in link table */ 3097 static inline int buf_map_to_qm_sg(struct device *dev, 3098 struct dpaa2_sg_entry *qm_sg, 3099 struct caam_hash_state *state) 3100 { 3101 int buflen = state->buflen; 3102 3103 if (!buflen) 3104 return 0; 3105 3106 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3107 DMA_TO_DEVICE); 3108 if (dma_mapping_error(dev, state->buf_dma)) { 3109 dev_err(dev, "unable to map buf\n"); 3110 state->buf_dma = 0; 3111 return -ENOMEM; 3112 } 3113 3114 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3115 3116 return 0; 3117 } 3118 3119 /* Map state->caam_ctx, and add it to link table */ 3120 static inline int ctx_map_to_qm_sg(struct device *dev, 3121 struct caam_hash_state *state, int ctx_len, 3122 struct dpaa2_sg_entry *qm_sg, u32 flag) 3123 { 3124 state->ctx_dma_len = ctx_len; 3125 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3126 if (dma_mapping_error(dev, state->ctx_dma)) { 3127 dev_err(dev, "unable to map ctx\n"); 3128 state->ctx_dma = 0; 3129 return -ENOMEM; 3130 } 3131 3132 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3133 3134 return 0; 3135 } 3136 3137 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3138 { 3139 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3140 int digestsize = crypto_ahash_digestsize(ahash); 3141 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3142 struct caam_flc *flc; 3143 u32 *desc; 3144 3145 /* ahash_update shared descriptor */ 3146 flc = &ctx->flc[UPDATE]; 3147 desc = flc->sh_desc; 3148 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3149 ctx->ctx_len, true, priv->sec_attr.era); 3150 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3151 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3152 desc_bytes(desc), DMA_BIDIRECTIONAL); 3153 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3154 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3155 1); 3156 3157 /* ahash_update_first shared descriptor */ 3158 flc = &ctx->flc[UPDATE_FIRST]; 3159 desc = flc->sh_desc; 3160 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3161 ctx->ctx_len, false, priv->sec_attr.era); 3162 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3163 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3164 desc_bytes(desc), DMA_BIDIRECTIONAL); 3165 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3166 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3167 1); 3168 3169 /* ahash_final shared descriptor */ 3170 flc = &ctx->flc[FINALIZE]; 3171 desc = flc->sh_desc; 3172 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3173 ctx->ctx_len, true, priv->sec_attr.era); 3174 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3175 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3176 desc_bytes(desc), DMA_BIDIRECTIONAL); 3177 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3178 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3179 1); 3180 3181 /* ahash_digest shared descriptor */ 3182 flc = &ctx->flc[DIGEST]; 3183 desc = flc->sh_desc; 3184 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3185 ctx->ctx_len, false, priv->sec_attr.era); 3186 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3187 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3188 desc_bytes(desc), DMA_BIDIRECTIONAL); 3189 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3190 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3191 1); 3192 3193 return 0; 3194 } 3195 3196 struct split_key_sh_result { 3197 struct completion completion; 3198 int err; 3199 struct device *dev; 3200 }; 3201 3202 static void split_key_sh_done(void *cbk_ctx, u32 err) 3203 { 3204 struct split_key_sh_result *res = cbk_ctx; 3205 3206 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3207 3208 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3209 complete(&res->completion); 3210 } 3211 3212 /* Digest hash size if it is too large */ 3213 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3214 u32 digestsize) 3215 { 3216 struct caam_request *req_ctx; 3217 u32 *desc; 3218 struct split_key_sh_result result; 3219 dma_addr_t key_dma; 3220 struct caam_flc *flc; 3221 dma_addr_t flc_dma; 3222 int ret = -ENOMEM; 3223 struct dpaa2_fl_entry *in_fle, *out_fle; 3224 3225 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL); 3226 if (!req_ctx) 3227 return -ENOMEM; 3228 3229 in_fle = &req_ctx->fd_flt[1]; 3230 out_fle = &req_ctx->fd_flt[0]; 3231 3232 flc = kzalloc(sizeof(*flc), GFP_KERNEL); 3233 if (!flc) 3234 goto err_flc; 3235 3236 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3237 if (dma_mapping_error(ctx->dev, key_dma)) { 3238 dev_err(ctx->dev, "unable to map key memory\n"); 3239 goto err_key_dma; 3240 } 3241 3242 desc = flc->sh_desc; 3243 3244 init_sh_desc(desc, 0); 3245 3246 /* descriptor to perform unkeyed hash on key_in */ 3247 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3248 OP_ALG_AS_INITFINAL); 3249 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3250 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3251 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3252 LDST_SRCDST_BYTE_CONTEXT); 3253 3254 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3255 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3256 desc_bytes(desc), DMA_TO_DEVICE); 3257 if (dma_mapping_error(ctx->dev, flc_dma)) { 3258 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3259 goto err_flc_dma; 3260 } 3261 3262 dpaa2_fl_set_final(in_fle, true); 3263 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3264 dpaa2_fl_set_addr(in_fle, key_dma); 3265 dpaa2_fl_set_len(in_fle, *keylen); 3266 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3267 dpaa2_fl_set_addr(out_fle, key_dma); 3268 dpaa2_fl_set_len(out_fle, digestsize); 3269 3270 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3271 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3272 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3273 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3274 1); 3275 3276 result.err = 0; 3277 init_completion(&result.completion); 3278 result.dev = ctx->dev; 3279 3280 req_ctx->flc = flc; 3281 req_ctx->flc_dma = flc_dma; 3282 req_ctx->cbk = split_key_sh_done; 3283 req_ctx->ctx = &result; 3284 3285 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3286 if (ret == -EINPROGRESS) { 3287 /* in progress */ 3288 wait_for_completion(&result.completion); 3289 ret = result.err; 3290 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3291 DUMP_PREFIX_ADDRESS, 16, 4, key, 3292 digestsize, 1); 3293 } 3294 3295 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3296 DMA_TO_DEVICE); 3297 err_flc_dma: 3298 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3299 err_key_dma: 3300 kfree(flc); 3301 err_flc: 3302 kfree(req_ctx); 3303 3304 *keylen = digestsize; 3305 3306 return ret; 3307 } 3308 3309 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3310 unsigned int keylen) 3311 { 3312 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3313 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3314 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3315 int ret; 3316 u8 *hashed_key = NULL; 3317 3318 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3319 3320 if (keylen > blocksize) { 3321 unsigned int aligned_len = 3322 ALIGN(keylen, dma_get_cache_alignment()); 3323 3324 if (aligned_len < keylen) 3325 return -EOVERFLOW; 3326 3327 hashed_key = kmemdup(key, aligned_len, GFP_KERNEL); 3328 if (!hashed_key) 3329 return -ENOMEM; 3330 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3331 if (ret) 3332 goto bad_free_key; 3333 key = hashed_key; 3334 } 3335 3336 ctx->adata.keylen = keylen; 3337 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3338 OP_ALG_ALGSEL_MASK); 3339 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3340 goto bad_free_key; 3341 3342 ctx->adata.key_virt = key; 3343 ctx->adata.key_inline = true; 3344 3345 /* 3346 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3347 * in invalid opcodes (last bytes of user key) in the resulting 3348 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3349 * addresses are needed. 3350 */ 3351 if (keylen > ctx->adata.keylen_pad) { 3352 memcpy(ctx->key, key, keylen); 3353 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3354 ctx->adata.keylen_pad, 3355 DMA_TO_DEVICE); 3356 } 3357 3358 ret = ahash_set_sh_desc(ahash); 3359 kfree(hashed_key); 3360 return ret; 3361 bad_free_key: 3362 kfree(hashed_key); 3363 return -EINVAL; 3364 } 3365 3366 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3367 struct ahash_request *req) 3368 { 3369 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3370 3371 if (edesc->src_nents) 3372 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3373 3374 if (edesc->qm_sg_bytes) 3375 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3376 DMA_TO_DEVICE); 3377 3378 if (state->buf_dma) { 3379 dma_unmap_single(dev, state->buf_dma, state->buflen, 3380 DMA_TO_DEVICE); 3381 state->buf_dma = 0; 3382 } 3383 } 3384 3385 static inline void ahash_unmap_ctx(struct device *dev, 3386 struct ahash_edesc *edesc, 3387 struct ahash_request *req, u32 flag) 3388 { 3389 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3390 3391 if (state->ctx_dma) { 3392 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3393 state->ctx_dma = 0; 3394 } 3395 ahash_unmap(dev, edesc, req); 3396 } 3397 3398 static void ahash_done(void *cbk_ctx, u32 status) 3399 { 3400 struct crypto_async_request *areq = cbk_ctx; 3401 struct ahash_request *req = ahash_request_cast(areq); 3402 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3403 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3404 struct ahash_edesc *edesc = state->caam_req.edesc; 3405 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3406 int digestsize = crypto_ahash_digestsize(ahash); 3407 int ecode = 0; 3408 3409 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3410 3411 if (unlikely(status)) 3412 ecode = caam_qi2_strstatus(ctx->dev, status); 3413 3414 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3415 memcpy(req->result, state->caam_ctx, digestsize); 3416 qi_cache_free(edesc); 3417 3418 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3419 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3420 ctx->ctx_len, 1); 3421 3422 ahash_request_complete(req, ecode); 3423 } 3424 3425 static void ahash_done_bi(void *cbk_ctx, u32 status) 3426 { 3427 struct crypto_async_request *areq = cbk_ctx; 3428 struct ahash_request *req = ahash_request_cast(areq); 3429 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3430 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3431 struct ahash_edesc *edesc = state->caam_req.edesc; 3432 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3433 int ecode = 0; 3434 3435 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3436 3437 if (unlikely(status)) 3438 ecode = caam_qi2_strstatus(ctx->dev, status); 3439 3440 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3441 qi_cache_free(edesc); 3442 3443 scatterwalk_map_and_copy(state->buf, req->src, 3444 req->nbytes - state->next_buflen, 3445 state->next_buflen, 0); 3446 state->buflen = state->next_buflen; 3447 3448 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3449 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3450 state->buflen, 1); 3451 3452 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3453 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3454 ctx->ctx_len, 1); 3455 if (req->result) 3456 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3457 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3458 crypto_ahash_digestsize(ahash), 1); 3459 3460 ahash_request_complete(req, ecode); 3461 } 3462 3463 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3464 { 3465 struct crypto_async_request *areq = cbk_ctx; 3466 struct ahash_request *req = ahash_request_cast(areq); 3467 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3468 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3469 struct ahash_edesc *edesc = state->caam_req.edesc; 3470 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3471 int digestsize = crypto_ahash_digestsize(ahash); 3472 int ecode = 0; 3473 3474 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3475 3476 if (unlikely(status)) 3477 ecode = caam_qi2_strstatus(ctx->dev, status); 3478 3479 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3480 memcpy(req->result, state->caam_ctx, digestsize); 3481 qi_cache_free(edesc); 3482 3483 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3484 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3485 ctx->ctx_len, 1); 3486 3487 ahash_request_complete(req, ecode); 3488 } 3489 3490 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3491 { 3492 struct crypto_async_request *areq = cbk_ctx; 3493 struct ahash_request *req = ahash_request_cast(areq); 3494 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3495 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3496 struct ahash_edesc *edesc = state->caam_req.edesc; 3497 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3498 int ecode = 0; 3499 3500 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3501 3502 if (unlikely(status)) 3503 ecode = caam_qi2_strstatus(ctx->dev, status); 3504 3505 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3506 qi_cache_free(edesc); 3507 3508 scatterwalk_map_and_copy(state->buf, req->src, 3509 req->nbytes - state->next_buflen, 3510 state->next_buflen, 0); 3511 state->buflen = state->next_buflen; 3512 3513 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3514 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3515 state->buflen, 1); 3516 3517 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3518 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3519 ctx->ctx_len, 1); 3520 if (req->result) 3521 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3522 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3523 crypto_ahash_digestsize(ahash), 1); 3524 3525 ahash_request_complete(req, ecode); 3526 } 3527 3528 static int ahash_update_ctx(struct ahash_request *req) 3529 { 3530 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3531 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3532 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3533 struct caam_request *req_ctx = &state->caam_req; 3534 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3535 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3536 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3537 GFP_KERNEL : GFP_ATOMIC; 3538 u8 *buf = state->buf; 3539 int *buflen = &state->buflen; 3540 int *next_buflen = &state->next_buflen; 3541 int in_len = *buflen + req->nbytes, to_hash; 3542 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3543 struct ahash_edesc *edesc; 3544 int ret = 0; 3545 3546 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3547 to_hash = in_len - *next_buflen; 3548 3549 if (to_hash) { 3550 struct dpaa2_sg_entry *sg_table; 3551 int src_len = req->nbytes - *next_buflen; 3552 3553 src_nents = sg_nents_for_len(req->src, src_len); 3554 if (src_nents < 0) { 3555 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3556 return src_nents; 3557 } 3558 3559 if (src_nents) { 3560 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3561 DMA_TO_DEVICE); 3562 if (!mapped_nents) { 3563 dev_err(ctx->dev, "unable to DMA map source\n"); 3564 return -ENOMEM; 3565 } 3566 } else { 3567 mapped_nents = 0; 3568 } 3569 3570 /* allocate space for base edesc and link tables */ 3571 edesc = qi_cache_zalloc(flags); 3572 if (!edesc) { 3573 dma_unmap_sg(ctx->dev, req->src, src_nents, 3574 DMA_TO_DEVICE); 3575 return -ENOMEM; 3576 } 3577 3578 edesc->src_nents = src_nents; 3579 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3580 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3581 sizeof(*sg_table); 3582 sg_table = &edesc->sgt[0]; 3583 3584 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3585 DMA_BIDIRECTIONAL); 3586 if (ret) 3587 goto unmap_ctx; 3588 3589 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3590 if (ret) 3591 goto unmap_ctx; 3592 3593 if (mapped_nents) { 3594 sg_to_qm_sg_last(req->src, src_len, 3595 sg_table + qm_sg_src_index, 0); 3596 } else { 3597 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3598 true); 3599 } 3600 3601 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3602 qm_sg_bytes, DMA_TO_DEVICE); 3603 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3604 dev_err(ctx->dev, "unable to map S/G table\n"); 3605 ret = -ENOMEM; 3606 goto unmap_ctx; 3607 } 3608 edesc->qm_sg_bytes = qm_sg_bytes; 3609 3610 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3611 dpaa2_fl_set_final(in_fle, true); 3612 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3613 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3614 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3615 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3616 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3617 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3618 3619 req_ctx->flc = &ctx->flc[UPDATE]; 3620 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3621 req_ctx->cbk = ahash_done_bi; 3622 req_ctx->ctx = &req->base; 3623 req_ctx->edesc = edesc; 3624 3625 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3626 if (ret != -EINPROGRESS && 3627 !(ret == -EBUSY && 3628 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3629 goto unmap_ctx; 3630 } else if (*next_buflen) { 3631 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3632 req->nbytes, 0); 3633 *buflen = *next_buflen; 3634 3635 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3636 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3637 *buflen, 1); 3638 } 3639 3640 return ret; 3641 unmap_ctx: 3642 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3643 qi_cache_free(edesc); 3644 return ret; 3645 } 3646 3647 static int ahash_final_ctx(struct ahash_request *req) 3648 { 3649 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3650 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3651 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3652 struct caam_request *req_ctx = &state->caam_req; 3653 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3654 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3655 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3656 GFP_KERNEL : GFP_ATOMIC; 3657 int buflen = state->buflen; 3658 int qm_sg_bytes; 3659 int digestsize = crypto_ahash_digestsize(ahash); 3660 struct ahash_edesc *edesc; 3661 struct dpaa2_sg_entry *sg_table; 3662 int ret; 3663 3664 /* allocate space for base edesc and link tables */ 3665 edesc = qi_cache_zalloc(flags); 3666 if (!edesc) 3667 return -ENOMEM; 3668 3669 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3670 sg_table = &edesc->sgt[0]; 3671 3672 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3673 DMA_BIDIRECTIONAL); 3674 if (ret) 3675 goto unmap_ctx; 3676 3677 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3678 if (ret) 3679 goto unmap_ctx; 3680 3681 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3682 3683 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3684 DMA_TO_DEVICE); 3685 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3686 dev_err(ctx->dev, "unable to map S/G table\n"); 3687 ret = -ENOMEM; 3688 goto unmap_ctx; 3689 } 3690 edesc->qm_sg_bytes = qm_sg_bytes; 3691 3692 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3693 dpaa2_fl_set_final(in_fle, true); 3694 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3695 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3696 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3697 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3698 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3699 dpaa2_fl_set_len(out_fle, digestsize); 3700 3701 req_ctx->flc = &ctx->flc[FINALIZE]; 3702 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3703 req_ctx->cbk = ahash_done_ctx_src; 3704 req_ctx->ctx = &req->base; 3705 req_ctx->edesc = edesc; 3706 3707 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3708 if (ret == -EINPROGRESS || 3709 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3710 return ret; 3711 3712 unmap_ctx: 3713 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3714 qi_cache_free(edesc); 3715 return ret; 3716 } 3717 3718 static int ahash_finup_ctx(struct ahash_request *req) 3719 { 3720 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3721 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3722 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3723 struct caam_request *req_ctx = &state->caam_req; 3724 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3725 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3726 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3727 GFP_KERNEL : GFP_ATOMIC; 3728 int buflen = state->buflen; 3729 int qm_sg_bytes, qm_sg_src_index; 3730 int src_nents, mapped_nents; 3731 int digestsize = crypto_ahash_digestsize(ahash); 3732 struct ahash_edesc *edesc; 3733 struct dpaa2_sg_entry *sg_table; 3734 int ret; 3735 3736 src_nents = sg_nents_for_len(req->src, req->nbytes); 3737 if (src_nents < 0) { 3738 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3739 return src_nents; 3740 } 3741 3742 if (src_nents) { 3743 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3744 DMA_TO_DEVICE); 3745 if (!mapped_nents) { 3746 dev_err(ctx->dev, "unable to DMA map source\n"); 3747 return -ENOMEM; 3748 } 3749 } else { 3750 mapped_nents = 0; 3751 } 3752 3753 /* allocate space for base edesc and link tables */ 3754 edesc = qi_cache_zalloc(flags); 3755 if (!edesc) { 3756 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3757 return -ENOMEM; 3758 } 3759 3760 edesc->src_nents = src_nents; 3761 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3762 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3763 sizeof(*sg_table); 3764 sg_table = &edesc->sgt[0]; 3765 3766 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3767 DMA_BIDIRECTIONAL); 3768 if (ret) 3769 goto unmap_ctx; 3770 3771 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3772 if (ret) 3773 goto unmap_ctx; 3774 3775 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3776 3777 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3778 DMA_TO_DEVICE); 3779 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3780 dev_err(ctx->dev, "unable to map S/G table\n"); 3781 ret = -ENOMEM; 3782 goto unmap_ctx; 3783 } 3784 edesc->qm_sg_bytes = qm_sg_bytes; 3785 3786 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3787 dpaa2_fl_set_final(in_fle, true); 3788 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3789 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3790 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3791 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3792 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3793 dpaa2_fl_set_len(out_fle, digestsize); 3794 3795 req_ctx->flc = &ctx->flc[FINALIZE]; 3796 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3797 req_ctx->cbk = ahash_done_ctx_src; 3798 req_ctx->ctx = &req->base; 3799 req_ctx->edesc = edesc; 3800 3801 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3802 if (ret == -EINPROGRESS || 3803 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3804 return ret; 3805 3806 unmap_ctx: 3807 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3808 qi_cache_free(edesc); 3809 return ret; 3810 } 3811 3812 static int ahash_digest(struct ahash_request *req) 3813 { 3814 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3815 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3816 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3817 struct caam_request *req_ctx = &state->caam_req; 3818 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3819 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3820 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3821 GFP_KERNEL : GFP_ATOMIC; 3822 int digestsize = crypto_ahash_digestsize(ahash); 3823 int src_nents, mapped_nents; 3824 struct ahash_edesc *edesc; 3825 int ret = -ENOMEM; 3826 3827 state->buf_dma = 0; 3828 3829 src_nents = sg_nents_for_len(req->src, req->nbytes); 3830 if (src_nents < 0) { 3831 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3832 return src_nents; 3833 } 3834 3835 if (src_nents) { 3836 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3837 DMA_TO_DEVICE); 3838 if (!mapped_nents) { 3839 dev_err(ctx->dev, "unable to map source for DMA\n"); 3840 return ret; 3841 } 3842 } else { 3843 mapped_nents = 0; 3844 } 3845 3846 /* allocate space for base edesc and link tables */ 3847 edesc = qi_cache_zalloc(flags); 3848 if (!edesc) { 3849 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3850 return ret; 3851 } 3852 3853 edesc->src_nents = src_nents; 3854 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3855 3856 if (mapped_nents > 1) { 3857 int qm_sg_bytes; 3858 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3859 3860 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3861 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3862 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3863 qm_sg_bytes, DMA_TO_DEVICE); 3864 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3865 dev_err(ctx->dev, "unable to map S/G table\n"); 3866 goto unmap; 3867 } 3868 edesc->qm_sg_bytes = qm_sg_bytes; 3869 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3870 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3871 } else { 3872 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3873 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3874 } 3875 3876 state->ctx_dma_len = digestsize; 3877 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3878 DMA_FROM_DEVICE); 3879 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3880 dev_err(ctx->dev, "unable to map ctx\n"); 3881 state->ctx_dma = 0; 3882 goto unmap; 3883 } 3884 3885 dpaa2_fl_set_final(in_fle, true); 3886 dpaa2_fl_set_len(in_fle, req->nbytes); 3887 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3888 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3889 dpaa2_fl_set_len(out_fle, digestsize); 3890 3891 req_ctx->flc = &ctx->flc[DIGEST]; 3892 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3893 req_ctx->cbk = ahash_done; 3894 req_ctx->ctx = &req->base; 3895 req_ctx->edesc = edesc; 3896 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3897 if (ret == -EINPROGRESS || 3898 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3899 return ret; 3900 3901 unmap: 3902 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3903 qi_cache_free(edesc); 3904 return ret; 3905 } 3906 3907 static int ahash_final_no_ctx(struct ahash_request *req) 3908 { 3909 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3910 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3911 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3912 struct caam_request *req_ctx = &state->caam_req; 3913 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3914 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3915 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3916 GFP_KERNEL : GFP_ATOMIC; 3917 u8 *buf = state->buf; 3918 int buflen = state->buflen; 3919 int digestsize = crypto_ahash_digestsize(ahash); 3920 struct ahash_edesc *edesc; 3921 int ret = -ENOMEM; 3922 3923 /* allocate space for base edesc and link tables */ 3924 edesc = qi_cache_zalloc(flags); 3925 if (!edesc) 3926 return ret; 3927 3928 if (buflen) { 3929 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3930 DMA_TO_DEVICE); 3931 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3932 dev_err(ctx->dev, "unable to map src\n"); 3933 goto unmap; 3934 } 3935 } 3936 3937 state->ctx_dma_len = digestsize; 3938 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3939 DMA_FROM_DEVICE); 3940 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3941 dev_err(ctx->dev, "unable to map ctx\n"); 3942 state->ctx_dma = 0; 3943 goto unmap; 3944 } 3945 3946 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3947 dpaa2_fl_set_final(in_fle, true); 3948 /* 3949 * crypto engine requires the input entry to be present when 3950 * "frame list" FD is used. 3951 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3952 * in_fle zeroized (except for "Final" flag) is the best option. 3953 */ 3954 if (buflen) { 3955 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3956 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3957 dpaa2_fl_set_len(in_fle, buflen); 3958 } 3959 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3960 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3961 dpaa2_fl_set_len(out_fle, digestsize); 3962 3963 req_ctx->flc = &ctx->flc[DIGEST]; 3964 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3965 req_ctx->cbk = ahash_done; 3966 req_ctx->ctx = &req->base; 3967 req_ctx->edesc = edesc; 3968 3969 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3970 if (ret == -EINPROGRESS || 3971 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3972 return ret; 3973 3974 unmap: 3975 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3976 qi_cache_free(edesc); 3977 return ret; 3978 } 3979 3980 static int ahash_update_no_ctx(struct ahash_request *req) 3981 { 3982 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3983 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3984 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3985 struct caam_request *req_ctx = &state->caam_req; 3986 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3987 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3988 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3989 GFP_KERNEL : GFP_ATOMIC; 3990 u8 *buf = state->buf; 3991 int *buflen = &state->buflen; 3992 int *next_buflen = &state->next_buflen; 3993 int in_len = *buflen + req->nbytes, to_hash; 3994 int qm_sg_bytes, src_nents, mapped_nents; 3995 struct ahash_edesc *edesc; 3996 int ret = 0; 3997 3998 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3999 to_hash = in_len - *next_buflen; 4000 4001 if (to_hash) { 4002 struct dpaa2_sg_entry *sg_table; 4003 int src_len = req->nbytes - *next_buflen; 4004 4005 src_nents = sg_nents_for_len(req->src, src_len); 4006 if (src_nents < 0) { 4007 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4008 return src_nents; 4009 } 4010 4011 if (src_nents) { 4012 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4013 DMA_TO_DEVICE); 4014 if (!mapped_nents) { 4015 dev_err(ctx->dev, "unable to DMA map source\n"); 4016 return -ENOMEM; 4017 } 4018 } else { 4019 mapped_nents = 0; 4020 } 4021 4022 /* allocate space for base edesc and link tables */ 4023 edesc = qi_cache_zalloc(flags); 4024 if (!edesc) { 4025 dma_unmap_sg(ctx->dev, req->src, src_nents, 4026 DMA_TO_DEVICE); 4027 return -ENOMEM; 4028 } 4029 4030 edesc->src_nents = src_nents; 4031 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 4032 sizeof(*sg_table); 4033 sg_table = &edesc->sgt[0]; 4034 4035 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4036 if (ret) 4037 goto unmap_ctx; 4038 4039 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 4040 4041 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4042 qm_sg_bytes, DMA_TO_DEVICE); 4043 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4044 dev_err(ctx->dev, "unable to map S/G table\n"); 4045 ret = -ENOMEM; 4046 goto unmap_ctx; 4047 } 4048 edesc->qm_sg_bytes = qm_sg_bytes; 4049 4050 state->ctx_dma_len = ctx->ctx_len; 4051 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4052 ctx->ctx_len, DMA_FROM_DEVICE); 4053 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4054 dev_err(ctx->dev, "unable to map ctx\n"); 4055 state->ctx_dma = 0; 4056 ret = -ENOMEM; 4057 goto unmap_ctx; 4058 } 4059 4060 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4061 dpaa2_fl_set_final(in_fle, true); 4062 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4063 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4064 dpaa2_fl_set_len(in_fle, to_hash); 4065 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4066 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4067 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4068 4069 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4070 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4071 req_ctx->cbk = ahash_done_ctx_dst; 4072 req_ctx->ctx = &req->base; 4073 req_ctx->edesc = edesc; 4074 4075 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4076 if (ret != -EINPROGRESS && 4077 !(ret == -EBUSY && 4078 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4079 goto unmap_ctx; 4080 4081 state->update = ahash_update_ctx; 4082 state->finup = ahash_finup_ctx; 4083 state->final = ahash_final_ctx; 4084 } else if (*next_buflen) { 4085 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 4086 req->nbytes, 0); 4087 *buflen = *next_buflen; 4088 4089 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4090 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4091 *buflen, 1); 4092 } 4093 4094 return ret; 4095 unmap_ctx: 4096 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4097 qi_cache_free(edesc); 4098 return ret; 4099 } 4100 4101 static int ahash_finup_no_ctx(struct ahash_request *req) 4102 { 4103 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4104 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4105 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4106 struct caam_request *req_ctx = &state->caam_req; 4107 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4108 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4109 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4110 GFP_KERNEL : GFP_ATOMIC; 4111 int buflen = state->buflen; 4112 int qm_sg_bytes, src_nents, mapped_nents; 4113 int digestsize = crypto_ahash_digestsize(ahash); 4114 struct ahash_edesc *edesc; 4115 struct dpaa2_sg_entry *sg_table; 4116 int ret = -ENOMEM; 4117 4118 src_nents = sg_nents_for_len(req->src, req->nbytes); 4119 if (src_nents < 0) { 4120 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4121 return src_nents; 4122 } 4123 4124 if (src_nents) { 4125 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4126 DMA_TO_DEVICE); 4127 if (!mapped_nents) { 4128 dev_err(ctx->dev, "unable to DMA map source\n"); 4129 return ret; 4130 } 4131 } else { 4132 mapped_nents = 0; 4133 } 4134 4135 /* allocate space for base edesc and link tables */ 4136 edesc = qi_cache_zalloc(flags); 4137 if (!edesc) { 4138 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4139 return ret; 4140 } 4141 4142 edesc->src_nents = src_nents; 4143 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4144 sg_table = &edesc->sgt[0]; 4145 4146 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4147 if (ret) 4148 goto unmap; 4149 4150 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4151 4152 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4153 DMA_TO_DEVICE); 4154 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4155 dev_err(ctx->dev, "unable to map S/G table\n"); 4156 ret = -ENOMEM; 4157 goto unmap; 4158 } 4159 edesc->qm_sg_bytes = qm_sg_bytes; 4160 4161 state->ctx_dma_len = digestsize; 4162 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4163 DMA_FROM_DEVICE); 4164 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4165 dev_err(ctx->dev, "unable to map ctx\n"); 4166 state->ctx_dma = 0; 4167 ret = -ENOMEM; 4168 goto unmap; 4169 } 4170 4171 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4172 dpaa2_fl_set_final(in_fle, true); 4173 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4174 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4175 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4176 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4177 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4178 dpaa2_fl_set_len(out_fle, digestsize); 4179 4180 req_ctx->flc = &ctx->flc[DIGEST]; 4181 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4182 req_ctx->cbk = ahash_done; 4183 req_ctx->ctx = &req->base; 4184 req_ctx->edesc = edesc; 4185 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4186 if (ret != -EINPROGRESS && 4187 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4188 goto unmap; 4189 4190 return ret; 4191 unmap: 4192 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4193 qi_cache_free(edesc); 4194 return ret; 4195 } 4196 4197 static int ahash_update_first(struct ahash_request *req) 4198 { 4199 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4200 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4201 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4202 struct caam_request *req_ctx = &state->caam_req; 4203 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4204 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4205 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4206 GFP_KERNEL : GFP_ATOMIC; 4207 u8 *buf = state->buf; 4208 int *buflen = &state->buflen; 4209 int *next_buflen = &state->next_buflen; 4210 int to_hash; 4211 int src_nents, mapped_nents; 4212 struct ahash_edesc *edesc; 4213 int ret = 0; 4214 4215 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4216 1); 4217 to_hash = req->nbytes - *next_buflen; 4218 4219 if (to_hash) { 4220 struct dpaa2_sg_entry *sg_table; 4221 int src_len = req->nbytes - *next_buflen; 4222 4223 src_nents = sg_nents_for_len(req->src, src_len); 4224 if (src_nents < 0) { 4225 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4226 return src_nents; 4227 } 4228 4229 if (src_nents) { 4230 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4231 DMA_TO_DEVICE); 4232 if (!mapped_nents) { 4233 dev_err(ctx->dev, "unable to map source for DMA\n"); 4234 return -ENOMEM; 4235 } 4236 } else { 4237 mapped_nents = 0; 4238 } 4239 4240 /* allocate space for base edesc and link tables */ 4241 edesc = qi_cache_zalloc(flags); 4242 if (!edesc) { 4243 dma_unmap_sg(ctx->dev, req->src, src_nents, 4244 DMA_TO_DEVICE); 4245 return -ENOMEM; 4246 } 4247 4248 edesc->src_nents = src_nents; 4249 sg_table = &edesc->sgt[0]; 4250 4251 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4252 dpaa2_fl_set_final(in_fle, true); 4253 dpaa2_fl_set_len(in_fle, to_hash); 4254 4255 if (mapped_nents > 1) { 4256 int qm_sg_bytes; 4257 4258 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4259 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4260 sizeof(*sg_table); 4261 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4262 qm_sg_bytes, 4263 DMA_TO_DEVICE); 4264 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4265 dev_err(ctx->dev, "unable to map S/G table\n"); 4266 ret = -ENOMEM; 4267 goto unmap_ctx; 4268 } 4269 edesc->qm_sg_bytes = qm_sg_bytes; 4270 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4271 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4272 } else { 4273 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4274 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4275 } 4276 4277 state->ctx_dma_len = ctx->ctx_len; 4278 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4279 ctx->ctx_len, DMA_FROM_DEVICE); 4280 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4281 dev_err(ctx->dev, "unable to map ctx\n"); 4282 state->ctx_dma = 0; 4283 ret = -ENOMEM; 4284 goto unmap_ctx; 4285 } 4286 4287 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4288 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4289 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4290 4291 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4292 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4293 req_ctx->cbk = ahash_done_ctx_dst; 4294 req_ctx->ctx = &req->base; 4295 req_ctx->edesc = edesc; 4296 4297 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4298 if (ret != -EINPROGRESS && 4299 !(ret == -EBUSY && req->base.flags & 4300 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4301 goto unmap_ctx; 4302 4303 state->update = ahash_update_ctx; 4304 state->finup = ahash_finup_ctx; 4305 state->final = ahash_final_ctx; 4306 } else if (*next_buflen) { 4307 state->update = ahash_update_no_ctx; 4308 state->finup = ahash_finup_no_ctx; 4309 state->final = ahash_final_no_ctx; 4310 scatterwalk_map_and_copy(buf, req->src, 0, 4311 req->nbytes, 0); 4312 *buflen = *next_buflen; 4313 4314 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4315 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4316 *buflen, 1); 4317 } 4318 4319 return ret; 4320 unmap_ctx: 4321 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4322 qi_cache_free(edesc); 4323 return ret; 4324 } 4325 4326 static int ahash_finup_first(struct ahash_request *req) 4327 { 4328 return ahash_digest(req); 4329 } 4330 4331 static int ahash_init(struct ahash_request *req) 4332 { 4333 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4334 4335 state->update = ahash_update_first; 4336 state->finup = ahash_finup_first; 4337 state->final = ahash_final_no_ctx; 4338 4339 state->ctx_dma = 0; 4340 state->ctx_dma_len = 0; 4341 state->buf_dma = 0; 4342 state->buflen = 0; 4343 state->next_buflen = 0; 4344 4345 return 0; 4346 } 4347 4348 static int ahash_update(struct ahash_request *req) 4349 { 4350 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4351 4352 return state->update(req); 4353 } 4354 4355 static int ahash_finup(struct ahash_request *req) 4356 { 4357 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4358 4359 return state->finup(req); 4360 } 4361 4362 static int ahash_final(struct ahash_request *req) 4363 { 4364 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4365 4366 return state->final(req); 4367 } 4368 4369 static int ahash_export(struct ahash_request *req, void *out) 4370 { 4371 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4372 struct caam_export_state *export = out; 4373 u8 *buf = state->buf; 4374 int len = state->buflen; 4375 4376 memcpy(export->buf, buf, len); 4377 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4378 export->buflen = len; 4379 export->update = state->update; 4380 export->final = state->final; 4381 export->finup = state->finup; 4382 4383 return 0; 4384 } 4385 4386 static int ahash_import(struct ahash_request *req, const void *in) 4387 { 4388 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4389 const struct caam_export_state *export = in; 4390 4391 memset(state, 0, sizeof(*state)); 4392 memcpy(state->buf, export->buf, export->buflen); 4393 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4394 state->buflen = export->buflen; 4395 state->update = export->update; 4396 state->final = export->final; 4397 state->finup = export->finup; 4398 4399 return 0; 4400 } 4401 4402 struct caam_hash_template { 4403 char name[CRYPTO_MAX_ALG_NAME]; 4404 char driver_name[CRYPTO_MAX_ALG_NAME]; 4405 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4406 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4407 unsigned int blocksize; 4408 struct ahash_alg template_ahash; 4409 u32 alg_type; 4410 }; 4411 4412 /* ahash descriptors */ 4413 static struct caam_hash_template driver_hash[] = { 4414 { 4415 .name = "sha1", 4416 .driver_name = "sha1-caam-qi2", 4417 .hmac_name = "hmac(sha1)", 4418 .hmac_driver_name = "hmac-sha1-caam-qi2", 4419 .blocksize = SHA1_BLOCK_SIZE, 4420 .template_ahash = { 4421 .init = ahash_init, 4422 .update = ahash_update, 4423 .final = ahash_final, 4424 .finup = ahash_finup, 4425 .digest = ahash_digest, 4426 .export = ahash_export, 4427 .import = ahash_import, 4428 .setkey = ahash_setkey, 4429 .halg = { 4430 .digestsize = SHA1_DIGEST_SIZE, 4431 .statesize = sizeof(struct caam_export_state), 4432 }, 4433 }, 4434 .alg_type = OP_ALG_ALGSEL_SHA1, 4435 }, { 4436 .name = "sha224", 4437 .driver_name = "sha224-caam-qi2", 4438 .hmac_name = "hmac(sha224)", 4439 .hmac_driver_name = "hmac-sha224-caam-qi2", 4440 .blocksize = SHA224_BLOCK_SIZE, 4441 .template_ahash = { 4442 .init = ahash_init, 4443 .update = ahash_update, 4444 .final = ahash_final, 4445 .finup = ahash_finup, 4446 .digest = ahash_digest, 4447 .export = ahash_export, 4448 .import = ahash_import, 4449 .setkey = ahash_setkey, 4450 .halg = { 4451 .digestsize = SHA224_DIGEST_SIZE, 4452 .statesize = sizeof(struct caam_export_state), 4453 }, 4454 }, 4455 .alg_type = OP_ALG_ALGSEL_SHA224, 4456 }, { 4457 .name = "sha256", 4458 .driver_name = "sha256-caam-qi2", 4459 .hmac_name = "hmac(sha256)", 4460 .hmac_driver_name = "hmac-sha256-caam-qi2", 4461 .blocksize = SHA256_BLOCK_SIZE, 4462 .template_ahash = { 4463 .init = ahash_init, 4464 .update = ahash_update, 4465 .final = ahash_final, 4466 .finup = ahash_finup, 4467 .digest = ahash_digest, 4468 .export = ahash_export, 4469 .import = ahash_import, 4470 .setkey = ahash_setkey, 4471 .halg = { 4472 .digestsize = SHA256_DIGEST_SIZE, 4473 .statesize = sizeof(struct caam_export_state), 4474 }, 4475 }, 4476 .alg_type = OP_ALG_ALGSEL_SHA256, 4477 }, { 4478 .name = "sha384", 4479 .driver_name = "sha384-caam-qi2", 4480 .hmac_name = "hmac(sha384)", 4481 .hmac_driver_name = "hmac-sha384-caam-qi2", 4482 .blocksize = SHA384_BLOCK_SIZE, 4483 .template_ahash = { 4484 .init = ahash_init, 4485 .update = ahash_update, 4486 .final = ahash_final, 4487 .finup = ahash_finup, 4488 .digest = ahash_digest, 4489 .export = ahash_export, 4490 .import = ahash_import, 4491 .setkey = ahash_setkey, 4492 .halg = { 4493 .digestsize = SHA384_DIGEST_SIZE, 4494 .statesize = sizeof(struct caam_export_state), 4495 }, 4496 }, 4497 .alg_type = OP_ALG_ALGSEL_SHA384, 4498 }, { 4499 .name = "sha512", 4500 .driver_name = "sha512-caam-qi2", 4501 .hmac_name = "hmac(sha512)", 4502 .hmac_driver_name = "hmac-sha512-caam-qi2", 4503 .blocksize = SHA512_BLOCK_SIZE, 4504 .template_ahash = { 4505 .init = ahash_init, 4506 .update = ahash_update, 4507 .final = ahash_final, 4508 .finup = ahash_finup, 4509 .digest = ahash_digest, 4510 .export = ahash_export, 4511 .import = ahash_import, 4512 .setkey = ahash_setkey, 4513 .halg = { 4514 .digestsize = SHA512_DIGEST_SIZE, 4515 .statesize = sizeof(struct caam_export_state), 4516 }, 4517 }, 4518 .alg_type = OP_ALG_ALGSEL_SHA512, 4519 }, { 4520 .name = "md5", 4521 .driver_name = "md5-caam-qi2", 4522 .hmac_name = "hmac(md5)", 4523 .hmac_driver_name = "hmac-md5-caam-qi2", 4524 .blocksize = MD5_BLOCK_WORDS * 4, 4525 .template_ahash = { 4526 .init = ahash_init, 4527 .update = ahash_update, 4528 .final = ahash_final, 4529 .finup = ahash_finup, 4530 .digest = ahash_digest, 4531 .export = ahash_export, 4532 .import = ahash_import, 4533 .setkey = ahash_setkey, 4534 .halg = { 4535 .digestsize = MD5_DIGEST_SIZE, 4536 .statesize = sizeof(struct caam_export_state), 4537 }, 4538 }, 4539 .alg_type = OP_ALG_ALGSEL_MD5, 4540 } 4541 }; 4542 4543 struct caam_hash_alg { 4544 struct list_head entry; 4545 struct device *dev; 4546 int alg_type; 4547 struct ahash_alg ahash_alg; 4548 }; 4549 4550 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4551 { 4552 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4553 struct crypto_alg *base = tfm->__crt_alg; 4554 struct hash_alg_common *halg = 4555 container_of(base, struct hash_alg_common, base); 4556 struct ahash_alg *alg = 4557 container_of(halg, struct ahash_alg, halg); 4558 struct caam_hash_alg *caam_hash = 4559 container_of(alg, struct caam_hash_alg, ahash_alg); 4560 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4561 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4562 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4563 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4564 HASH_MSG_LEN + 32, 4565 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4566 HASH_MSG_LEN + 64, 4567 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4568 dma_addr_t dma_addr; 4569 int i; 4570 4571 ctx->dev = caam_hash->dev; 4572 4573 if (alg->setkey) { 4574 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4575 ARRAY_SIZE(ctx->key), 4576 DMA_TO_DEVICE, 4577 DMA_ATTR_SKIP_CPU_SYNC); 4578 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4579 dev_err(ctx->dev, "unable to map key\n"); 4580 return -ENOMEM; 4581 } 4582 } 4583 4584 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4585 DMA_BIDIRECTIONAL, 4586 DMA_ATTR_SKIP_CPU_SYNC); 4587 if (dma_mapping_error(ctx->dev, dma_addr)) { 4588 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4589 if (ctx->adata.key_dma) 4590 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4591 ARRAY_SIZE(ctx->key), 4592 DMA_TO_DEVICE, 4593 DMA_ATTR_SKIP_CPU_SYNC); 4594 return -ENOMEM; 4595 } 4596 4597 for (i = 0; i < HASH_NUM_OP; i++) 4598 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4599 4600 /* copy descriptor header template value */ 4601 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4602 4603 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4604 OP_ALG_ALGSEL_SUBMASK) >> 4605 OP_ALG_ALGSEL_SHIFT]; 4606 4607 crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state)); 4608 4609 /* 4610 * For keyed hash algorithms shared descriptors 4611 * will be created later in setkey() callback 4612 */ 4613 return alg->setkey ? 0 : ahash_set_sh_desc(ahash); 4614 } 4615 4616 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4617 { 4618 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4619 4620 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4621 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4622 if (ctx->adata.key_dma) 4623 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4624 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4625 DMA_ATTR_SKIP_CPU_SYNC); 4626 } 4627 4628 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4629 struct caam_hash_template *template, bool keyed) 4630 { 4631 struct caam_hash_alg *t_alg; 4632 struct ahash_alg *halg; 4633 struct crypto_alg *alg; 4634 4635 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4636 if (!t_alg) 4637 return ERR_PTR(-ENOMEM); 4638 4639 t_alg->ahash_alg = template->template_ahash; 4640 halg = &t_alg->ahash_alg; 4641 alg = &halg->halg.base; 4642 4643 if (keyed) { 4644 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4645 template->hmac_name); 4646 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4647 template->hmac_driver_name); 4648 } else { 4649 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4650 template->name); 4651 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4652 template->driver_name); 4653 t_alg->ahash_alg.setkey = NULL; 4654 } 4655 alg->cra_module = THIS_MODULE; 4656 alg->cra_init = caam_hash_cra_init; 4657 alg->cra_exit = caam_hash_cra_exit; 4658 alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding(); 4659 alg->cra_priority = CAAM_CRA_PRIORITY; 4660 alg->cra_blocksize = template->blocksize; 4661 alg->cra_alignmask = 0; 4662 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4663 4664 t_alg->alg_type = template->alg_type; 4665 t_alg->dev = dev; 4666 4667 return t_alg; 4668 } 4669 4670 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4671 { 4672 struct dpaa2_caam_priv_per_cpu *ppriv; 4673 4674 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4675 napi_schedule_irqoff(&ppriv->napi); 4676 } 4677 4678 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4679 { 4680 struct device *dev = priv->dev; 4681 struct dpaa2_io_notification_ctx *nctx; 4682 struct dpaa2_caam_priv_per_cpu *ppriv; 4683 int err, i = 0, cpu; 4684 4685 for_each_online_cpu(cpu) { 4686 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4687 ppriv->priv = priv; 4688 nctx = &ppriv->nctx; 4689 nctx->is_cdan = 0; 4690 nctx->id = ppriv->rsp_fqid; 4691 nctx->desired_cpu = cpu; 4692 nctx->cb = dpaa2_caam_fqdan_cb; 4693 4694 /* Register notification callbacks */ 4695 ppriv->dpio = dpaa2_io_service_select(cpu); 4696 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4697 if (unlikely(err)) { 4698 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4699 nctx->cb = NULL; 4700 /* 4701 * If no affine DPIO for this core, there's probably 4702 * none available for next cores either. Signal we want 4703 * to retry later, in case the DPIO devices weren't 4704 * probed yet. 4705 */ 4706 err = -EPROBE_DEFER; 4707 goto err; 4708 } 4709 4710 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4711 dev); 4712 if (unlikely(!ppriv->store)) { 4713 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4714 err = -ENOMEM; 4715 goto err; 4716 } 4717 4718 if (++i == priv->num_pairs) 4719 break; 4720 } 4721 4722 return 0; 4723 4724 err: 4725 for_each_online_cpu(cpu) { 4726 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4727 if (!ppriv->nctx.cb) 4728 break; 4729 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4730 } 4731 4732 for_each_online_cpu(cpu) { 4733 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4734 if (!ppriv->store) 4735 break; 4736 dpaa2_io_store_destroy(ppriv->store); 4737 } 4738 4739 return err; 4740 } 4741 4742 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4743 { 4744 struct dpaa2_caam_priv_per_cpu *ppriv; 4745 int i = 0, cpu; 4746 4747 for_each_online_cpu(cpu) { 4748 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4749 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4750 priv->dev); 4751 dpaa2_io_store_destroy(ppriv->store); 4752 4753 if (++i == priv->num_pairs) 4754 return; 4755 } 4756 } 4757 4758 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4759 { 4760 struct dpseci_rx_queue_cfg rx_queue_cfg; 4761 struct device *dev = priv->dev; 4762 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4763 struct dpaa2_caam_priv_per_cpu *ppriv; 4764 int err = 0, i = 0, cpu; 4765 4766 /* Configure Rx queues */ 4767 for_each_online_cpu(cpu) { 4768 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4769 4770 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4771 DPSECI_QUEUE_OPT_USER_CTX; 4772 rx_queue_cfg.order_preservation_en = 0; 4773 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4774 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4775 /* 4776 * Rx priority (WQ) doesn't really matter, since we use 4777 * pull mode, i.e. volatile dequeues from specific FQs 4778 */ 4779 rx_queue_cfg.dest_cfg.priority = 0; 4780 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4781 4782 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4783 &rx_queue_cfg); 4784 if (err) { 4785 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4786 err); 4787 return err; 4788 } 4789 4790 if (++i == priv->num_pairs) 4791 break; 4792 } 4793 4794 return err; 4795 } 4796 4797 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4798 { 4799 struct device *dev = priv->dev; 4800 4801 if (!priv->cscn_mem) 4802 return; 4803 4804 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4805 kfree(priv->cscn_mem); 4806 } 4807 4808 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4809 { 4810 struct device *dev = priv->dev; 4811 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4812 int err; 4813 4814 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 4815 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 4816 if (err) 4817 dev_err(dev, "dpseci_reset() failed\n"); 4818 } 4819 4820 dpaa2_dpseci_congestion_free(priv); 4821 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4822 } 4823 4824 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4825 const struct dpaa2_fd *fd) 4826 { 4827 struct caam_request *req; 4828 u32 fd_err; 4829 4830 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4831 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4832 return; 4833 } 4834 4835 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4836 if (unlikely(fd_err)) 4837 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4838 4839 /* 4840 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4841 * in FD[ERR] or FD[FRC]. 4842 */ 4843 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4844 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4845 DMA_BIDIRECTIONAL); 4846 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4847 } 4848 4849 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4850 { 4851 int err; 4852 4853 /* Retry while portal is busy */ 4854 do { 4855 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4856 ppriv->store); 4857 } while (err == -EBUSY); 4858 4859 if (unlikely(err)) 4860 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4861 4862 return err; 4863 } 4864 4865 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4866 { 4867 struct dpaa2_dq *dq; 4868 int cleaned = 0, is_last; 4869 4870 do { 4871 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4872 if (unlikely(!dq)) { 4873 if (unlikely(!is_last)) { 4874 dev_dbg(ppriv->priv->dev, 4875 "FQ %d returned no valid frames\n", 4876 ppriv->rsp_fqid); 4877 /* 4878 * MUST retry until we get some sort of 4879 * valid response token (be it "empty dequeue" 4880 * or a valid frame). 4881 */ 4882 continue; 4883 } 4884 break; 4885 } 4886 4887 /* Process FD */ 4888 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4889 cleaned++; 4890 } while (!is_last); 4891 4892 return cleaned; 4893 } 4894 4895 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4896 { 4897 struct dpaa2_caam_priv_per_cpu *ppriv; 4898 struct dpaa2_caam_priv *priv; 4899 int err, cleaned = 0, store_cleaned; 4900 4901 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4902 priv = ppriv->priv; 4903 4904 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4905 return 0; 4906 4907 do { 4908 store_cleaned = dpaa2_caam_store_consume(ppriv); 4909 cleaned += store_cleaned; 4910 4911 if (store_cleaned == 0 || 4912 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4913 break; 4914 4915 /* Try to dequeue some more */ 4916 err = dpaa2_caam_pull_fq(ppriv); 4917 if (unlikely(err)) 4918 break; 4919 } while (1); 4920 4921 if (cleaned < budget) { 4922 napi_complete_done(napi, cleaned); 4923 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4924 if (unlikely(err)) 4925 dev_err(priv->dev, "Notification rearm failed: %d\n", 4926 err); 4927 } 4928 4929 return cleaned; 4930 } 4931 4932 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4933 u16 token) 4934 { 4935 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4936 struct device *dev = priv->dev; 4937 unsigned int alignmask; 4938 int err; 4939 4940 /* 4941 * Congestion group feature supported starting with DPSECI API v5.1 4942 * and only when object has been created with this capability. 4943 */ 4944 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4945 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4946 return 0; 4947 4948 alignmask = DPAA2_CSCN_ALIGN - 1; 4949 alignmask |= dma_get_cache_alignment() - 1; 4950 priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1), 4951 GFP_KERNEL); 4952 if (!priv->cscn_mem) 4953 return -ENOMEM; 4954 4955 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem, 4956 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4957 if (dma_mapping_error(dev, priv->cscn_dma)) { 4958 dev_err(dev, "Error mapping CSCN memory area\n"); 4959 err = -ENOMEM; 4960 goto err_dma_map; 4961 } 4962 4963 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4964 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4965 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4966 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4967 cong_notif_cfg.message_iova = priv->cscn_dma; 4968 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4969 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4970 DPSECI_CGN_MODE_COHERENT_WRITE; 4971 4972 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4973 &cong_notif_cfg); 4974 if (err) { 4975 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4976 goto err_set_cong; 4977 } 4978 4979 return 0; 4980 4981 err_set_cong: 4982 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4983 err_dma_map: 4984 kfree(priv->cscn_mem); 4985 4986 return err; 4987 } 4988 4989 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4990 { 4991 struct device *dev = &ls_dev->dev; 4992 struct dpaa2_caam_priv *priv; 4993 struct dpaa2_caam_priv_per_cpu *ppriv; 4994 int err, cpu; 4995 u8 i; 4996 4997 priv = dev_get_drvdata(dev); 4998 4999 priv->dev = dev; 5000 priv->dpsec_id = ls_dev->obj_desc.id; 5001 5002 /* Get a handle for the DPSECI this interface is associate with */ 5003 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 5004 if (err) { 5005 dev_err(dev, "dpseci_open() failed: %d\n", err); 5006 goto err_open; 5007 } 5008 5009 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 5010 &priv->minor_ver); 5011 if (err) { 5012 dev_err(dev, "dpseci_get_api_version() failed\n"); 5013 goto err_get_vers; 5014 } 5015 5016 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 5017 5018 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 5019 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 5020 if (err) { 5021 dev_err(dev, "dpseci_reset() failed\n"); 5022 goto err_get_vers; 5023 } 5024 } 5025 5026 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 5027 &priv->dpseci_attr); 5028 if (err) { 5029 dev_err(dev, "dpseci_get_attributes() failed\n"); 5030 goto err_get_vers; 5031 } 5032 5033 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 5034 &priv->sec_attr); 5035 if (err) { 5036 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 5037 goto err_get_vers; 5038 } 5039 5040 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 5041 if (err) { 5042 dev_err(dev, "setup_congestion() failed\n"); 5043 goto err_get_vers; 5044 } 5045 5046 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 5047 priv->dpseci_attr.num_tx_queues); 5048 if (priv->num_pairs > num_online_cpus()) { 5049 dev_warn(dev, "%d queues won't be used\n", 5050 priv->num_pairs - num_online_cpus()); 5051 priv->num_pairs = num_online_cpus(); 5052 } 5053 5054 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 5055 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5056 &priv->rx_queue_attr[i]); 5057 if (err) { 5058 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 5059 goto err_get_rx_queue; 5060 } 5061 } 5062 5063 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 5064 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5065 &priv->tx_queue_attr[i]); 5066 if (err) { 5067 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 5068 goto err_get_rx_queue; 5069 } 5070 } 5071 5072 i = 0; 5073 for_each_online_cpu(cpu) { 5074 u8 j; 5075 5076 j = i % priv->num_pairs; 5077 5078 ppriv = per_cpu_ptr(priv->ppriv, cpu); 5079 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 5080 5081 /* 5082 * Allow all cores to enqueue, while only some of them 5083 * will take part in dequeuing. 5084 */ 5085 if (++i > priv->num_pairs) 5086 continue; 5087 5088 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 5089 ppriv->prio = j; 5090 5091 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 5092 priv->rx_queue_attr[j].fqid, 5093 priv->tx_queue_attr[j].fqid); 5094 5095 ppriv->net_dev.dev = *dev; 5096 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 5097 netif_napi_add_tx_weight(&ppriv->net_dev, &ppriv->napi, 5098 dpaa2_dpseci_poll, 5099 DPAA2_CAAM_NAPI_WEIGHT); 5100 } 5101 5102 return 0; 5103 5104 err_get_rx_queue: 5105 dpaa2_dpseci_congestion_free(priv); 5106 err_get_vers: 5107 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5108 err_open: 5109 return err; 5110 } 5111 5112 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 5113 { 5114 struct device *dev = priv->dev; 5115 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5116 struct dpaa2_caam_priv_per_cpu *ppriv; 5117 int i; 5118 5119 for (i = 0; i < priv->num_pairs; i++) { 5120 ppriv = per_cpu_ptr(priv->ppriv, i); 5121 napi_enable(&ppriv->napi); 5122 } 5123 5124 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 5125 } 5126 5127 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 5128 { 5129 struct device *dev = priv->dev; 5130 struct dpaa2_caam_priv_per_cpu *ppriv; 5131 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5132 int i, err = 0, enabled; 5133 5134 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5135 if (err) { 5136 dev_err(dev, "dpseci_disable() failed\n"); 5137 return err; 5138 } 5139 5140 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5141 if (err) { 5142 dev_err(dev, "dpseci_is_enabled() failed\n"); 5143 return err; 5144 } 5145 5146 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 5147 5148 for (i = 0; i < priv->num_pairs; i++) { 5149 ppriv = per_cpu_ptr(priv->ppriv, i); 5150 napi_disable(&ppriv->napi); 5151 netif_napi_del(&ppriv->napi); 5152 } 5153 5154 return 0; 5155 } 5156 5157 static struct list_head hash_list; 5158 5159 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5160 { 5161 struct device *dev; 5162 struct dpaa2_caam_priv *priv; 5163 int i, err = 0; 5164 bool registered = false; 5165 5166 /* 5167 * There is no way to get CAAM endianness - there is no direct register 5168 * space access and MC f/w does not provide this attribute. 5169 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5170 * property. 5171 */ 5172 caam_little_end = true; 5173 5174 caam_imx = false; 5175 5176 dev = &dpseci_dev->dev; 5177 5178 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5179 if (!priv) 5180 return -ENOMEM; 5181 5182 dev_set_drvdata(dev, priv); 5183 5184 priv->domain = iommu_get_domain_for_dev(dev); 5185 5186 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5187 0, 0, NULL); 5188 if (!qi_cache) { 5189 dev_err(dev, "Can't allocate SEC cache\n"); 5190 return -ENOMEM; 5191 } 5192 5193 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5194 if (err) { 5195 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5196 goto err_dma_mask; 5197 } 5198 5199 /* Obtain a MC portal */ 5200 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5201 if (err) { 5202 if (err == -ENXIO) 5203 err = -EPROBE_DEFER; 5204 else 5205 dev_err(dev, "MC portal allocation failed\n"); 5206 5207 goto err_dma_mask; 5208 } 5209 5210 priv->ppriv = alloc_percpu(*priv->ppriv); 5211 if (!priv->ppriv) { 5212 dev_err(dev, "alloc_percpu() failed\n"); 5213 err = -ENOMEM; 5214 goto err_alloc_ppriv; 5215 } 5216 5217 /* DPSECI initialization */ 5218 err = dpaa2_dpseci_setup(dpseci_dev); 5219 if (err) { 5220 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5221 goto err_dpseci_setup; 5222 } 5223 5224 /* DPIO */ 5225 err = dpaa2_dpseci_dpio_setup(priv); 5226 if (err) { 5227 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); 5228 goto err_dpio_setup; 5229 } 5230 5231 /* DPSECI binding to DPIO */ 5232 err = dpaa2_dpseci_bind(priv); 5233 if (err) { 5234 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5235 goto err_bind; 5236 } 5237 5238 /* DPSECI enable */ 5239 err = dpaa2_dpseci_enable(priv); 5240 if (err) { 5241 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5242 goto err_bind; 5243 } 5244 5245 dpaa2_dpseci_debugfs_init(priv); 5246 5247 /* register crypto algorithms the device supports */ 5248 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5249 struct caam_skcipher_alg *t_alg = driver_algs + i; 5250 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5251 5252 /* Skip DES algorithms if not supported by device */ 5253 if (!priv->sec_attr.des_acc_num && 5254 (alg_sel == OP_ALG_ALGSEL_3DES || 5255 alg_sel == OP_ALG_ALGSEL_DES)) 5256 continue; 5257 5258 /* Skip AES algorithms if not supported by device */ 5259 if (!priv->sec_attr.aes_acc_num && 5260 alg_sel == OP_ALG_ALGSEL_AES) 5261 continue; 5262 5263 /* Skip CHACHA20 algorithms if not supported by device */ 5264 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5265 !priv->sec_attr.ccha_acc_num) 5266 continue; 5267 5268 t_alg->caam.dev = dev; 5269 caam_skcipher_alg_init(t_alg); 5270 5271 err = crypto_register_skcipher(&t_alg->skcipher); 5272 if (err) { 5273 dev_warn(dev, "%s alg registration failed: %d\n", 5274 t_alg->skcipher.base.cra_driver_name, err); 5275 continue; 5276 } 5277 5278 t_alg->registered = true; 5279 registered = true; 5280 } 5281 5282 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5283 struct caam_aead_alg *t_alg = driver_aeads + i; 5284 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5285 OP_ALG_ALGSEL_MASK; 5286 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5287 OP_ALG_ALGSEL_MASK; 5288 5289 /* Skip DES algorithms if not supported by device */ 5290 if (!priv->sec_attr.des_acc_num && 5291 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5292 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5293 continue; 5294 5295 /* Skip AES algorithms if not supported by device */ 5296 if (!priv->sec_attr.aes_acc_num && 5297 c1_alg_sel == OP_ALG_ALGSEL_AES) 5298 continue; 5299 5300 /* Skip CHACHA20 algorithms if not supported by device */ 5301 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5302 !priv->sec_attr.ccha_acc_num) 5303 continue; 5304 5305 /* Skip POLY1305 algorithms if not supported by device */ 5306 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5307 !priv->sec_attr.ptha_acc_num) 5308 continue; 5309 5310 /* 5311 * Skip algorithms requiring message digests 5312 * if MD not supported by device. 5313 */ 5314 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5315 !priv->sec_attr.md_acc_num) 5316 continue; 5317 5318 t_alg->caam.dev = dev; 5319 caam_aead_alg_init(t_alg); 5320 5321 err = crypto_register_aead(&t_alg->aead); 5322 if (err) { 5323 dev_warn(dev, "%s alg registration failed: %d\n", 5324 t_alg->aead.base.cra_driver_name, err); 5325 continue; 5326 } 5327 5328 t_alg->registered = true; 5329 registered = true; 5330 } 5331 if (registered) 5332 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5333 5334 /* register hash algorithms the device supports */ 5335 INIT_LIST_HEAD(&hash_list); 5336 5337 /* 5338 * Skip registration of any hashing algorithms if MD block 5339 * is not present. 5340 */ 5341 if (!priv->sec_attr.md_acc_num) 5342 return 0; 5343 5344 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5345 struct caam_hash_alg *t_alg; 5346 struct caam_hash_template *alg = driver_hash + i; 5347 5348 /* register hmac version */ 5349 t_alg = caam_hash_alloc(dev, alg, true); 5350 if (IS_ERR(t_alg)) { 5351 err = PTR_ERR(t_alg); 5352 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5353 alg->hmac_driver_name, err); 5354 continue; 5355 } 5356 5357 err = crypto_register_ahash(&t_alg->ahash_alg); 5358 if (err) { 5359 dev_warn(dev, "%s alg registration failed: %d\n", 5360 t_alg->ahash_alg.halg.base.cra_driver_name, 5361 err); 5362 kfree(t_alg); 5363 } else { 5364 list_add_tail(&t_alg->entry, &hash_list); 5365 } 5366 5367 /* register unkeyed version */ 5368 t_alg = caam_hash_alloc(dev, alg, false); 5369 if (IS_ERR(t_alg)) { 5370 err = PTR_ERR(t_alg); 5371 dev_warn(dev, "%s alg allocation failed: %d\n", 5372 alg->driver_name, err); 5373 continue; 5374 } 5375 5376 err = crypto_register_ahash(&t_alg->ahash_alg); 5377 if (err) { 5378 dev_warn(dev, "%s alg registration failed: %d\n", 5379 t_alg->ahash_alg.halg.base.cra_driver_name, 5380 err); 5381 kfree(t_alg); 5382 } else { 5383 list_add_tail(&t_alg->entry, &hash_list); 5384 } 5385 } 5386 if (!list_empty(&hash_list)) 5387 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5388 5389 return err; 5390 5391 err_bind: 5392 dpaa2_dpseci_dpio_free(priv); 5393 err_dpio_setup: 5394 dpaa2_dpseci_free(priv); 5395 err_dpseci_setup: 5396 free_percpu(priv->ppriv); 5397 err_alloc_ppriv: 5398 fsl_mc_portal_free(priv->mc_io); 5399 err_dma_mask: 5400 kmem_cache_destroy(qi_cache); 5401 5402 return err; 5403 } 5404 5405 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5406 { 5407 struct device *dev; 5408 struct dpaa2_caam_priv *priv; 5409 int i; 5410 5411 dev = &ls_dev->dev; 5412 priv = dev_get_drvdata(dev); 5413 5414 dpaa2_dpseci_debugfs_exit(priv); 5415 5416 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5417 struct caam_aead_alg *t_alg = driver_aeads + i; 5418 5419 if (t_alg->registered) 5420 crypto_unregister_aead(&t_alg->aead); 5421 } 5422 5423 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5424 struct caam_skcipher_alg *t_alg = driver_algs + i; 5425 5426 if (t_alg->registered) 5427 crypto_unregister_skcipher(&t_alg->skcipher); 5428 } 5429 5430 if (hash_list.next) { 5431 struct caam_hash_alg *t_hash_alg, *p; 5432 5433 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5434 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5435 list_del(&t_hash_alg->entry); 5436 kfree(t_hash_alg); 5437 } 5438 } 5439 5440 dpaa2_dpseci_disable(priv); 5441 dpaa2_dpseci_dpio_free(priv); 5442 dpaa2_dpseci_free(priv); 5443 free_percpu(priv->ppriv); 5444 fsl_mc_portal_free(priv->mc_io); 5445 kmem_cache_destroy(qi_cache); 5446 5447 return 0; 5448 } 5449 5450 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5451 { 5452 struct dpaa2_fd fd; 5453 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5454 struct dpaa2_caam_priv_per_cpu *ppriv; 5455 int err = 0, i; 5456 5457 if (IS_ERR(req)) 5458 return PTR_ERR(req); 5459 5460 if (priv->cscn_mem) { 5461 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5462 DPAA2_CSCN_SIZE, 5463 DMA_FROM_DEVICE); 5464 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) { 5465 dev_dbg_ratelimited(dev, "Dropping request\n"); 5466 return -EBUSY; 5467 } 5468 } 5469 5470 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5471 5472 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5473 DMA_BIDIRECTIONAL); 5474 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5475 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5476 goto err_out; 5477 } 5478 5479 memset(&fd, 0, sizeof(fd)); 5480 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5481 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5482 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5483 dpaa2_fd_set_flc(&fd, req->flc_dma); 5484 5485 ppriv = raw_cpu_ptr(priv->ppriv); 5486 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5487 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5488 &fd); 5489 if (err != -EBUSY) 5490 break; 5491 5492 cpu_relax(); 5493 } 5494 5495 if (unlikely(err)) { 5496 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5497 goto err_out; 5498 } 5499 5500 return -EINPROGRESS; 5501 5502 err_out: 5503 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5504 DMA_BIDIRECTIONAL); 5505 return -EIO; 5506 } 5507 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5508 5509 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5510 { 5511 .vendor = FSL_MC_VENDOR_FREESCALE, 5512 .obj_type = "dpseci", 5513 }, 5514 { .vendor = 0x0 } 5515 }; 5516 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table); 5517 5518 static struct fsl_mc_driver dpaa2_caam_driver = { 5519 .driver = { 5520 .name = KBUILD_MODNAME, 5521 .owner = THIS_MODULE, 5522 }, 5523 .probe = dpaa2_caam_probe, 5524 .remove = dpaa2_caam_remove, 5525 .match_id_table = dpaa2_caam_match_id_table 5526 }; 5527 5528 MODULE_LICENSE("Dual BSD/GPL"); 5529 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5530 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5531 5532 module_fsl_mc_driver(dpaa2_caam_driver); 5533