1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include "dpseci-debugfs.h" 19 #include <linux/fsl/mc.h> 20 #include <soc/fsl/dpaa2-io.h> 21 #include <soc/fsl/dpaa2-fd.h> 22 23 #define CAAM_CRA_PRIORITY 2000 24 25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 26 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 27 SHA512_DIGEST_SIZE * 2) 28 29 /* 30 * This is a a cache of buffers, from which the users of CAAM QI driver 31 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 32 * NOTE: A more elegant solution would be to have some headroom in the frames 33 * being processed. This can be added by the dpaa2-eth driver. This would 34 * pose a problem for userspace application processing which cannot 35 * know of this limitation. So for now, this will work. 36 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 37 */ 38 static struct kmem_cache *qi_cache; 39 40 struct caam_alg_entry { 41 struct device *dev; 42 int class1_alg_type; 43 int class2_alg_type; 44 bool rfc3686; 45 bool geniv; 46 bool nodkp; 47 }; 48 49 struct caam_aead_alg { 50 struct aead_alg aead; 51 struct caam_alg_entry caam; 52 bool registered; 53 }; 54 55 struct caam_skcipher_alg { 56 struct skcipher_alg skcipher; 57 struct caam_alg_entry caam; 58 bool registered; 59 }; 60 61 /** 62 * caam_ctx - per-session context 63 * @flc: Flow Contexts array 64 * @key: [authentication key], encryption key 65 * @flc_dma: I/O virtual addresses of the Flow Contexts 66 * @key_dma: I/O virtual address of the key 67 * @dir: DMA direction for mapping key and Flow Contexts 68 * @dev: dpseci device 69 * @adata: authentication algorithm details 70 * @cdata: encryption algorithm details 71 * @authsize: authentication tag (a.k.a. ICV / MAC) size 72 */ 73 struct caam_ctx { 74 struct caam_flc flc[NUM_OP]; 75 u8 key[CAAM_MAX_KEY_SIZE]; 76 dma_addr_t flc_dma[NUM_OP]; 77 dma_addr_t key_dma; 78 enum dma_data_direction dir; 79 struct device *dev; 80 struct alginfo adata; 81 struct alginfo cdata; 82 unsigned int authsize; 83 }; 84 85 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 86 dma_addr_t iova_addr) 87 { 88 phys_addr_t phys_addr; 89 90 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 91 iova_addr; 92 93 return phys_to_virt(phys_addr); 94 } 95 96 /* 97 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 98 * 99 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 100 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 101 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 102 * hosting 16 SG entries. 103 * 104 * @flags - flags that would be used for the equivalent kmalloc(..) call 105 * 106 * Returns a pointer to a retrieved buffer on success or NULL on failure. 107 */ 108 static inline void *qi_cache_zalloc(gfp_t flags) 109 { 110 return kmem_cache_zalloc(qi_cache, flags); 111 } 112 113 /* 114 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 115 * 116 * @obj - buffer previously allocated by qi_cache_zalloc 117 * 118 * No checking is being done, the call is a passthrough call to 119 * kmem_cache_free(...) 120 */ 121 static inline void qi_cache_free(void *obj) 122 { 123 kmem_cache_free(qi_cache, obj); 124 } 125 126 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 127 { 128 switch (crypto_tfm_alg_type(areq->tfm)) { 129 case CRYPTO_ALG_TYPE_SKCIPHER: 130 return skcipher_request_ctx(skcipher_request_cast(areq)); 131 case CRYPTO_ALG_TYPE_AEAD: 132 return aead_request_ctx(container_of(areq, struct aead_request, 133 base)); 134 case CRYPTO_ALG_TYPE_AHASH: 135 return ahash_request_ctx(ahash_request_cast(areq)); 136 default: 137 return ERR_PTR(-EINVAL); 138 } 139 } 140 141 static void caam_unmap(struct device *dev, struct scatterlist *src, 142 struct scatterlist *dst, int src_nents, 143 int dst_nents, dma_addr_t iv_dma, int ivsize, 144 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 145 int qm_sg_bytes) 146 { 147 if (dst != src) { 148 if (src_nents) 149 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 150 if (dst_nents) 151 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 152 } else { 153 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 154 } 155 156 if (iv_dma) 157 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 158 159 if (qm_sg_bytes) 160 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 161 } 162 163 static int aead_set_sh_desc(struct crypto_aead *aead) 164 { 165 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 166 typeof(*alg), aead); 167 struct caam_ctx *ctx = crypto_aead_ctx(aead); 168 unsigned int ivsize = crypto_aead_ivsize(aead); 169 struct device *dev = ctx->dev; 170 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 171 struct caam_flc *flc; 172 u32 *desc; 173 u32 ctx1_iv_off = 0; 174 u32 *nonce = NULL; 175 unsigned int data_len[2]; 176 u32 inl_mask; 177 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 178 OP_ALG_AAI_CTR_MOD128); 179 const bool is_rfc3686 = alg->caam.rfc3686; 180 181 if (!ctx->cdata.keylen || !ctx->authsize) 182 return 0; 183 184 /* 185 * AES-CTR needs to load IV in CONTEXT1 reg 186 * at an offset of 128bits (16bytes) 187 * CONTEXT1[255:128] = IV 188 */ 189 if (ctr_mode) 190 ctx1_iv_off = 16; 191 192 /* 193 * RFC3686 specific: 194 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 195 */ 196 if (is_rfc3686) { 197 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 198 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 199 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 200 } 201 202 /* 203 * In case |user key| > |derived key|, using DKP<imm,imm> would result 204 * in invalid opcodes (last bytes of user key) in the resulting 205 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 206 * addresses are needed. 207 */ 208 ctx->adata.key_virt = ctx->key; 209 ctx->adata.key_dma = ctx->key_dma; 210 211 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 212 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 213 214 data_len[0] = ctx->adata.keylen_pad; 215 data_len[1] = ctx->cdata.keylen; 216 217 /* aead_encrypt shared descriptor */ 218 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 219 DESC_QI_AEAD_ENC_LEN) + 220 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 221 DESC_JOB_IO_LEN, data_len, &inl_mask, 222 ARRAY_SIZE(data_len)) < 0) 223 return -EINVAL; 224 225 ctx->adata.key_inline = !!(inl_mask & 1); 226 ctx->cdata.key_inline = !!(inl_mask & 2); 227 228 flc = &ctx->flc[ENCRYPT]; 229 desc = flc->sh_desc; 230 231 if (alg->caam.geniv) 232 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 233 ivsize, ctx->authsize, is_rfc3686, 234 nonce, ctx1_iv_off, true, 235 priv->sec_attr.era); 236 else 237 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 238 ivsize, ctx->authsize, is_rfc3686, nonce, 239 ctx1_iv_off, true, priv->sec_attr.era); 240 241 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 242 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 243 sizeof(flc->flc) + desc_bytes(desc), 244 ctx->dir); 245 246 /* aead_decrypt shared descriptor */ 247 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 248 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 249 DESC_JOB_IO_LEN, data_len, &inl_mask, 250 ARRAY_SIZE(data_len)) < 0) 251 return -EINVAL; 252 253 ctx->adata.key_inline = !!(inl_mask & 1); 254 ctx->cdata.key_inline = !!(inl_mask & 2); 255 256 flc = &ctx->flc[DECRYPT]; 257 desc = flc->sh_desc; 258 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 259 ivsize, ctx->authsize, alg->caam.geniv, 260 is_rfc3686, nonce, ctx1_iv_off, true, 261 priv->sec_attr.era); 262 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 263 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 264 sizeof(flc->flc) + desc_bytes(desc), 265 ctx->dir); 266 267 return 0; 268 } 269 270 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 271 { 272 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 273 274 ctx->authsize = authsize; 275 aead_set_sh_desc(authenc); 276 277 return 0; 278 } 279 280 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 281 unsigned int keylen) 282 { 283 struct caam_ctx *ctx = crypto_aead_ctx(aead); 284 struct device *dev = ctx->dev; 285 struct crypto_authenc_keys keys; 286 287 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 288 goto badkey; 289 290 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 291 keys.authkeylen + keys.enckeylen, keys.enckeylen, 292 keys.authkeylen); 293 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 294 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 295 296 ctx->adata.keylen = keys.authkeylen; 297 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 298 OP_ALG_ALGSEL_MASK); 299 300 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 301 goto badkey; 302 303 memcpy(ctx->key, keys.authkey, keys.authkeylen); 304 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 305 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 306 keys.enckeylen, ctx->dir); 307 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 308 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 309 ctx->adata.keylen_pad + keys.enckeylen, 1); 310 311 ctx->cdata.keylen = keys.enckeylen; 312 313 memzero_explicit(&keys, sizeof(keys)); 314 return aead_set_sh_desc(aead); 315 badkey: 316 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 317 memzero_explicit(&keys, sizeof(keys)); 318 return -EINVAL; 319 } 320 321 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 322 unsigned int keylen) 323 { 324 struct crypto_authenc_keys keys; 325 int err; 326 327 err = crypto_authenc_extractkeys(&keys, key, keylen); 328 if (unlikely(err)) 329 goto badkey; 330 331 err = -EINVAL; 332 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 333 goto badkey; 334 335 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 336 aead_setkey(aead, key, keylen); 337 338 out: 339 memzero_explicit(&keys, sizeof(keys)); 340 return err; 341 342 badkey: 343 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 344 goto out; 345 } 346 347 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 348 bool encrypt) 349 { 350 struct crypto_aead *aead = crypto_aead_reqtfm(req); 351 struct caam_request *req_ctx = aead_request_ctx(req); 352 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 353 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 354 struct caam_ctx *ctx = crypto_aead_ctx(aead); 355 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 356 typeof(*alg), aead); 357 struct device *dev = ctx->dev; 358 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 359 GFP_KERNEL : GFP_ATOMIC; 360 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 361 int src_len, dst_len = 0; 362 struct aead_edesc *edesc; 363 dma_addr_t qm_sg_dma, iv_dma = 0; 364 int ivsize = 0; 365 unsigned int authsize = ctx->authsize; 366 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 367 int in_len, out_len; 368 struct dpaa2_sg_entry *sg_table; 369 370 /* allocate space for base edesc, link tables and IV */ 371 edesc = qi_cache_zalloc(GFP_DMA | flags); 372 if (unlikely(!edesc)) { 373 dev_err(dev, "could not allocate extended descriptor\n"); 374 return ERR_PTR(-ENOMEM); 375 } 376 377 if (unlikely(req->dst != req->src)) { 378 src_len = req->assoclen + req->cryptlen; 379 dst_len = src_len + (encrypt ? authsize : (-authsize)); 380 381 src_nents = sg_nents_for_len(req->src, src_len); 382 if (unlikely(src_nents < 0)) { 383 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 384 src_len); 385 qi_cache_free(edesc); 386 return ERR_PTR(src_nents); 387 } 388 389 dst_nents = sg_nents_for_len(req->dst, dst_len); 390 if (unlikely(dst_nents < 0)) { 391 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 392 dst_len); 393 qi_cache_free(edesc); 394 return ERR_PTR(dst_nents); 395 } 396 397 if (src_nents) { 398 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 399 DMA_TO_DEVICE); 400 if (unlikely(!mapped_src_nents)) { 401 dev_err(dev, "unable to map source\n"); 402 qi_cache_free(edesc); 403 return ERR_PTR(-ENOMEM); 404 } 405 } else { 406 mapped_src_nents = 0; 407 } 408 409 if (dst_nents) { 410 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 411 DMA_FROM_DEVICE); 412 if (unlikely(!mapped_dst_nents)) { 413 dev_err(dev, "unable to map destination\n"); 414 dma_unmap_sg(dev, req->src, src_nents, 415 DMA_TO_DEVICE); 416 qi_cache_free(edesc); 417 return ERR_PTR(-ENOMEM); 418 } 419 } else { 420 mapped_dst_nents = 0; 421 } 422 } else { 423 src_len = req->assoclen + req->cryptlen + 424 (encrypt ? authsize : 0); 425 426 src_nents = sg_nents_for_len(req->src, src_len); 427 if (unlikely(src_nents < 0)) { 428 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 429 src_len); 430 qi_cache_free(edesc); 431 return ERR_PTR(src_nents); 432 } 433 434 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 435 DMA_BIDIRECTIONAL); 436 if (unlikely(!mapped_src_nents)) { 437 dev_err(dev, "unable to map source\n"); 438 qi_cache_free(edesc); 439 return ERR_PTR(-ENOMEM); 440 } 441 } 442 443 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 444 ivsize = crypto_aead_ivsize(aead); 445 446 /* 447 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 448 * Input is not contiguous. 449 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 450 * the end of the table by allocating more S/G entries. Logic: 451 * if (src != dst && output S/G) 452 * pad output S/G, if needed 453 * else if (src == dst && S/G) 454 * overlapping S/Gs; pad one of them 455 * else if (input S/G) ... 456 * pad input S/G, if needed 457 */ 458 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 459 if (mapped_dst_nents > 1) 460 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 461 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 462 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 463 1 + !!ivsize + 464 pad_sg_nents(mapped_src_nents)); 465 else 466 qm_sg_nents = pad_sg_nents(qm_sg_nents); 467 468 sg_table = &edesc->sgt[0]; 469 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 470 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 471 CAAM_QI_MEMCACHE_SIZE)) { 472 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 473 qm_sg_nents, ivsize); 474 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 475 0, DMA_NONE, 0, 0); 476 qi_cache_free(edesc); 477 return ERR_PTR(-ENOMEM); 478 } 479 480 if (ivsize) { 481 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 482 483 /* Make sure IV is located in a DMAable area */ 484 memcpy(iv, req->iv, ivsize); 485 486 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 487 if (dma_mapping_error(dev, iv_dma)) { 488 dev_err(dev, "unable to map IV\n"); 489 caam_unmap(dev, req->src, req->dst, src_nents, 490 dst_nents, 0, 0, DMA_NONE, 0, 0); 491 qi_cache_free(edesc); 492 return ERR_PTR(-ENOMEM); 493 } 494 } 495 496 edesc->src_nents = src_nents; 497 edesc->dst_nents = dst_nents; 498 edesc->iv_dma = iv_dma; 499 500 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 501 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 502 /* 503 * The associated data comes already with the IV but we need 504 * to skip it when we authenticate or encrypt... 505 */ 506 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 507 else 508 edesc->assoclen = cpu_to_caam32(req->assoclen); 509 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 510 DMA_TO_DEVICE); 511 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 512 dev_err(dev, "unable to map assoclen\n"); 513 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 514 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 515 qi_cache_free(edesc); 516 return ERR_PTR(-ENOMEM); 517 } 518 519 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 520 qm_sg_index++; 521 if (ivsize) { 522 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 523 qm_sg_index++; 524 } 525 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 526 qm_sg_index += mapped_src_nents; 527 528 if (mapped_dst_nents > 1) 529 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 530 531 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 532 if (dma_mapping_error(dev, qm_sg_dma)) { 533 dev_err(dev, "unable to map S/G table\n"); 534 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 535 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 536 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 537 qi_cache_free(edesc); 538 return ERR_PTR(-ENOMEM); 539 } 540 541 edesc->qm_sg_dma = qm_sg_dma; 542 edesc->qm_sg_bytes = qm_sg_bytes; 543 544 out_len = req->assoclen + req->cryptlen + 545 (encrypt ? ctx->authsize : (-ctx->authsize)); 546 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 547 548 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 549 dpaa2_fl_set_final(in_fle, true); 550 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 551 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 552 dpaa2_fl_set_len(in_fle, in_len); 553 554 if (req->dst == req->src) { 555 if (mapped_src_nents == 1) { 556 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 557 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 558 } else { 559 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 560 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 561 (1 + !!ivsize) * sizeof(*sg_table)); 562 } 563 } else if (!mapped_dst_nents) { 564 /* 565 * crypto engine requires the output entry to be present when 566 * "frame list" FD is used. 567 * Since engine does not support FMT=2'b11 (unused entry type), 568 * leaving out_fle zeroized is the best option. 569 */ 570 goto skip_out_fle; 571 } else if (mapped_dst_nents == 1) { 572 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 573 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 574 } else { 575 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 576 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 577 sizeof(*sg_table)); 578 } 579 580 dpaa2_fl_set_len(out_fle, out_len); 581 582 skip_out_fle: 583 return edesc; 584 } 585 586 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 587 { 588 struct caam_ctx *ctx = crypto_aead_ctx(aead); 589 unsigned int ivsize = crypto_aead_ivsize(aead); 590 struct device *dev = ctx->dev; 591 struct caam_flc *flc; 592 u32 *desc; 593 594 if (!ctx->cdata.keylen || !ctx->authsize) 595 return 0; 596 597 flc = &ctx->flc[ENCRYPT]; 598 desc = flc->sh_desc; 599 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 600 ctx->authsize, true, true); 601 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 602 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 603 sizeof(flc->flc) + desc_bytes(desc), 604 ctx->dir); 605 606 flc = &ctx->flc[DECRYPT]; 607 desc = flc->sh_desc; 608 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 609 ctx->authsize, false, true); 610 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 611 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 612 sizeof(flc->flc) + desc_bytes(desc), 613 ctx->dir); 614 615 return 0; 616 } 617 618 static int chachapoly_setauthsize(struct crypto_aead *aead, 619 unsigned int authsize) 620 { 621 struct caam_ctx *ctx = crypto_aead_ctx(aead); 622 623 if (authsize != POLY1305_DIGEST_SIZE) 624 return -EINVAL; 625 626 ctx->authsize = authsize; 627 return chachapoly_set_sh_desc(aead); 628 } 629 630 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 631 unsigned int keylen) 632 { 633 struct caam_ctx *ctx = crypto_aead_ctx(aead); 634 unsigned int ivsize = crypto_aead_ivsize(aead); 635 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 636 637 if (keylen != CHACHA_KEY_SIZE + saltlen) { 638 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 639 return -EINVAL; 640 } 641 642 ctx->cdata.key_virt = key; 643 ctx->cdata.keylen = keylen - saltlen; 644 645 return chachapoly_set_sh_desc(aead); 646 } 647 648 static int gcm_set_sh_desc(struct crypto_aead *aead) 649 { 650 struct caam_ctx *ctx = crypto_aead_ctx(aead); 651 struct device *dev = ctx->dev; 652 unsigned int ivsize = crypto_aead_ivsize(aead); 653 struct caam_flc *flc; 654 u32 *desc; 655 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 656 ctx->cdata.keylen; 657 658 if (!ctx->cdata.keylen || !ctx->authsize) 659 return 0; 660 661 /* 662 * AES GCM encrypt shared descriptor 663 * Job Descriptor and Shared Descriptor 664 * must fit into the 64-word Descriptor h/w Buffer 665 */ 666 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 667 ctx->cdata.key_inline = true; 668 ctx->cdata.key_virt = ctx->key; 669 } else { 670 ctx->cdata.key_inline = false; 671 ctx->cdata.key_dma = ctx->key_dma; 672 } 673 674 flc = &ctx->flc[ENCRYPT]; 675 desc = flc->sh_desc; 676 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 677 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 678 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 679 sizeof(flc->flc) + desc_bytes(desc), 680 ctx->dir); 681 682 /* 683 * Job Descriptor and Shared Descriptors 684 * must all fit into the 64-word Descriptor h/w Buffer 685 */ 686 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 687 ctx->cdata.key_inline = true; 688 ctx->cdata.key_virt = ctx->key; 689 } else { 690 ctx->cdata.key_inline = false; 691 ctx->cdata.key_dma = ctx->key_dma; 692 } 693 694 flc = &ctx->flc[DECRYPT]; 695 desc = flc->sh_desc; 696 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 697 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 698 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 699 sizeof(flc->flc) + desc_bytes(desc), 700 ctx->dir); 701 702 return 0; 703 } 704 705 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 706 { 707 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 708 int err; 709 710 err = crypto_gcm_check_authsize(authsize); 711 if (err) 712 return err; 713 714 ctx->authsize = authsize; 715 gcm_set_sh_desc(authenc); 716 717 return 0; 718 } 719 720 static int gcm_setkey(struct crypto_aead *aead, 721 const u8 *key, unsigned int keylen) 722 { 723 struct caam_ctx *ctx = crypto_aead_ctx(aead); 724 struct device *dev = ctx->dev; 725 int ret; 726 727 ret = aes_check_keylen(keylen); 728 if (ret) { 729 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 730 return ret; 731 } 732 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 733 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 734 735 memcpy(ctx->key, key, keylen); 736 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 737 ctx->cdata.keylen = keylen; 738 739 return gcm_set_sh_desc(aead); 740 } 741 742 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 743 { 744 struct caam_ctx *ctx = crypto_aead_ctx(aead); 745 struct device *dev = ctx->dev; 746 unsigned int ivsize = crypto_aead_ivsize(aead); 747 struct caam_flc *flc; 748 u32 *desc; 749 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 750 ctx->cdata.keylen; 751 752 if (!ctx->cdata.keylen || !ctx->authsize) 753 return 0; 754 755 ctx->cdata.key_virt = ctx->key; 756 757 /* 758 * RFC4106 encrypt shared descriptor 759 * Job Descriptor and Shared Descriptor 760 * must fit into the 64-word Descriptor h/w Buffer 761 */ 762 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 763 ctx->cdata.key_inline = true; 764 } else { 765 ctx->cdata.key_inline = false; 766 ctx->cdata.key_dma = ctx->key_dma; 767 } 768 769 flc = &ctx->flc[ENCRYPT]; 770 desc = flc->sh_desc; 771 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 772 true); 773 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 774 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 775 sizeof(flc->flc) + desc_bytes(desc), 776 ctx->dir); 777 778 /* 779 * Job Descriptor and Shared Descriptors 780 * must all fit into the 64-word Descriptor h/w Buffer 781 */ 782 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 783 ctx->cdata.key_inline = true; 784 } else { 785 ctx->cdata.key_inline = false; 786 ctx->cdata.key_dma = ctx->key_dma; 787 } 788 789 flc = &ctx->flc[DECRYPT]; 790 desc = flc->sh_desc; 791 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 792 true); 793 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 794 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 795 sizeof(flc->flc) + desc_bytes(desc), 796 ctx->dir); 797 798 return 0; 799 } 800 801 static int rfc4106_setauthsize(struct crypto_aead *authenc, 802 unsigned int authsize) 803 { 804 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 805 int err; 806 807 err = crypto_rfc4106_check_authsize(authsize); 808 if (err) 809 return err; 810 811 ctx->authsize = authsize; 812 rfc4106_set_sh_desc(authenc); 813 814 return 0; 815 } 816 817 static int rfc4106_setkey(struct crypto_aead *aead, 818 const u8 *key, unsigned int keylen) 819 { 820 struct caam_ctx *ctx = crypto_aead_ctx(aead); 821 struct device *dev = ctx->dev; 822 int ret; 823 824 ret = aes_check_keylen(keylen - 4); 825 if (ret) { 826 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 827 return ret; 828 } 829 830 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 831 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 832 833 memcpy(ctx->key, key, keylen); 834 /* 835 * The last four bytes of the key material are used as the salt value 836 * in the nonce. Update the AES key length. 837 */ 838 ctx->cdata.keylen = keylen - 4; 839 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 840 ctx->dir); 841 842 return rfc4106_set_sh_desc(aead); 843 } 844 845 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 846 { 847 struct caam_ctx *ctx = crypto_aead_ctx(aead); 848 struct device *dev = ctx->dev; 849 unsigned int ivsize = crypto_aead_ivsize(aead); 850 struct caam_flc *flc; 851 u32 *desc; 852 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 853 ctx->cdata.keylen; 854 855 if (!ctx->cdata.keylen || !ctx->authsize) 856 return 0; 857 858 ctx->cdata.key_virt = ctx->key; 859 860 /* 861 * RFC4543 encrypt shared descriptor 862 * Job Descriptor and Shared Descriptor 863 * must fit into the 64-word Descriptor h/w Buffer 864 */ 865 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 866 ctx->cdata.key_inline = true; 867 } else { 868 ctx->cdata.key_inline = false; 869 ctx->cdata.key_dma = ctx->key_dma; 870 } 871 872 flc = &ctx->flc[ENCRYPT]; 873 desc = flc->sh_desc; 874 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 875 true); 876 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 877 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 878 sizeof(flc->flc) + desc_bytes(desc), 879 ctx->dir); 880 881 /* 882 * Job Descriptor and Shared Descriptors 883 * must all fit into the 64-word Descriptor h/w Buffer 884 */ 885 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 886 ctx->cdata.key_inline = true; 887 } else { 888 ctx->cdata.key_inline = false; 889 ctx->cdata.key_dma = ctx->key_dma; 890 } 891 892 flc = &ctx->flc[DECRYPT]; 893 desc = flc->sh_desc; 894 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 895 true); 896 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 897 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 898 sizeof(flc->flc) + desc_bytes(desc), 899 ctx->dir); 900 901 return 0; 902 } 903 904 static int rfc4543_setauthsize(struct crypto_aead *authenc, 905 unsigned int authsize) 906 { 907 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 908 909 if (authsize != 16) 910 return -EINVAL; 911 912 ctx->authsize = authsize; 913 rfc4543_set_sh_desc(authenc); 914 915 return 0; 916 } 917 918 static int rfc4543_setkey(struct crypto_aead *aead, 919 const u8 *key, unsigned int keylen) 920 { 921 struct caam_ctx *ctx = crypto_aead_ctx(aead); 922 struct device *dev = ctx->dev; 923 int ret; 924 925 ret = aes_check_keylen(keylen - 4); 926 if (ret) { 927 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 928 return ret; 929 } 930 931 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 932 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 933 934 memcpy(ctx->key, key, keylen); 935 /* 936 * The last four bytes of the key material are used as the salt value 937 * in the nonce. Update the AES key length. 938 */ 939 ctx->cdata.keylen = keylen - 4; 940 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 941 ctx->dir); 942 943 return rfc4543_set_sh_desc(aead); 944 } 945 946 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 947 unsigned int keylen, const u32 ctx1_iv_off) 948 { 949 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 950 struct caam_skcipher_alg *alg = 951 container_of(crypto_skcipher_alg(skcipher), 952 struct caam_skcipher_alg, skcipher); 953 struct device *dev = ctx->dev; 954 struct caam_flc *flc; 955 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 956 u32 *desc; 957 const bool is_rfc3686 = alg->caam.rfc3686; 958 959 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 960 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 961 962 ctx->cdata.keylen = keylen; 963 ctx->cdata.key_virt = key; 964 ctx->cdata.key_inline = true; 965 966 /* skcipher_encrypt shared descriptor */ 967 flc = &ctx->flc[ENCRYPT]; 968 desc = flc->sh_desc; 969 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 970 ctx1_iv_off); 971 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 972 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 973 sizeof(flc->flc) + desc_bytes(desc), 974 ctx->dir); 975 976 /* skcipher_decrypt shared descriptor */ 977 flc = &ctx->flc[DECRYPT]; 978 desc = flc->sh_desc; 979 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 980 ctx1_iv_off); 981 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 982 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 983 sizeof(flc->flc) + desc_bytes(desc), 984 ctx->dir); 985 986 return 0; 987 } 988 989 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 990 const u8 *key, unsigned int keylen) 991 { 992 int err; 993 994 err = aes_check_keylen(keylen); 995 if (err) { 996 crypto_skcipher_set_flags(skcipher, 997 CRYPTO_TFM_RES_BAD_KEY_LEN); 998 return err; 999 } 1000 1001 return skcipher_setkey(skcipher, key, keylen, 0); 1002 } 1003 1004 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 1005 const u8 *key, unsigned int keylen) 1006 { 1007 u32 ctx1_iv_off; 1008 int err; 1009 1010 /* 1011 * RFC3686 specific: 1012 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1013 * | *key = {KEY, NONCE} 1014 */ 1015 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1016 keylen -= CTR_RFC3686_NONCE_SIZE; 1017 1018 err = aes_check_keylen(keylen); 1019 if (err) { 1020 crypto_skcipher_set_flags(skcipher, 1021 CRYPTO_TFM_RES_BAD_KEY_LEN); 1022 return err; 1023 } 1024 1025 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1026 } 1027 1028 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1029 const u8 *key, unsigned int keylen) 1030 { 1031 u32 ctx1_iv_off; 1032 int err; 1033 1034 /* 1035 * AES-CTR needs to load IV in CONTEXT1 reg 1036 * at an offset of 128bits (16bytes) 1037 * CONTEXT1[255:128] = IV 1038 */ 1039 ctx1_iv_off = 16; 1040 1041 err = aes_check_keylen(keylen); 1042 if (err) { 1043 crypto_skcipher_set_flags(skcipher, 1044 CRYPTO_TFM_RES_BAD_KEY_LEN); 1045 return err; 1046 } 1047 1048 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1049 } 1050 1051 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1052 const u8 *key, unsigned int keylen) 1053 { 1054 if (keylen != CHACHA_KEY_SIZE) { 1055 crypto_skcipher_set_flags(skcipher, 1056 CRYPTO_TFM_RES_BAD_KEY_LEN); 1057 return -EINVAL; 1058 } 1059 1060 return skcipher_setkey(skcipher, key, keylen, 0); 1061 } 1062 1063 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1064 const u8 *key, unsigned int keylen) 1065 { 1066 return verify_skcipher_des_key(skcipher, key) ?: 1067 skcipher_setkey(skcipher, key, keylen, 0); 1068 } 1069 1070 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1071 const u8 *key, unsigned int keylen) 1072 { 1073 return verify_skcipher_des3_key(skcipher, key) ?: 1074 skcipher_setkey(skcipher, key, keylen, 0); 1075 } 1076 1077 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1078 unsigned int keylen) 1079 { 1080 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1081 struct device *dev = ctx->dev; 1082 struct caam_flc *flc; 1083 u32 *desc; 1084 1085 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 1086 dev_err(dev, "key size mismatch\n"); 1087 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 1088 return -EINVAL; 1089 } 1090 1091 ctx->cdata.keylen = keylen; 1092 ctx->cdata.key_virt = key; 1093 ctx->cdata.key_inline = true; 1094 1095 /* xts_skcipher_encrypt shared descriptor */ 1096 flc = &ctx->flc[ENCRYPT]; 1097 desc = flc->sh_desc; 1098 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1099 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1100 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1101 sizeof(flc->flc) + desc_bytes(desc), 1102 ctx->dir); 1103 1104 /* xts_skcipher_decrypt shared descriptor */ 1105 flc = &ctx->flc[DECRYPT]; 1106 desc = flc->sh_desc; 1107 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1108 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1109 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1110 sizeof(flc->flc) + desc_bytes(desc), 1111 ctx->dir); 1112 1113 return 0; 1114 } 1115 1116 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1117 { 1118 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1119 struct caam_request *req_ctx = skcipher_request_ctx(req); 1120 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1121 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1122 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1123 struct device *dev = ctx->dev; 1124 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1125 GFP_KERNEL : GFP_ATOMIC; 1126 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1127 struct skcipher_edesc *edesc; 1128 dma_addr_t iv_dma; 1129 u8 *iv; 1130 int ivsize = crypto_skcipher_ivsize(skcipher); 1131 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1132 struct dpaa2_sg_entry *sg_table; 1133 1134 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1135 if (unlikely(src_nents < 0)) { 1136 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1137 req->cryptlen); 1138 return ERR_PTR(src_nents); 1139 } 1140 1141 if (unlikely(req->dst != req->src)) { 1142 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1143 if (unlikely(dst_nents < 0)) { 1144 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1145 req->cryptlen); 1146 return ERR_PTR(dst_nents); 1147 } 1148 1149 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1150 DMA_TO_DEVICE); 1151 if (unlikely(!mapped_src_nents)) { 1152 dev_err(dev, "unable to map source\n"); 1153 return ERR_PTR(-ENOMEM); 1154 } 1155 1156 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1157 DMA_FROM_DEVICE); 1158 if (unlikely(!mapped_dst_nents)) { 1159 dev_err(dev, "unable to map destination\n"); 1160 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1161 return ERR_PTR(-ENOMEM); 1162 } 1163 } else { 1164 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1165 DMA_BIDIRECTIONAL); 1166 if (unlikely(!mapped_src_nents)) { 1167 dev_err(dev, "unable to map source\n"); 1168 return ERR_PTR(-ENOMEM); 1169 } 1170 } 1171 1172 qm_sg_ents = 1 + mapped_src_nents; 1173 dst_sg_idx = qm_sg_ents; 1174 1175 /* 1176 * Input, output HW S/G tables: [IV, src][dst, IV] 1177 * IV entries point to the same buffer 1178 * If src == dst, S/G entries are reused (S/G tables overlap) 1179 * 1180 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1181 * the end of the table by allocating more S/G entries. 1182 */ 1183 if (req->src != req->dst) 1184 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1185 else 1186 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1187 1188 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1189 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1190 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1191 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1192 qm_sg_ents, ivsize); 1193 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1194 0, DMA_NONE, 0, 0); 1195 return ERR_PTR(-ENOMEM); 1196 } 1197 1198 /* allocate space for base edesc, link tables and IV */ 1199 edesc = qi_cache_zalloc(GFP_DMA | flags); 1200 if (unlikely(!edesc)) { 1201 dev_err(dev, "could not allocate extended descriptor\n"); 1202 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1203 0, DMA_NONE, 0, 0); 1204 return ERR_PTR(-ENOMEM); 1205 } 1206 1207 /* Make sure IV is located in a DMAable area */ 1208 sg_table = &edesc->sgt[0]; 1209 iv = (u8 *)(sg_table + qm_sg_ents); 1210 memcpy(iv, req->iv, ivsize); 1211 1212 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1213 if (dma_mapping_error(dev, iv_dma)) { 1214 dev_err(dev, "unable to map IV\n"); 1215 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1216 0, DMA_NONE, 0, 0); 1217 qi_cache_free(edesc); 1218 return ERR_PTR(-ENOMEM); 1219 } 1220 1221 edesc->src_nents = src_nents; 1222 edesc->dst_nents = dst_nents; 1223 edesc->iv_dma = iv_dma; 1224 edesc->qm_sg_bytes = qm_sg_bytes; 1225 1226 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1227 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1228 1229 if (req->src != req->dst) 1230 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1231 1232 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1233 ivsize, 0); 1234 1235 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1236 DMA_TO_DEVICE); 1237 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1238 dev_err(dev, "unable to map S/G table\n"); 1239 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1240 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1241 qi_cache_free(edesc); 1242 return ERR_PTR(-ENOMEM); 1243 } 1244 1245 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1246 dpaa2_fl_set_final(in_fle, true); 1247 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1248 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1249 1250 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1251 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1252 1253 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1254 1255 if (req->src == req->dst) 1256 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1257 sizeof(*sg_table)); 1258 else 1259 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1260 sizeof(*sg_table)); 1261 1262 return edesc; 1263 } 1264 1265 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1266 struct aead_request *req) 1267 { 1268 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1269 int ivsize = crypto_aead_ivsize(aead); 1270 1271 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1272 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1273 edesc->qm_sg_bytes); 1274 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1275 } 1276 1277 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1278 struct skcipher_request *req) 1279 { 1280 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1281 int ivsize = crypto_skcipher_ivsize(skcipher); 1282 1283 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1284 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1285 edesc->qm_sg_bytes); 1286 } 1287 1288 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1289 { 1290 struct crypto_async_request *areq = cbk_ctx; 1291 struct aead_request *req = container_of(areq, struct aead_request, 1292 base); 1293 struct caam_request *req_ctx = to_caam_req(areq); 1294 struct aead_edesc *edesc = req_ctx->edesc; 1295 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1296 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1297 int ecode = 0; 1298 1299 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1300 1301 if (unlikely(status)) 1302 ecode = caam_qi2_strstatus(ctx->dev, status); 1303 1304 aead_unmap(ctx->dev, edesc, req); 1305 qi_cache_free(edesc); 1306 aead_request_complete(req, ecode); 1307 } 1308 1309 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1310 { 1311 struct crypto_async_request *areq = cbk_ctx; 1312 struct aead_request *req = container_of(areq, struct aead_request, 1313 base); 1314 struct caam_request *req_ctx = to_caam_req(areq); 1315 struct aead_edesc *edesc = req_ctx->edesc; 1316 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1317 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1318 int ecode = 0; 1319 1320 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1321 1322 if (unlikely(status)) 1323 ecode = caam_qi2_strstatus(ctx->dev, status); 1324 1325 aead_unmap(ctx->dev, edesc, req); 1326 qi_cache_free(edesc); 1327 aead_request_complete(req, ecode); 1328 } 1329 1330 static int aead_encrypt(struct aead_request *req) 1331 { 1332 struct aead_edesc *edesc; 1333 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1334 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1335 struct caam_request *caam_req = aead_request_ctx(req); 1336 int ret; 1337 1338 /* allocate extended descriptor */ 1339 edesc = aead_edesc_alloc(req, true); 1340 if (IS_ERR(edesc)) 1341 return PTR_ERR(edesc); 1342 1343 caam_req->flc = &ctx->flc[ENCRYPT]; 1344 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1345 caam_req->cbk = aead_encrypt_done; 1346 caam_req->ctx = &req->base; 1347 caam_req->edesc = edesc; 1348 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1349 if (ret != -EINPROGRESS && 1350 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1351 aead_unmap(ctx->dev, edesc, req); 1352 qi_cache_free(edesc); 1353 } 1354 1355 return ret; 1356 } 1357 1358 static int aead_decrypt(struct aead_request *req) 1359 { 1360 struct aead_edesc *edesc; 1361 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1362 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1363 struct caam_request *caam_req = aead_request_ctx(req); 1364 int ret; 1365 1366 /* allocate extended descriptor */ 1367 edesc = aead_edesc_alloc(req, false); 1368 if (IS_ERR(edesc)) 1369 return PTR_ERR(edesc); 1370 1371 caam_req->flc = &ctx->flc[DECRYPT]; 1372 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1373 caam_req->cbk = aead_decrypt_done; 1374 caam_req->ctx = &req->base; 1375 caam_req->edesc = edesc; 1376 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1377 if (ret != -EINPROGRESS && 1378 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1379 aead_unmap(ctx->dev, edesc, req); 1380 qi_cache_free(edesc); 1381 } 1382 1383 return ret; 1384 } 1385 1386 static int ipsec_gcm_encrypt(struct aead_request *req) 1387 { 1388 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1389 } 1390 1391 static int ipsec_gcm_decrypt(struct aead_request *req) 1392 { 1393 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1394 } 1395 1396 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1397 { 1398 struct crypto_async_request *areq = cbk_ctx; 1399 struct skcipher_request *req = skcipher_request_cast(areq); 1400 struct caam_request *req_ctx = to_caam_req(areq); 1401 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1402 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1403 struct skcipher_edesc *edesc = req_ctx->edesc; 1404 int ecode = 0; 1405 int ivsize = crypto_skcipher_ivsize(skcipher); 1406 1407 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1408 1409 if (unlikely(status)) 1410 ecode = caam_qi2_strstatus(ctx->dev, status); 1411 1412 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1413 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1414 edesc->src_nents > 1 ? 100 : ivsize, 1); 1415 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1416 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1417 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1418 1419 skcipher_unmap(ctx->dev, edesc, req); 1420 1421 /* 1422 * The crypto API expects us to set the IV (req->iv) to the last 1423 * ciphertext block (CBC mode) or last counter (CTR mode). 1424 * This is used e.g. by the CTS mode. 1425 */ 1426 if (!ecode) 1427 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1428 ivsize); 1429 1430 qi_cache_free(edesc); 1431 skcipher_request_complete(req, ecode); 1432 } 1433 1434 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1435 { 1436 struct crypto_async_request *areq = cbk_ctx; 1437 struct skcipher_request *req = skcipher_request_cast(areq); 1438 struct caam_request *req_ctx = to_caam_req(areq); 1439 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1440 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1441 struct skcipher_edesc *edesc = req_ctx->edesc; 1442 int ecode = 0; 1443 int ivsize = crypto_skcipher_ivsize(skcipher); 1444 1445 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1446 1447 if (unlikely(status)) 1448 ecode = caam_qi2_strstatus(ctx->dev, status); 1449 1450 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1451 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1452 edesc->src_nents > 1 ? 100 : ivsize, 1); 1453 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1454 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1455 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1456 1457 skcipher_unmap(ctx->dev, edesc, req); 1458 1459 /* 1460 * The crypto API expects us to set the IV (req->iv) to the last 1461 * ciphertext block (CBC mode) or last counter (CTR mode). 1462 * This is used e.g. by the CTS mode. 1463 */ 1464 if (!ecode) 1465 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1466 ivsize); 1467 1468 qi_cache_free(edesc); 1469 skcipher_request_complete(req, ecode); 1470 } 1471 1472 static int skcipher_encrypt(struct skcipher_request *req) 1473 { 1474 struct skcipher_edesc *edesc; 1475 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1476 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1477 struct caam_request *caam_req = skcipher_request_ctx(req); 1478 int ret; 1479 1480 if (!req->cryptlen) 1481 return 0; 1482 1483 /* allocate extended descriptor */ 1484 edesc = skcipher_edesc_alloc(req); 1485 if (IS_ERR(edesc)) 1486 return PTR_ERR(edesc); 1487 1488 caam_req->flc = &ctx->flc[ENCRYPT]; 1489 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1490 caam_req->cbk = skcipher_encrypt_done; 1491 caam_req->ctx = &req->base; 1492 caam_req->edesc = edesc; 1493 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1494 if (ret != -EINPROGRESS && 1495 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1496 skcipher_unmap(ctx->dev, edesc, req); 1497 qi_cache_free(edesc); 1498 } 1499 1500 return ret; 1501 } 1502 1503 static int skcipher_decrypt(struct skcipher_request *req) 1504 { 1505 struct skcipher_edesc *edesc; 1506 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1507 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1508 struct caam_request *caam_req = skcipher_request_ctx(req); 1509 int ret; 1510 1511 if (!req->cryptlen) 1512 return 0; 1513 /* allocate extended descriptor */ 1514 edesc = skcipher_edesc_alloc(req); 1515 if (IS_ERR(edesc)) 1516 return PTR_ERR(edesc); 1517 1518 caam_req->flc = &ctx->flc[DECRYPT]; 1519 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1520 caam_req->cbk = skcipher_decrypt_done; 1521 caam_req->ctx = &req->base; 1522 caam_req->edesc = edesc; 1523 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1524 if (ret != -EINPROGRESS && 1525 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1526 skcipher_unmap(ctx->dev, edesc, req); 1527 qi_cache_free(edesc); 1528 } 1529 1530 return ret; 1531 } 1532 1533 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1534 bool uses_dkp) 1535 { 1536 dma_addr_t dma_addr; 1537 int i; 1538 1539 /* copy descriptor header template value */ 1540 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1541 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1542 1543 ctx->dev = caam->dev; 1544 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1545 1546 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1547 offsetof(struct caam_ctx, flc_dma), 1548 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1549 if (dma_mapping_error(ctx->dev, dma_addr)) { 1550 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1551 return -ENOMEM; 1552 } 1553 1554 for (i = 0; i < NUM_OP; i++) 1555 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1556 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1557 1558 return 0; 1559 } 1560 1561 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1562 { 1563 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1564 struct caam_skcipher_alg *caam_alg = 1565 container_of(alg, typeof(*caam_alg), skcipher); 1566 1567 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); 1568 return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false); 1569 } 1570 1571 static int caam_cra_init_aead(struct crypto_aead *tfm) 1572 { 1573 struct aead_alg *alg = crypto_aead_alg(tfm); 1574 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1575 aead); 1576 1577 crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); 1578 return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam, 1579 !caam_alg->caam.nodkp); 1580 } 1581 1582 static void caam_exit_common(struct caam_ctx *ctx) 1583 { 1584 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1585 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1586 DMA_ATTR_SKIP_CPU_SYNC); 1587 } 1588 1589 static void caam_cra_exit(struct crypto_skcipher *tfm) 1590 { 1591 caam_exit_common(crypto_skcipher_ctx(tfm)); 1592 } 1593 1594 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1595 { 1596 caam_exit_common(crypto_aead_ctx(tfm)); 1597 } 1598 1599 static struct caam_skcipher_alg driver_algs[] = { 1600 { 1601 .skcipher = { 1602 .base = { 1603 .cra_name = "cbc(aes)", 1604 .cra_driver_name = "cbc-aes-caam-qi2", 1605 .cra_blocksize = AES_BLOCK_SIZE, 1606 }, 1607 .setkey = aes_skcipher_setkey, 1608 .encrypt = skcipher_encrypt, 1609 .decrypt = skcipher_decrypt, 1610 .min_keysize = AES_MIN_KEY_SIZE, 1611 .max_keysize = AES_MAX_KEY_SIZE, 1612 .ivsize = AES_BLOCK_SIZE, 1613 }, 1614 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1615 }, 1616 { 1617 .skcipher = { 1618 .base = { 1619 .cra_name = "cbc(des3_ede)", 1620 .cra_driver_name = "cbc-3des-caam-qi2", 1621 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1622 }, 1623 .setkey = des3_skcipher_setkey, 1624 .encrypt = skcipher_encrypt, 1625 .decrypt = skcipher_decrypt, 1626 .min_keysize = DES3_EDE_KEY_SIZE, 1627 .max_keysize = DES3_EDE_KEY_SIZE, 1628 .ivsize = DES3_EDE_BLOCK_SIZE, 1629 }, 1630 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1631 }, 1632 { 1633 .skcipher = { 1634 .base = { 1635 .cra_name = "cbc(des)", 1636 .cra_driver_name = "cbc-des-caam-qi2", 1637 .cra_blocksize = DES_BLOCK_SIZE, 1638 }, 1639 .setkey = des_skcipher_setkey, 1640 .encrypt = skcipher_encrypt, 1641 .decrypt = skcipher_decrypt, 1642 .min_keysize = DES_KEY_SIZE, 1643 .max_keysize = DES_KEY_SIZE, 1644 .ivsize = DES_BLOCK_SIZE, 1645 }, 1646 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1647 }, 1648 { 1649 .skcipher = { 1650 .base = { 1651 .cra_name = "ctr(aes)", 1652 .cra_driver_name = "ctr-aes-caam-qi2", 1653 .cra_blocksize = 1, 1654 }, 1655 .setkey = ctr_skcipher_setkey, 1656 .encrypt = skcipher_encrypt, 1657 .decrypt = skcipher_decrypt, 1658 .min_keysize = AES_MIN_KEY_SIZE, 1659 .max_keysize = AES_MAX_KEY_SIZE, 1660 .ivsize = AES_BLOCK_SIZE, 1661 .chunksize = AES_BLOCK_SIZE, 1662 }, 1663 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1664 OP_ALG_AAI_CTR_MOD128, 1665 }, 1666 { 1667 .skcipher = { 1668 .base = { 1669 .cra_name = "rfc3686(ctr(aes))", 1670 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1671 .cra_blocksize = 1, 1672 }, 1673 .setkey = rfc3686_skcipher_setkey, 1674 .encrypt = skcipher_encrypt, 1675 .decrypt = skcipher_decrypt, 1676 .min_keysize = AES_MIN_KEY_SIZE + 1677 CTR_RFC3686_NONCE_SIZE, 1678 .max_keysize = AES_MAX_KEY_SIZE + 1679 CTR_RFC3686_NONCE_SIZE, 1680 .ivsize = CTR_RFC3686_IV_SIZE, 1681 .chunksize = AES_BLOCK_SIZE, 1682 }, 1683 .caam = { 1684 .class1_alg_type = OP_ALG_ALGSEL_AES | 1685 OP_ALG_AAI_CTR_MOD128, 1686 .rfc3686 = true, 1687 }, 1688 }, 1689 { 1690 .skcipher = { 1691 .base = { 1692 .cra_name = "xts(aes)", 1693 .cra_driver_name = "xts-aes-caam-qi2", 1694 .cra_blocksize = AES_BLOCK_SIZE, 1695 }, 1696 .setkey = xts_skcipher_setkey, 1697 .encrypt = skcipher_encrypt, 1698 .decrypt = skcipher_decrypt, 1699 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1700 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1701 .ivsize = AES_BLOCK_SIZE, 1702 }, 1703 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1704 }, 1705 { 1706 .skcipher = { 1707 .base = { 1708 .cra_name = "chacha20", 1709 .cra_driver_name = "chacha20-caam-qi2", 1710 .cra_blocksize = 1, 1711 }, 1712 .setkey = chacha20_skcipher_setkey, 1713 .encrypt = skcipher_encrypt, 1714 .decrypt = skcipher_decrypt, 1715 .min_keysize = CHACHA_KEY_SIZE, 1716 .max_keysize = CHACHA_KEY_SIZE, 1717 .ivsize = CHACHA_IV_SIZE, 1718 }, 1719 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1720 }, 1721 }; 1722 1723 static struct caam_aead_alg driver_aeads[] = { 1724 { 1725 .aead = { 1726 .base = { 1727 .cra_name = "rfc4106(gcm(aes))", 1728 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1729 .cra_blocksize = 1, 1730 }, 1731 .setkey = rfc4106_setkey, 1732 .setauthsize = rfc4106_setauthsize, 1733 .encrypt = ipsec_gcm_encrypt, 1734 .decrypt = ipsec_gcm_decrypt, 1735 .ivsize = 8, 1736 .maxauthsize = AES_BLOCK_SIZE, 1737 }, 1738 .caam = { 1739 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1740 .nodkp = true, 1741 }, 1742 }, 1743 { 1744 .aead = { 1745 .base = { 1746 .cra_name = "rfc4543(gcm(aes))", 1747 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1748 .cra_blocksize = 1, 1749 }, 1750 .setkey = rfc4543_setkey, 1751 .setauthsize = rfc4543_setauthsize, 1752 .encrypt = ipsec_gcm_encrypt, 1753 .decrypt = ipsec_gcm_decrypt, 1754 .ivsize = 8, 1755 .maxauthsize = AES_BLOCK_SIZE, 1756 }, 1757 .caam = { 1758 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1759 .nodkp = true, 1760 }, 1761 }, 1762 /* Galois Counter Mode */ 1763 { 1764 .aead = { 1765 .base = { 1766 .cra_name = "gcm(aes)", 1767 .cra_driver_name = "gcm-aes-caam-qi2", 1768 .cra_blocksize = 1, 1769 }, 1770 .setkey = gcm_setkey, 1771 .setauthsize = gcm_setauthsize, 1772 .encrypt = aead_encrypt, 1773 .decrypt = aead_decrypt, 1774 .ivsize = 12, 1775 .maxauthsize = AES_BLOCK_SIZE, 1776 }, 1777 .caam = { 1778 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1779 .nodkp = true, 1780 } 1781 }, 1782 /* single-pass ipsec_esp descriptor */ 1783 { 1784 .aead = { 1785 .base = { 1786 .cra_name = "authenc(hmac(md5),cbc(aes))", 1787 .cra_driver_name = "authenc-hmac-md5-" 1788 "cbc-aes-caam-qi2", 1789 .cra_blocksize = AES_BLOCK_SIZE, 1790 }, 1791 .setkey = aead_setkey, 1792 .setauthsize = aead_setauthsize, 1793 .encrypt = aead_encrypt, 1794 .decrypt = aead_decrypt, 1795 .ivsize = AES_BLOCK_SIZE, 1796 .maxauthsize = MD5_DIGEST_SIZE, 1797 }, 1798 .caam = { 1799 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1800 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1801 OP_ALG_AAI_HMAC_PRECOMP, 1802 } 1803 }, 1804 { 1805 .aead = { 1806 .base = { 1807 .cra_name = "echainiv(authenc(hmac(md5)," 1808 "cbc(aes)))", 1809 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1810 "cbc-aes-caam-qi2", 1811 .cra_blocksize = AES_BLOCK_SIZE, 1812 }, 1813 .setkey = aead_setkey, 1814 .setauthsize = aead_setauthsize, 1815 .encrypt = aead_encrypt, 1816 .decrypt = aead_decrypt, 1817 .ivsize = AES_BLOCK_SIZE, 1818 .maxauthsize = MD5_DIGEST_SIZE, 1819 }, 1820 .caam = { 1821 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1822 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1823 OP_ALG_AAI_HMAC_PRECOMP, 1824 .geniv = true, 1825 } 1826 }, 1827 { 1828 .aead = { 1829 .base = { 1830 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1831 .cra_driver_name = "authenc-hmac-sha1-" 1832 "cbc-aes-caam-qi2", 1833 .cra_blocksize = AES_BLOCK_SIZE, 1834 }, 1835 .setkey = aead_setkey, 1836 .setauthsize = aead_setauthsize, 1837 .encrypt = aead_encrypt, 1838 .decrypt = aead_decrypt, 1839 .ivsize = AES_BLOCK_SIZE, 1840 .maxauthsize = SHA1_DIGEST_SIZE, 1841 }, 1842 .caam = { 1843 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1844 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1845 OP_ALG_AAI_HMAC_PRECOMP, 1846 } 1847 }, 1848 { 1849 .aead = { 1850 .base = { 1851 .cra_name = "echainiv(authenc(hmac(sha1)," 1852 "cbc(aes)))", 1853 .cra_driver_name = "echainiv-authenc-" 1854 "hmac-sha1-cbc-aes-caam-qi2", 1855 .cra_blocksize = AES_BLOCK_SIZE, 1856 }, 1857 .setkey = aead_setkey, 1858 .setauthsize = aead_setauthsize, 1859 .encrypt = aead_encrypt, 1860 .decrypt = aead_decrypt, 1861 .ivsize = AES_BLOCK_SIZE, 1862 .maxauthsize = SHA1_DIGEST_SIZE, 1863 }, 1864 .caam = { 1865 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1866 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1867 OP_ALG_AAI_HMAC_PRECOMP, 1868 .geniv = true, 1869 }, 1870 }, 1871 { 1872 .aead = { 1873 .base = { 1874 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1875 .cra_driver_name = "authenc-hmac-sha224-" 1876 "cbc-aes-caam-qi2", 1877 .cra_blocksize = AES_BLOCK_SIZE, 1878 }, 1879 .setkey = aead_setkey, 1880 .setauthsize = aead_setauthsize, 1881 .encrypt = aead_encrypt, 1882 .decrypt = aead_decrypt, 1883 .ivsize = AES_BLOCK_SIZE, 1884 .maxauthsize = SHA224_DIGEST_SIZE, 1885 }, 1886 .caam = { 1887 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1888 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1889 OP_ALG_AAI_HMAC_PRECOMP, 1890 } 1891 }, 1892 { 1893 .aead = { 1894 .base = { 1895 .cra_name = "echainiv(authenc(hmac(sha224)," 1896 "cbc(aes)))", 1897 .cra_driver_name = "echainiv-authenc-" 1898 "hmac-sha224-cbc-aes-caam-qi2", 1899 .cra_blocksize = AES_BLOCK_SIZE, 1900 }, 1901 .setkey = aead_setkey, 1902 .setauthsize = aead_setauthsize, 1903 .encrypt = aead_encrypt, 1904 .decrypt = aead_decrypt, 1905 .ivsize = AES_BLOCK_SIZE, 1906 .maxauthsize = SHA224_DIGEST_SIZE, 1907 }, 1908 .caam = { 1909 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1910 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1911 OP_ALG_AAI_HMAC_PRECOMP, 1912 .geniv = true, 1913 } 1914 }, 1915 { 1916 .aead = { 1917 .base = { 1918 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1919 .cra_driver_name = "authenc-hmac-sha256-" 1920 "cbc-aes-caam-qi2", 1921 .cra_blocksize = AES_BLOCK_SIZE, 1922 }, 1923 .setkey = aead_setkey, 1924 .setauthsize = aead_setauthsize, 1925 .encrypt = aead_encrypt, 1926 .decrypt = aead_decrypt, 1927 .ivsize = AES_BLOCK_SIZE, 1928 .maxauthsize = SHA256_DIGEST_SIZE, 1929 }, 1930 .caam = { 1931 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1932 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1933 OP_ALG_AAI_HMAC_PRECOMP, 1934 } 1935 }, 1936 { 1937 .aead = { 1938 .base = { 1939 .cra_name = "echainiv(authenc(hmac(sha256)," 1940 "cbc(aes)))", 1941 .cra_driver_name = "echainiv-authenc-" 1942 "hmac-sha256-cbc-aes-" 1943 "caam-qi2", 1944 .cra_blocksize = AES_BLOCK_SIZE, 1945 }, 1946 .setkey = aead_setkey, 1947 .setauthsize = aead_setauthsize, 1948 .encrypt = aead_encrypt, 1949 .decrypt = aead_decrypt, 1950 .ivsize = AES_BLOCK_SIZE, 1951 .maxauthsize = SHA256_DIGEST_SIZE, 1952 }, 1953 .caam = { 1954 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1955 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1956 OP_ALG_AAI_HMAC_PRECOMP, 1957 .geniv = true, 1958 } 1959 }, 1960 { 1961 .aead = { 1962 .base = { 1963 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1964 .cra_driver_name = "authenc-hmac-sha384-" 1965 "cbc-aes-caam-qi2", 1966 .cra_blocksize = AES_BLOCK_SIZE, 1967 }, 1968 .setkey = aead_setkey, 1969 .setauthsize = aead_setauthsize, 1970 .encrypt = aead_encrypt, 1971 .decrypt = aead_decrypt, 1972 .ivsize = AES_BLOCK_SIZE, 1973 .maxauthsize = SHA384_DIGEST_SIZE, 1974 }, 1975 .caam = { 1976 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1977 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1978 OP_ALG_AAI_HMAC_PRECOMP, 1979 } 1980 }, 1981 { 1982 .aead = { 1983 .base = { 1984 .cra_name = "echainiv(authenc(hmac(sha384)," 1985 "cbc(aes)))", 1986 .cra_driver_name = "echainiv-authenc-" 1987 "hmac-sha384-cbc-aes-" 1988 "caam-qi2", 1989 .cra_blocksize = AES_BLOCK_SIZE, 1990 }, 1991 .setkey = aead_setkey, 1992 .setauthsize = aead_setauthsize, 1993 .encrypt = aead_encrypt, 1994 .decrypt = aead_decrypt, 1995 .ivsize = AES_BLOCK_SIZE, 1996 .maxauthsize = SHA384_DIGEST_SIZE, 1997 }, 1998 .caam = { 1999 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2000 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2001 OP_ALG_AAI_HMAC_PRECOMP, 2002 .geniv = true, 2003 } 2004 }, 2005 { 2006 .aead = { 2007 .base = { 2008 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2009 .cra_driver_name = "authenc-hmac-sha512-" 2010 "cbc-aes-caam-qi2", 2011 .cra_blocksize = AES_BLOCK_SIZE, 2012 }, 2013 .setkey = aead_setkey, 2014 .setauthsize = aead_setauthsize, 2015 .encrypt = aead_encrypt, 2016 .decrypt = aead_decrypt, 2017 .ivsize = AES_BLOCK_SIZE, 2018 .maxauthsize = SHA512_DIGEST_SIZE, 2019 }, 2020 .caam = { 2021 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2022 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2023 OP_ALG_AAI_HMAC_PRECOMP, 2024 } 2025 }, 2026 { 2027 .aead = { 2028 .base = { 2029 .cra_name = "echainiv(authenc(hmac(sha512)," 2030 "cbc(aes)))", 2031 .cra_driver_name = "echainiv-authenc-" 2032 "hmac-sha512-cbc-aes-" 2033 "caam-qi2", 2034 .cra_blocksize = AES_BLOCK_SIZE, 2035 }, 2036 .setkey = aead_setkey, 2037 .setauthsize = aead_setauthsize, 2038 .encrypt = aead_encrypt, 2039 .decrypt = aead_decrypt, 2040 .ivsize = AES_BLOCK_SIZE, 2041 .maxauthsize = SHA512_DIGEST_SIZE, 2042 }, 2043 .caam = { 2044 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2045 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2046 OP_ALG_AAI_HMAC_PRECOMP, 2047 .geniv = true, 2048 } 2049 }, 2050 { 2051 .aead = { 2052 .base = { 2053 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2054 .cra_driver_name = "authenc-hmac-md5-" 2055 "cbc-des3_ede-caam-qi2", 2056 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2057 }, 2058 .setkey = des3_aead_setkey, 2059 .setauthsize = aead_setauthsize, 2060 .encrypt = aead_encrypt, 2061 .decrypt = aead_decrypt, 2062 .ivsize = DES3_EDE_BLOCK_SIZE, 2063 .maxauthsize = MD5_DIGEST_SIZE, 2064 }, 2065 .caam = { 2066 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2067 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2068 OP_ALG_AAI_HMAC_PRECOMP, 2069 } 2070 }, 2071 { 2072 .aead = { 2073 .base = { 2074 .cra_name = "echainiv(authenc(hmac(md5)," 2075 "cbc(des3_ede)))", 2076 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2077 "cbc-des3_ede-caam-qi2", 2078 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2079 }, 2080 .setkey = des3_aead_setkey, 2081 .setauthsize = aead_setauthsize, 2082 .encrypt = aead_encrypt, 2083 .decrypt = aead_decrypt, 2084 .ivsize = DES3_EDE_BLOCK_SIZE, 2085 .maxauthsize = MD5_DIGEST_SIZE, 2086 }, 2087 .caam = { 2088 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2089 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2090 OP_ALG_AAI_HMAC_PRECOMP, 2091 .geniv = true, 2092 } 2093 }, 2094 { 2095 .aead = { 2096 .base = { 2097 .cra_name = "authenc(hmac(sha1)," 2098 "cbc(des3_ede))", 2099 .cra_driver_name = "authenc-hmac-sha1-" 2100 "cbc-des3_ede-caam-qi2", 2101 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2102 }, 2103 .setkey = des3_aead_setkey, 2104 .setauthsize = aead_setauthsize, 2105 .encrypt = aead_encrypt, 2106 .decrypt = aead_decrypt, 2107 .ivsize = DES3_EDE_BLOCK_SIZE, 2108 .maxauthsize = SHA1_DIGEST_SIZE, 2109 }, 2110 .caam = { 2111 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2112 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2113 OP_ALG_AAI_HMAC_PRECOMP, 2114 }, 2115 }, 2116 { 2117 .aead = { 2118 .base = { 2119 .cra_name = "echainiv(authenc(hmac(sha1)," 2120 "cbc(des3_ede)))", 2121 .cra_driver_name = "echainiv-authenc-" 2122 "hmac-sha1-" 2123 "cbc-des3_ede-caam-qi2", 2124 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2125 }, 2126 .setkey = des3_aead_setkey, 2127 .setauthsize = aead_setauthsize, 2128 .encrypt = aead_encrypt, 2129 .decrypt = aead_decrypt, 2130 .ivsize = DES3_EDE_BLOCK_SIZE, 2131 .maxauthsize = SHA1_DIGEST_SIZE, 2132 }, 2133 .caam = { 2134 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2135 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2136 OP_ALG_AAI_HMAC_PRECOMP, 2137 .geniv = true, 2138 } 2139 }, 2140 { 2141 .aead = { 2142 .base = { 2143 .cra_name = "authenc(hmac(sha224)," 2144 "cbc(des3_ede))", 2145 .cra_driver_name = "authenc-hmac-sha224-" 2146 "cbc-des3_ede-caam-qi2", 2147 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2148 }, 2149 .setkey = des3_aead_setkey, 2150 .setauthsize = aead_setauthsize, 2151 .encrypt = aead_encrypt, 2152 .decrypt = aead_decrypt, 2153 .ivsize = DES3_EDE_BLOCK_SIZE, 2154 .maxauthsize = SHA224_DIGEST_SIZE, 2155 }, 2156 .caam = { 2157 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2158 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2159 OP_ALG_AAI_HMAC_PRECOMP, 2160 }, 2161 }, 2162 { 2163 .aead = { 2164 .base = { 2165 .cra_name = "echainiv(authenc(hmac(sha224)," 2166 "cbc(des3_ede)))", 2167 .cra_driver_name = "echainiv-authenc-" 2168 "hmac-sha224-" 2169 "cbc-des3_ede-caam-qi2", 2170 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2171 }, 2172 .setkey = des3_aead_setkey, 2173 .setauthsize = aead_setauthsize, 2174 .encrypt = aead_encrypt, 2175 .decrypt = aead_decrypt, 2176 .ivsize = DES3_EDE_BLOCK_SIZE, 2177 .maxauthsize = SHA224_DIGEST_SIZE, 2178 }, 2179 .caam = { 2180 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2181 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2182 OP_ALG_AAI_HMAC_PRECOMP, 2183 .geniv = true, 2184 } 2185 }, 2186 { 2187 .aead = { 2188 .base = { 2189 .cra_name = "authenc(hmac(sha256)," 2190 "cbc(des3_ede))", 2191 .cra_driver_name = "authenc-hmac-sha256-" 2192 "cbc-des3_ede-caam-qi2", 2193 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2194 }, 2195 .setkey = des3_aead_setkey, 2196 .setauthsize = aead_setauthsize, 2197 .encrypt = aead_encrypt, 2198 .decrypt = aead_decrypt, 2199 .ivsize = DES3_EDE_BLOCK_SIZE, 2200 .maxauthsize = SHA256_DIGEST_SIZE, 2201 }, 2202 .caam = { 2203 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2204 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2205 OP_ALG_AAI_HMAC_PRECOMP, 2206 }, 2207 }, 2208 { 2209 .aead = { 2210 .base = { 2211 .cra_name = "echainiv(authenc(hmac(sha256)," 2212 "cbc(des3_ede)))", 2213 .cra_driver_name = "echainiv-authenc-" 2214 "hmac-sha256-" 2215 "cbc-des3_ede-caam-qi2", 2216 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2217 }, 2218 .setkey = des3_aead_setkey, 2219 .setauthsize = aead_setauthsize, 2220 .encrypt = aead_encrypt, 2221 .decrypt = aead_decrypt, 2222 .ivsize = DES3_EDE_BLOCK_SIZE, 2223 .maxauthsize = SHA256_DIGEST_SIZE, 2224 }, 2225 .caam = { 2226 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2227 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2228 OP_ALG_AAI_HMAC_PRECOMP, 2229 .geniv = true, 2230 } 2231 }, 2232 { 2233 .aead = { 2234 .base = { 2235 .cra_name = "authenc(hmac(sha384)," 2236 "cbc(des3_ede))", 2237 .cra_driver_name = "authenc-hmac-sha384-" 2238 "cbc-des3_ede-caam-qi2", 2239 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2240 }, 2241 .setkey = des3_aead_setkey, 2242 .setauthsize = aead_setauthsize, 2243 .encrypt = aead_encrypt, 2244 .decrypt = aead_decrypt, 2245 .ivsize = DES3_EDE_BLOCK_SIZE, 2246 .maxauthsize = SHA384_DIGEST_SIZE, 2247 }, 2248 .caam = { 2249 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2250 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2251 OP_ALG_AAI_HMAC_PRECOMP, 2252 }, 2253 }, 2254 { 2255 .aead = { 2256 .base = { 2257 .cra_name = "echainiv(authenc(hmac(sha384)," 2258 "cbc(des3_ede)))", 2259 .cra_driver_name = "echainiv-authenc-" 2260 "hmac-sha384-" 2261 "cbc-des3_ede-caam-qi2", 2262 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2263 }, 2264 .setkey = des3_aead_setkey, 2265 .setauthsize = aead_setauthsize, 2266 .encrypt = aead_encrypt, 2267 .decrypt = aead_decrypt, 2268 .ivsize = DES3_EDE_BLOCK_SIZE, 2269 .maxauthsize = SHA384_DIGEST_SIZE, 2270 }, 2271 .caam = { 2272 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2273 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2274 OP_ALG_AAI_HMAC_PRECOMP, 2275 .geniv = true, 2276 } 2277 }, 2278 { 2279 .aead = { 2280 .base = { 2281 .cra_name = "authenc(hmac(sha512)," 2282 "cbc(des3_ede))", 2283 .cra_driver_name = "authenc-hmac-sha512-" 2284 "cbc-des3_ede-caam-qi2", 2285 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2286 }, 2287 .setkey = des3_aead_setkey, 2288 .setauthsize = aead_setauthsize, 2289 .encrypt = aead_encrypt, 2290 .decrypt = aead_decrypt, 2291 .ivsize = DES3_EDE_BLOCK_SIZE, 2292 .maxauthsize = SHA512_DIGEST_SIZE, 2293 }, 2294 .caam = { 2295 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2296 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2297 OP_ALG_AAI_HMAC_PRECOMP, 2298 }, 2299 }, 2300 { 2301 .aead = { 2302 .base = { 2303 .cra_name = "echainiv(authenc(hmac(sha512)," 2304 "cbc(des3_ede)))", 2305 .cra_driver_name = "echainiv-authenc-" 2306 "hmac-sha512-" 2307 "cbc-des3_ede-caam-qi2", 2308 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2309 }, 2310 .setkey = des3_aead_setkey, 2311 .setauthsize = aead_setauthsize, 2312 .encrypt = aead_encrypt, 2313 .decrypt = aead_decrypt, 2314 .ivsize = DES3_EDE_BLOCK_SIZE, 2315 .maxauthsize = SHA512_DIGEST_SIZE, 2316 }, 2317 .caam = { 2318 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2319 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2320 OP_ALG_AAI_HMAC_PRECOMP, 2321 .geniv = true, 2322 } 2323 }, 2324 { 2325 .aead = { 2326 .base = { 2327 .cra_name = "authenc(hmac(md5),cbc(des))", 2328 .cra_driver_name = "authenc-hmac-md5-" 2329 "cbc-des-caam-qi2", 2330 .cra_blocksize = DES_BLOCK_SIZE, 2331 }, 2332 .setkey = aead_setkey, 2333 .setauthsize = aead_setauthsize, 2334 .encrypt = aead_encrypt, 2335 .decrypt = aead_decrypt, 2336 .ivsize = DES_BLOCK_SIZE, 2337 .maxauthsize = MD5_DIGEST_SIZE, 2338 }, 2339 .caam = { 2340 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2341 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2342 OP_ALG_AAI_HMAC_PRECOMP, 2343 }, 2344 }, 2345 { 2346 .aead = { 2347 .base = { 2348 .cra_name = "echainiv(authenc(hmac(md5)," 2349 "cbc(des)))", 2350 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2351 "cbc-des-caam-qi2", 2352 .cra_blocksize = DES_BLOCK_SIZE, 2353 }, 2354 .setkey = aead_setkey, 2355 .setauthsize = aead_setauthsize, 2356 .encrypt = aead_encrypt, 2357 .decrypt = aead_decrypt, 2358 .ivsize = DES_BLOCK_SIZE, 2359 .maxauthsize = MD5_DIGEST_SIZE, 2360 }, 2361 .caam = { 2362 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2363 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2364 OP_ALG_AAI_HMAC_PRECOMP, 2365 .geniv = true, 2366 } 2367 }, 2368 { 2369 .aead = { 2370 .base = { 2371 .cra_name = "authenc(hmac(sha1),cbc(des))", 2372 .cra_driver_name = "authenc-hmac-sha1-" 2373 "cbc-des-caam-qi2", 2374 .cra_blocksize = DES_BLOCK_SIZE, 2375 }, 2376 .setkey = aead_setkey, 2377 .setauthsize = aead_setauthsize, 2378 .encrypt = aead_encrypt, 2379 .decrypt = aead_decrypt, 2380 .ivsize = DES_BLOCK_SIZE, 2381 .maxauthsize = SHA1_DIGEST_SIZE, 2382 }, 2383 .caam = { 2384 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2385 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2386 OP_ALG_AAI_HMAC_PRECOMP, 2387 }, 2388 }, 2389 { 2390 .aead = { 2391 .base = { 2392 .cra_name = "echainiv(authenc(hmac(sha1)," 2393 "cbc(des)))", 2394 .cra_driver_name = "echainiv-authenc-" 2395 "hmac-sha1-cbc-des-caam-qi2", 2396 .cra_blocksize = DES_BLOCK_SIZE, 2397 }, 2398 .setkey = aead_setkey, 2399 .setauthsize = aead_setauthsize, 2400 .encrypt = aead_encrypt, 2401 .decrypt = aead_decrypt, 2402 .ivsize = DES_BLOCK_SIZE, 2403 .maxauthsize = SHA1_DIGEST_SIZE, 2404 }, 2405 .caam = { 2406 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2407 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2408 OP_ALG_AAI_HMAC_PRECOMP, 2409 .geniv = true, 2410 } 2411 }, 2412 { 2413 .aead = { 2414 .base = { 2415 .cra_name = "authenc(hmac(sha224),cbc(des))", 2416 .cra_driver_name = "authenc-hmac-sha224-" 2417 "cbc-des-caam-qi2", 2418 .cra_blocksize = DES_BLOCK_SIZE, 2419 }, 2420 .setkey = aead_setkey, 2421 .setauthsize = aead_setauthsize, 2422 .encrypt = aead_encrypt, 2423 .decrypt = aead_decrypt, 2424 .ivsize = DES_BLOCK_SIZE, 2425 .maxauthsize = SHA224_DIGEST_SIZE, 2426 }, 2427 .caam = { 2428 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2429 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2430 OP_ALG_AAI_HMAC_PRECOMP, 2431 }, 2432 }, 2433 { 2434 .aead = { 2435 .base = { 2436 .cra_name = "echainiv(authenc(hmac(sha224)," 2437 "cbc(des)))", 2438 .cra_driver_name = "echainiv-authenc-" 2439 "hmac-sha224-cbc-des-" 2440 "caam-qi2", 2441 .cra_blocksize = DES_BLOCK_SIZE, 2442 }, 2443 .setkey = aead_setkey, 2444 .setauthsize = aead_setauthsize, 2445 .encrypt = aead_encrypt, 2446 .decrypt = aead_decrypt, 2447 .ivsize = DES_BLOCK_SIZE, 2448 .maxauthsize = SHA224_DIGEST_SIZE, 2449 }, 2450 .caam = { 2451 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2452 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2453 OP_ALG_AAI_HMAC_PRECOMP, 2454 .geniv = true, 2455 } 2456 }, 2457 { 2458 .aead = { 2459 .base = { 2460 .cra_name = "authenc(hmac(sha256),cbc(des))", 2461 .cra_driver_name = "authenc-hmac-sha256-" 2462 "cbc-des-caam-qi2", 2463 .cra_blocksize = DES_BLOCK_SIZE, 2464 }, 2465 .setkey = aead_setkey, 2466 .setauthsize = aead_setauthsize, 2467 .encrypt = aead_encrypt, 2468 .decrypt = aead_decrypt, 2469 .ivsize = DES_BLOCK_SIZE, 2470 .maxauthsize = SHA256_DIGEST_SIZE, 2471 }, 2472 .caam = { 2473 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2474 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2475 OP_ALG_AAI_HMAC_PRECOMP, 2476 }, 2477 }, 2478 { 2479 .aead = { 2480 .base = { 2481 .cra_name = "echainiv(authenc(hmac(sha256)," 2482 "cbc(des)))", 2483 .cra_driver_name = "echainiv-authenc-" 2484 "hmac-sha256-cbc-desi-" 2485 "caam-qi2", 2486 .cra_blocksize = DES_BLOCK_SIZE, 2487 }, 2488 .setkey = aead_setkey, 2489 .setauthsize = aead_setauthsize, 2490 .encrypt = aead_encrypt, 2491 .decrypt = aead_decrypt, 2492 .ivsize = DES_BLOCK_SIZE, 2493 .maxauthsize = SHA256_DIGEST_SIZE, 2494 }, 2495 .caam = { 2496 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2497 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2498 OP_ALG_AAI_HMAC_PRECOMP, 2499 .geniv = true, 2500 }, 2501 }, 2502 { 2503 .aead = { 2504 .base = { 2505 .cra_name = "authenc(hmac(sha384),cbc(des))", 2506 .cra_driver_name = "authenc-hmac-sha384-" 2507 "cbc-des-caam-qi2", 2508 .cra_blocksize = DES_BLOCK_SIZE, 2509 }, 2510 .setkey = aead_setkey, 2511 .setauthsize = aead_setauthsize, 2512 .encrypt = aead_encrypt, 2513 .decrypt = aead_decrypt, 2514 .ivsize = DES_BLOCK_SIZE, 2515 .maxauthsize = SHA384_DIGEST_SIZE, 2516 }, 2517 .caam = { 2518 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2519 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2520 OP_ALG_AAI_HMAC_PRECOMP, 2521 }, 2522 }, 2523 { 2524 .aead = { 2525 .base = { 2526 .cra_name = "echainiv(authenc(hmac(sha384)," 2527 "cbc(des)))", 2528 .cra_driver_name = "echainiv-authenc-" 2529 "hmac-sha384-cbc-des-" 2530 "caam-qi2", 2531 .cra_blocksize = DES_BLOCK_SIZE, 2532 }, 2533 .setkey = aead_setkey, 2534 .setauthsize = aead_setauthsize, 2535 .encrypt = aead_encrypt, 2536 .decrypt = aead_decrypt, 2537 .ivsize = DES_BLOCK_SIZE, 2538 .maxauthsize = SHA384_DIGEST_SIZE, 2539 }, 2540 .caam = { 2541 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2542 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2543 OP_ALG_AAI_HMAC_PRECOMP, 2544 .geniv = true, 2545 } 2546 }, 2547 { 2548 .aead = { 2549 .base = { 2550 .cra_name = "authenc(hmac(sha512),cbc(des))", 2551 .cra_driver_name = "authenc-hmac-sha512-" 2552 "cbc-des-caam-qi2", 2553 .cra_blocksize = DES_BLOCK_SIZE, 2554 }, 2555 .setkey = aead_setkey, 2556 .setauthsize = aead_setauthsize, 2557 .encrypt = aead_encrypt, 2558 .decrypt = aead_decrypt, 2559 .ivsize = DES_BLOCK_SIZE, 2560 .maxauthsize = SHA512_DIGEST_SIZE, 2561 }, 2562 .caam = { 2563 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2564 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2565 OP_ALG_AAI_HMAC_PRECOMP, 2566 } 2567 }, 2568 { 2569 .aead = { 2570 .base = { 2571 .cra_name = "echainiv(authenc(hmac(sha512)," 2572 "cbc(des)))", 2573 .cra_driver_name = "echainiv-authenc-" 2574 "hmac-sha512-cbc-des-" 2575 "caam-qi2", 2576 .cra_blocksize = DES_BLOCK_SIZE, 2577 }, 2578 .setkey = aead_setkey, 2579 .setauthsize = aead_setauthsize, 2580 .encrypt = aead_encrypt, 2581 .decrypt = aead_decrypt, 2582 .ivsize = DES_BLOCK_SIZE, 2583 .maxauthsize = SHA512_DIGEST_SIZE, 2584 }, 2585 .caam = { 2586 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2587 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2588 OP_ALG_AAI_HMAC_PRECOMP, 2589 .geniv = true, 2590 } 2591 }, 2592 { 2593 .aead = { 2594 .base = { 2595 .cra_name = "authenc(hmac(md5)," 2596 "rfc3686(ctr(aes)))", 2597 .cra_driver_name = "authenc-hmac-md5-" 2598 "rfc3686-ctr-aes-caam-qi2", 2599 .cra_blocksize = 1, 2600 }, 2601 .setkey = aead_setkey, 2602 .setauthsize = aead_setauthsize, 2603 .encrypt = aead_encrypt, 2604 .decrypt = aead_decrypt, 2605 .ivsize = CTR_RFC3686_IV_SIZE, 2606 .maxauthsize = MD5_DIGEST_SIZE, 2607 }, 2608 .caam = { 2609 .class1_alg_type = OP_ALG_ALGSEL_AES | 2610 OP_ALG_AAI_CTR_MOD128, 2611 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2612 OP_ALG_AAI_HMAC_PRECOMP, 2613 .rfc3686 = true, 2614 }, 2615 }, 2616 { 2617 .aead = { 2618 .base = { 2619 .cra_name = "seqiv(authenc(" 2620 "hmac(md5),rfc3686(ctr(aes))))", 2621 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2622 "rfc3686-ctr-aes-caam-qi2", 2623 .cra_blocksize = 1, 2624 }, 2625 .setkey = aead_setkey, 2626 .setauthsize = aead_setauthsize, 2627 .encrypt = aead_encrypt, 2628 .decrypt = aead_decrypt, 2629 .ivsize = CTR_RFC3686_IV_SIZE, 2630 .maxauthsize = MD5_DIGEST_SIZE, 2631 }, 2632 .caam = { 2633 .class1_alg_type = OP_ALG_ALGSEL_AES | 2634 OP_ALG_AAI_CTR_MOD128, 2635 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2636 OP_ALG_AAI_HMAC_PRECOMP, 2637 .rfc3686 = true, 2638 .geniv = true, 2639 }, 2640 }, 2641 { 2642 .aead = { 2643 .base = { 2644 .cra_name = "authenc(hmac(sha1)," 2645 "rfc3686(ctr(aes)))", 2646 .cra_driver_name = "authenc-hmac-sha1-" 2647 "rfc3686-ctr-aes-caam-qi2", 2648 .cra_blocksize = 1, 2649 }, 2650 .setkey = aead_setkey, 2651 .setauthsize = aead_setauthsize, 2652 .encrypt = aead_encrypt, 2653 .decrypt = aead_decrypt, 2654 .ivsize = CTR_RFC3686_IV_SIZE, 2655 .maxauthsize = SHA1_DIGEST_SIZE, 2656 }, 2657 .caam = { 2658 .class1_alg_type = OP_ALG_ALGSEL_AES | 2659 OP_ALG_AAI_CTR_MOD128, 2660 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2661 OP_ALG_AAI_HMAC_PRECOMP, 2662 .rfc3686 = true, 2663 }, 2664 }, 2665 { 2666 .aead = { 2667 .base = { 2668 .cra_name = "seqiv(authenc(" 2669 "hmac(sha1),rfc3686(ctr(aes))))", 2670 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2671 "rfc3686-ctr-aes-caam-qi2", 2672 .cra_blocksize = 1, 2673 }, 2674 .setkey = aead_setkey, 2675 .setauthsize = aead_setauthsize, 2676 .encrypt = aead_encrypt, 2677 .decrypt = aead_decrypt, 2678 .ivsize = CTR_RFC3686_IV_SIZE, 2679 .maxauthsize = SHA1_DIGEST_SIZE, 2680 }, 2681 .caam = { 2682 .class1_alg_type = OP_ALG_ALGSEL_AES | 2683 OP_ALG_AAI_CTR_MOD128, 2684 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2685 OP_ALG_AAI_HMAC_PRECOMP, 2686 .rfc3686 = true, 2687 .geniv = true, 2688 }, 2689 }, 2690 { 2691 .aead = { 2692 .base = { 2693 .cra_name = "authenc(hmac(sha224)," 2694 "rfc3686(ctr(aes)))", 2695 .cra_driver_name = "authenc-hmac-sha224-" 2696 "rfc3686-ctr-aes-caam-qi2", 2697 .cra_blocksize = 1, 2698 }, 2699 .setkey = aead_setkey, 2700 .setauthsize = aead_setauthsize, 2701 .encrypt = aead_encrypt, 2702 .decrypt = aead_decrypt, 2703 .ivsize = CTR_RFC3686_IV_SIZE, 2704 .maxauthsize = SHA224_DIGEST_SIZE, 2705 }, 2706 .caam = { 2707 .class1_alg_type = OP_ALG_ALGSEL_AES | 2708 OP_ALG_AAI_CTR_MOD128, 2709 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2710 OP_ALG_AAI_HMAC_PRECOMP, 2711 .rfc3686 = true, 2712 }, 2713 }, 2714 { 2715 .aead = { 2716 .base = { 2717 .cra_name = "seqiv(authenc(" 2718 "hmac(sha224),rfc3686(ctr(aes))))", 2719 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2720 "rfc3686-ctr-aes-caam-qi2", 2721 .cra_blocksize = 1, 2722 }, 2723 .setkey = aead_setkey, 2724 .setauthsize = aead_setauthsize, 2725 .encrypt = aead_encrypt, 2726 .decrypt = aead_decrypt, 2727 .ivsize = CTR_RFC3686_IV_SIZE, 2728 .maxauthsize = SHA224_DIGEST_SIZE, 2729 }, 2730 .caam = { 2731 .class1_alg_type = OP_ALG_ALGSEL_AES | 2732 OP_ALG_AAI_CTR_MOD128, 2733 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2734 OP_ALG_AAI_HMAC_PRECOMP, 2735 .rfc3686 = true, 2736 .geniv = true, 2737 }, 2738 }, 2739 { 2740 .aead = { 2741 .base = { 2742 .cra_name = "authenc(hmac(sha256)," 2743 "rfc3686(ctr(aes)))", 2744 .cra_driver_name = "authenc-hmac-sha256-" 2745 "rfc3686-ctr-aes-caam-qi2", 2746 .cra_blocksize = 1, 2747 }, 2748 .setkey = aead_setkey, 2749 .setauthsize = aead_setauthsize, 2750 .encrypt = aead_encrypt, 2751 .decrypt = aead_decrypt, 2752 .ivsize = CTR_RFC3686_IV_SIZE, 2753 .maxauthsize = SHA256_DIGEST_SIZE, 2754 }, 2755 .caam = { 2756 .class1_alg_type = OP_ALG_ALGSEL_AES | 2757 OP_ALG_AAI_CTR_MOD128, 2758 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2759 OP_ALG_AAI_HMAC_PRECOMP, 2760 .rfc3686 = true, 2761 }, 2762 }, 2763 { 2764 .aead = { 2765 .base = { 2766 .cra_name = "seqiv(authenc(hmac(sha256)," 2767 "rfc3686(ctr(aes))))", 2768 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2769 "rfc3686-ctr-aes-caam-qi2", 2770 .cra_blocksize = 1, 2771 }, 2772 .setkey = aead_setkey, 2773 .setauthsize = aead_setauthsize, 2774 .encrypt = aead_encrypt, 2775 .decrypt = aead_decrypt, 2776 .ivsize = CTR_RFC3686_IV_SIZE, 2777 .maxauthsize = SHA256_DIGEST_SIZE, 2778 }, 2779 .caam = { 2780 .class1_alg_type = OP_ALG_ALGSEL_AES | 2781 OP_ALG_AAI_CTR_MOD128, 2782 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2783 OP_ALG_AAI_HMAC_PRECOMP, 2784 .rfc3686 = true, 2785 .geniv = true, 2786 }, 2787 }, 2788 { 2789 .aead = { 2790 .base = { 2791 .cra_name = "authenc(hmac(sha384)," 2792 "rfc3686(ctr(aes)))", 2793 .cra_driver_name = "authenc-hmac-sha384-" 2794 "rfc3686-ctr-aes-caam-qi2", 2795 .cra_blocksize = 1, 2796 }, 2797 .setkey = aead_setkey, 2798 .setauthsize = aead_setauthsize, 2799 .encrypt = aead_encrypt, 2800 .decrypt = aead_decrypt, 2801 .ivsize = CTR_RFC3686_IV_SIZE, 2802 .maxauthsize = SHA384_DIGEST_SIZE, 2803 }, 2804 .caam = { 2805 .class1_alg_type = OP_ALG_ALGSEL_AES | 2806 OP_ALG_AAI_CTR_MOD128, 2807 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2808 OP_ALG_AAI_HMAC_PRECOMP, 2809 .rfc3686 = true, 2810 }, 2811 }, 2812 { 2813 .aead = { 2814 .base = { 2815 .cra_name = "seqiv(authenc(hmac(sha384)," 2816 "rfc3686(ctr(aes))))", 2817 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2818 "rfc3686-ctr-aes-caam-qi2", 2819 .cra_blocksize = 1, 2820 }, 2821 .setkey = aead_setkey, 2822 .setauthsize = aead_setauthsize, 2823 .encrypt = aead_encrypt, 2824 .decrypt = aead_decrypt, 2825 .ivsize = CTR_RFC3686_IV_SIZE, 2826 .maxauthsize = SHA384_DIGEST_SIZE, 2827 }, 2828 .caam = { 2829 .class1_alg_type = OP_ALG_ALGSEL_AES | 2830 OP_ALG_AAI_CTR_MOD128, 2831 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2832 OP_ALG_AAI_HMAC_PRECOMP, 2833 .rfc3686 = true, 2834 .geniv = true, 2835 }, 2836 }, 2837 { 2838 .aead = { 2839 .base = { 2840 .cra_name = "rfc7539(chacha20,poly1305)", 2841 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2842 "caam-qi2", 2843 .cra_blocksize = 1, 2844 }, 2845 .setkey = chachapoly_setkey, 2846 .setauthsize = chachapoly_setauthsize, 2847 .encrypt = aead_encrypt, 2848 .decrypt = aead_decrypt, 2849 .ivsize = CHACHAPOLY_IV_SIZE, 2850 .maxauthsize = POLY1305_DIGEST_SIZE, 2851 }, 2852 .caam = { 2853 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2854 OP_ALG_AAI_AEAD, 2855 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2856 OP_ALG_AAI_AEAD, 2857 .nodkp = true, 2858 }, 2859 }, 2860 { 2861 .aead = { 2862 .base = { 2863 .cra_name = "rfc7539esp(chacha20,poly1305)", 2864 .cra_driver_name = "rfc7539esp-chacha20-" 2865 "poly1305-caam-qi2", 2866 .cra_blocksize = 1, 2867 }, 2868 .setkey = chachapoly_setkey, 2869 .setauthsize = chachapoly_setauthsize, 2870 .encrypt = aead_encrypt, 2871 .decrypt = aead_decrypt, 2872 .ivsize = 8, 2873 .maxauthsize = POLY1305_DIGEST_SIZE, 2874 }, 2875 .caam = { 2876 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2877 OP_ALG_AAI_AEAD, 2878 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2879 OP_ALG_AAI_AEAD, 2880 .nodkp = true, 2881 }, 2882 }, 2883 { 2884 .aead = { 2885 .base = { 2886 .cra_name = "authenc(hmac(sha512)," 2887 "rfc3686(ctr(aes)))", 2888 .cra_driver_name = "authenc-hmac-sha512-" 2889 "rfc3686-ctr-aes-caam-qi2", 2890 .cra_blocksize = 1, 2891 }, 2892 .setkey = aead_setkey, 2893 .setauthsize = aead_setauthsize, 2894 .encrypt = aead_encrypt, 2895 .decrypt = aead_decrypt, 2896 .ivsize = CTR_RFC3686_IV_SIZE, 2897 .maxauthsize = SHA512_DIGEST_SIZE, 2898 }, 2899 .caam = { 2900 .class1_alg_type = OP_ALG_ALGSEL_AES | 2901 OP_ALG_AAI_CTR_MOD128, 2902 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2903 OP_ALG_AAI_HMAC_PRECOMP, 2904 .rfc3686 = true, 2905 }, 2906 }, 2907 { 2908 .aead = { 2909 .base = { 2910 .cra_name = "seqiv(authenc(hmac(sha512)," 2911 "rfc3686(ctr(aes))))", 2912 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2913 "rfc3686-ctr-aes-caam-qi2", 2914 .cra_blocksize = 1, 2915 }, 2916 .setkey = aead_setkey, 2917 .setauthsize = aead_setauthsize, 2918 .encrypt = aead_encrypt, 2919 .decrypt = aead_decrypt, 2920 .ivsize = CTR_RFC3686_IV_SIZE, 2921 .maxauthsize = SHA512_DIGEST_SIZE, 2922 }, 2923 .caam = { 2924 .class1_alg_type = OP_ALG_ALGSEL_AES | 2925 OP_ALG_AAI_CTR_MOD128, 2926 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2927 OP_ALG_AAI_HMAC_PRECOMP, 2928 .rfc3686 = true, 2929 .geniv = true, 2930 }, 2931 }, 2932 }; 2933 2934 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2935 { 2936 struct skcipher_alg *alg = &t_alg->skcipher; 2937 2938 alg->base.cra_module = THIS_MODULE; 2939 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2940 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2941 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2942 2943 alg->init = caam_cra_init_skcipher; 2944 alg->exit = caam_cra_exit; 2945 } 2946 2947 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2948 { 2949 struct aead_alg *alg = &t_alg->aead; 2950 2951 alg->base.cra_module = THIS_MODULE; 2952 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2953 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2954 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2955 2956 alg->init = caam_cra_init_aead; 2957 alg->exit = caam_cra_exit_aead; 2958 } 2959 2960 /* max hash key is max split key size */ 2961 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 2962 2963 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 2964 2965 /* caam context sizes for hashes: running digest + 8 */ 2966 #define HASH_MSG_LEN 8 2967 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 2968 2969 enum hash_optype { 2970 UPDATE = 0, 2971 UPDATE_FIRST, 2972 FINALIZE, 2973 DIGEST, 2974 HASH_NUM_OP 2975 }; 2976 2977 /** 2978 * caam_hash_ctx - ahash per-session context 2979 * @flc: Flow Contexts array 2980 * @key: authentication key 2981 * @flc_dma: I/O virtual addresses of the Flow Contexts 2982 * @dev: dpseci device 2983 * @ctx_len: size of Context Register 2984 * @adata: hashing algorithm details 2985 */ 2986 struct caam_hash_ctx { 2987 struct caam_flc flc[HASH_NUM_OP]; 2988 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 2989 dma_addr_t flc_dma[HASH_NUM_OP]; 2990 struct device *dev; 2991 int ctx_len; 2992 struct alginfo adata; 2993 }; 2994 2995 /* ahash state */ 2996 struct caam_hash_state { 2997 struct caam_request caam_req; 2998 dma_addr_t buf_dma; 2999 dma_addr_t ctx_dma; 3000 int ctx_dma_len; 3001 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3002 int buflen; 3003 int next_buflen; 3004 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 3005 int (*update)(struct ahash_request *req); 3006 int (*final)(struct ahash_request *req); 3007 int (*finup)(struct ahash_request *req); 3008 }; 3009 3010 struct caam_export_state { 3011 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 3012 u8 caam_ctx[MAX_CTX_LEN]; 3013 int buflen; 3014 int (*update)(struct ahash_request *req); 3015 int (*final)(struct ahash_request *req); 3016 int (*finup)(struct ahash_request *req); 3017 }; 3018 3019 /* Map current buffer in state (if length > 0) and put it in link table */ 3020 static inline int buf_map_to_qm_sg(struct device *dev, 3021 struct dpaa2_sg_entry *qm_sg, 3022 struct caam_hash_state *state) 3023 { 3024 int buflen = state->buflen; 3025 3026 if (!buflen) 3027 return 0; 3028 3029 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3030 DMA_TO_DEVICE); 3031 if (dma_mapping_error(dev, state->buf_dma)) { 3032 dev_err(dev, "unable to map buf\n"); 3033 state->buf_dma = 0; 3034 return -ENOMEM; 3035 } 3036 3037 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3038 3039 return 0; 3040 } 3041 3042 /* Map state->caam_ctx, and add it to link table */ 3043 static inline int ctx_map_to_qm_sg(struct device *dev, 3044 struct caam_hash_state *state, int ctx_len, 3045 struct dpaa2_sg_entry *qm_sg, u32 flag) 3046 { 3047 state->ctx_dma_len = ctx_len; 3048 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3049 if (dma_mapping_error(dev, state->ctx_dma)) { 3050 dev_err(dev, "unable to map ctx\n"); 3051 state->ctx_dma = 0; 3052 return -ENOMEM; 3053 } 3054 3055 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3056 3057 return 0; 3058 } 3059 3060 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3061 { 3062 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3063 int digestsize = crypto_ahash_digestsize(ahash); 3064 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3065 struct caam_flc *flc; 3066 u32 *desc; 3067 3068 /* ahash_update shared descriptor */ 3069 flc = &ctx->flc[UPDATE]; 3070 desc = flc->sh_desc; 3071 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3072 ctx->ctx_len, true, priv->sec_attr.era); 3073 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3074 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3075 desc_bytes(desc), DMA_BIDIRECTIONAL); 3076 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3077 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3078 1); 3079 3080 /* ahash_update_first shared descriptor */ 3081 flc = &ctx->flc[UPDATE_FIRST]; 3082 desc = flc->sh_desc; 3083 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3084 ctx->ctx_len, false, priv->sec_attr.era); 3085 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3086 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3087 desc_bytes(desc), DMA_BIDIRECTIONAL); 3088 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3089 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3090 1); 3091 3092 /* ahash_final shared descriptor */ 3093 flc = &ctx->flc[FINALIZE]; 3094 desc = flc->sh_desc; 3095 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3096 ctx->ctx_len, true, priv->sec_attr.era); 3097 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3098 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3099 desc_bytes(desc), DMA_BIDIRECTIONAL); 3100 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3101 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3102 1); 3103 3104 /* ahash_digest shared descriptor */ 3105 flc = &ctx->flc[DIGEST]; 3106 desc = flc->sh_desc; 3107 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3108 ctx->ctx_len, false, priv->sec_attr.era); 3109 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3110 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3111 desc_bytes(desc), DMA_BIDIRECTIONAL); 3112 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3113 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3114 1); 3115 3116 return 0; 3117 } 3118 3119 struct split_key_sh_result { 3120 struct completion completion; 3121 int err; 3122 struct device *dev; 3123 }; 3124 3125 static void split_key_sh_done(void *cbk_ctx, u32 err) 3126 { 3127 struct split_key_sh_result *res = cbk_ctx; 3128 3129 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3130 3131 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3132 complete(&res->completion); 3133 } 3134 3135 /* Digest hash size if it is too large */ 3136 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3137 u32 digestsize) 3138 { 3139 struct caam_request *req_ctx; 3140 u32 *desc; 3141 struct split_key_sh_result result; 3142 dma_addr_t key_dma; 3143 struct caam_flc *flc; 3144 dma_addr_t flc_dma; 3145 int ret = -ENOMEM; 3146 struct dpaa2_fl_entry *in_fle, *out_fle; 3147 3148 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); 3149 if (!req_ctx) 3150 return -ENOMEM; 3151 3152 in_fle = &req_ctx->fd_flt[1]; 3153 out_fle = &req_ctx->fd_flt[0]; 3154 3155 flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); 3156 if (!flc) 3157 goto err_flc; 3158 3159 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3160 if (dma_mapping_error(ctx->dev, key_dma)) { 3161 dev_err(ctx->dev, "unable to map key memory\n"); 3162 goto err_key_dma; 3163 } 3164 3165 desc = flc->sh_desc; 3166 3167 init_sh_desc(desc, 0); 3168 3169 /* descriptor to perform unkeyed hash on key_in */ 3170 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3171 OP_ALG_AS_INITFINAL); 3172 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3173 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3174 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3175 LDST_SRCDST_BYTE_CONTEXT); 3176 3177 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3178 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3179 desc_bytes(desc), DMA_TO_DEVICE); 3180 if (dma_mapping_error(ctx->dev, flc_dma)) { 3181 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3182 goto err_flc_dma; 3183 } 3184 3185 dpaa2_fl_set_final(in_fle, true); 3186 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3187 dpaa2_fl_set_addr(in_fle, key_dma); 3188 dpaa2_fl_set_len(in_fle, *keylen); 3189 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3190 dpaa2_fl_set_addr(out_fle, key_dma); 3191 dpaa2_fl_set_len(out_fle, digestsize); 3192 3193 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3194 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3195 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3196 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3197 1); 3198 3199 result.err = 0; 3200 init_completion(&result.completion); 3201 result.dev = ctx->dev; 3202 3203 req_ctx->flc = flc; 3204 req_ctx->flc_dma = flc_dma; 3205 req_ctx->cbk = split_key_sh_done; 3206 req_ctx->ctx = &result; 3207 3208 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3209 if (ret == -EINPROGRESS) { 3210 /* in progress */ 3211 wait_for_completion(&result.completion); 3212 ret = result.err; 3213 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3214 DUMP_PREFIX_ADDRESS, 16, 4, key, 3215 digestsize, 1); 3216 } 3217 3218 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3219 DMA_TO_DEVICE); 3220 err_flc_dma: 3221 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3222 err_key_dma: 3223 kfree(flc); 3224 err_flc: 3225 kfree(req_ctx); 3226 3227 *keylen = digestsize; 3228 3229 return ret; 3230 } 3231 3232 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3233 unsigned int keylen) 3234 { 3235 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3236 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3237 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3238 int ret; 3239 u8 *hashed_key = NULL; 3240 3241 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3242 3243 if (keylen > blocksize) { 3244 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 3245 if (!hashed_key) 3246 return -ENOMEM; 3247 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3248 if (ret) 3249 goto bad_free_key; 3250 key = hashed_key; 3251 } 3252 3253 ctx->adata.keylen = keylen; 3254 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3255 OP_ALG_ALGSEL_MASK); 3256 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3257 goto bad_free_key; 3258 3259 ctx->adata.key_virt = key; 3260 ctx->adata.key_inline = true; 3261 3262 /* 3263 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3264 * in invalid opcodes (last bytes of user key) in the resulting 3265 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3266 * addresses are needed. 3267 */ 3268 if (keylen > ctx->adata.keylen_pad) { 3269 memcpy(ctx->key, key, keylen); 3270 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3271 ctx->adata.keylen_pad, 3272 DMA_TO_DEVICE); 3273 } 3274 3275 ret = ahash_set_sh_desc(ahash); 3276 kfree(hashed_key); 3277 return ret; 3278 bad_free_key: 3279 kfree(hashed_key); 3280 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 3281 return -EINVAL; 3282 } 3283 3284 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3285 struct ahash_request *req) 3286 { 3287 struct caam_hash_state *state = ahash_request_ctx(req); 3288 3289 if (edesc->src_nents) 3290 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3291 3292 if (edesc->qm_sg_bytes) 3293 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3294 DMA_TO_DEVICE); 3295 3296 if (state->buf_dma) { 3297 dma_unmap_single(dev, state->buf_dma, state->buflen, 3298 DMA_TO_DEVICE); 3299 state->buf_dma = 0; 3300 } 3301 } 3302 3303 static inline void ahash_unmap_ctx(struct device *dev, 3304 struct ahash_edesc *edesc, 3305 struct ahash_request *req, u32 flag) 3306 { 3307 struct caam_hash_state *state = ahash_request_ctx(req); 3308 3309 if (state->ctx_dma) { 3310 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3311 state->ctx_dma = 0; 3312 } 3313 ahash_unmap(dev, edesc, req); 3314 } 3315 3316 static void ahash_done(void *cbk_ctx, u32 status) 3317 { 3318 struct crypto_async_request *areq = cbk_ctx; 3319 struct ahash_request *req = ahash_request_cast(areq); 3320 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3321 struct caam_hash_state *state = ahash_request_ctx(req); 3322 struct ahash_edesc *edesc = state->caam_req.edesc; 3323 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3324 int digestsize = crypto_ahash_digestsize(ahash); 3325 int ecode = 0; 3326 3327 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3328 3329 if (unlikely(status)) 3330 ecode = caam_qi2_strstatus(ctx->dev, status); 3331 3332 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3333 memcpy(req->result, state->caam_ctx, digestsize); 3334 qi_cache_free(edesc); 3335 3336 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3337 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3338 ctx->ctx_len, 1); 3339 3340 req->base.complete(&req->base, ecode); 3341 } 3342 3343 static void ahash_done_bi(void *cbk_ctx, u32 status) 3344 { 3345 struct crypto_async_request *areq = cbk_ctx; 3346 struct ahash_request *req = ahash_request_cast(areq); 3347 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3348 struct caam_hash_state *state = ahash_request_ctx(req); 3349 struct ahash_edesc *edesc = state->caam_req.edesc; 3350 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3351 int ecode = 0; 3352 3353 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3354 3355 if (unlikely(status)) 3356 ecode = caam_qi2_strstatus(ctx->dev, status); 3357 3358 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3359 qi_cache_free(edesc); 3360 3361 scatterwalk_map_and_copy(state->buf, req->src, 3362 req->nbytes - state->next_buflen, 3363 state->next_buflen, 0); 3364 state->buflen = state->next_buflen; 3365 3366 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3367 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3368 state->buflen, 1); 3369 3370 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3371 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3372 ctx->ctx_len, 1); 3373 if (req->result) 3374 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3375 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3376 crypto_ahash_digestsize(ahash), 1); 3377 3378 req->base.complete(&req->base, ecode); 3379 } 3380 3381 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3382 { 3383 struct crypto_async_request *areq = cbk_ctx; 3384 struct ahash_request *req = ahash_request_cast(areq); 3385 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3386 struct caam_hash_state *state = ahash_request_ctx(req); 3387 struct ahash_edesc *edesc = state->caam_req.edesc; 3388 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3389 int digestsize = crypto_ahash_digestsize(ahash); 3390 int ecode = 0; 3391 3392 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3393 3394 if (unlikely(status)) 3395 ecode = caam_qi2_strstatus(ctx->dev, status); 3396 3397 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3398 memcpy(req->result, state->caam_ctx, digestsize); 3399 qi_cache_free(edesc); 3400 3401 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3402 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3403 ctx->ctx_len, 1); 3404 3405 req->base.complete(&req->base, ecode); 3406 } 3407 3408 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3409 { 3410 struct crypto_async_request *areq = cbk_ctx; 3411 struct ahash_request *req = ahash_request_cast(areq); 3412 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3413 struct caam_hash_state *state = ahash_request_ctx(req); 3414 struct ahash_edesc *edesc = state->caam_req.edesc; 3415 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3416 int ecode = 0; 3417 3418 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3419 3420 if (unlikely(status)) 3421 ecode = caam_qi2_strstatus(ctx->dev, status); 3422 3423 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3424 qi_cache_free(edesc); 3425 3426 scatterwalk_map_and_copy(state->buf, req->src, 3427 req->nbytes - state->next_buflen, 3428 state->next_buflen, 0); 3429 state->buflen = state->next_buflen; 3430 3431 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3432 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3433 state->buflen, 1); 3434 3435 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3436 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3437 ctx->ctx_len, 1); 3438 if (req->result) 3439 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3440 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3441 crypto_ahash_digestsize(ahash), 1); 3442 3443 req->base.complete(&req->base, ecode); 3444 } 3445 3446 static int ahash_update_ctx(struct ahash_request *req) 3447 { 3448 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3449 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3450 struct caam_hash_state *state = ahash_request_ctx(req); 3451 struct caam_request *req_ctx = &state->caam_req; 3452 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3453 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3454 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3455 GFP_KERNEL : GFP_ATOMIC; 3456 u8 *buf = state->buf; 3457 int *buflen = &state->buflen; 3458 int *next_buflen = &state->next_buflen; 3459 int in_len = *buflen + req->nbytes, to_hash; 3460 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3461 struct ahash_edesc *edesc; 3462 int ret = 0; 3463 3464 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3465 to_hash = in_len - *next_buflen; 3466 3467 if (to_hash) { 3468 struct dpaa2_sg_entry *sg_table; 3469 int src_len = req->nbytes - *next_buflen; 3470 3471 src_nents = sg_nents_for_len(req->src, src_len); 3472 if (src_nents < 0) { 3473 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3474 return src_nents; 3475 } 3476 3477 if (src_nents) { 3478 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3479 DMA_TO_DEVICE); 3480 if (!mapped_nents) { 3481 dev_err(ctx->dev, "unable to DMA map source\n"); 3482 return -ENOMEM; 3483 } 3484 } else { 3485 mapped_nents = 0; 3486 } 3487 3488 /* allocate space for base edesc and link tables */ 3489 edesc = qi_cache_zalloc(GFP_DMA | flags); 3490 if (!edesc) { 3491 dma_unmap_sg(ctx->dev, req->src, src_nents, 3492 DMA_TO_DEVICE); 3493 return -ENOMEM; 3494 } 3495 3496 edesc->src_nents = src_nents; 3497 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3498 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3499 sizeof(*sg_table); 3500 sg_table = &edesc->sgt[0]; 3501 3502 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3503 DMA_BIDIRECTIONAL); 3504 if (ret) 3505 goto unmap_ctx; 3506 3507 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3508 if (ret) 3509 goto unmap_ctx; 3510 3511 if (mapped_nents) { 3512 sg_to_qm_sg_last(req->src, src_len, 3513 sg_table + qm_sg_src_index, 0); 3514 } else { 3515 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3516 true); 3517 } 3518 3519 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3520 qm_sg_bytes, DMA_TO_DEVICE); 3521 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3522 dev_err(ctx->dev, "unable to map S/G table\n"); 3523 ret = -ENOMEM; 3524 goto unmap_ctx; 3525 } 3526 edesc->qm_sg_bytes = qm_sg_bytes; 3527 3528 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3529 dpaa2_fl_set_final(in_fle, true); 3530 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3531 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3532 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3533 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3534 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3535 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3536 3537 req_ctx->flc = &ctx->flc[UPDATE]; 3538 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3539 req_ctx->cbk = ahash_done_bi; 3540 req_ctx->ctx = &req->base; 3541 req_ctx->edesc = edesc; 3542 3543 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3544 if (ret != -EINPROGRESS && 3545 !(ret == -EBUSY && 3546 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3547 goto unmap_ctx; 3548 } else if (*next_buflen) { 3549 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3550 req->nbytes, 0); 3551 *buflen = *next_buflen; 3552 3553 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3554 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3555 *buflen, 1); 3556 } 3557 3558 return ret; 3559 unmap_ctx: 3560 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3561 qi_cache_free(edesc); 3562 return ret; 3563 } 3564 3565 static int ahash_final_ctx(struct ahash_request *req) 3566 { 3567 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3568 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3569 struct caam_hash_state *state = ahash_request_ctx(req); 3570 struct caam_request *req_ctx = &state->caam_req; 3571 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3572 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3573 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3574 GFP_KERNEL : GFP_ATOMIC; 3575 int buflen = state->buflen; 3576 int qm_sg_bytes; 3577 int digestsize = crypto_ahash_digestsize(ahash); 3578 struct ahash_edesc *edesc; 3579 struct dpaa2_sg_entry *sg_table; 3580 int ret; 3581 3582 /* allocate space for base edesc and link tables */ 3583 edesc = qi_cache_zalloc(GFP_DMA | flags); 3584 if (!edesc) 3585 return -ENOMEM; 3586 3587 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3588 sg_table = &edesc->sgt[0]; 3589 3590 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3591 DMA_BIDIRECTIONAL); 3592 if (ret) 3593 goto unmap_ctx; 3594 3595 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3596 if (ret) 3597 goto unmap_ctx; 3598 3599 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3600 3601 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3602 DMA_TO_DEVICE); 3603 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3604 dev_err(ctx->dev, "unable to map S/G table\n"); 3605 ret = -ENOMEM; 3606 goto unmap_ctx; 3607 } 3608 edesc->qm_sg_bytes = qm_sg_bytes; 3609 3610 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3611 dpaa2_fl_set_final(in_fle, true); 3612 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3613 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3614 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3615 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3616 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3617 dpaa2_fl_set_len(out_fle, digestsize); 3618 3619 req_ctx->flc = &ctx->flc[FINALIZE]; 3620 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3621 req_ctx->cbk = ahash_done_ctx_src; 3622 req_ctx->ctx = &req->base; 3623 req_ctx->edesc = edesc; 3624 3625 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3626 if (ret == -EINPROGRESS || 3627 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3628 return ret; 3629 3630 unmap_ctx: 3631 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3632 qi_cache_free(edesc); 3633 return ret; 3634 } 3635 3636 static int ahash_finup_ctx(struct ahash_request *req) 3637 { 3638 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3639 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3640 struct caam_hash_state *state = ahash_request_ctx(req); 3641 struct caam_request *req_ctx = &state->caam_req; 3642 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3643 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3644 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3645 GFP_KERNEL : GFP_ATOMIC; 3646 int buflen = state->buflen; 3647 int qm_sg_bytes, qm_sg_src_index; 3648 int src_nents, mapped_nents; 3649 int digestsize = crypto_ahash_digestsize(ahash); 3650 struct ahash_edesc *edesc; 3651 struct dpaa2_sg_entry *sg_table; 3652 int ret; 3653 3654 src_nents = sg_nents_for_len(req->src, req->nbytes); 3655 if (src_nents < 0) { 3656 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3657 return src_nents; 3658 } 3659 3660 if (src_nents) { 3661 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3662 DMA_TO_DEVICE); 3663 if (!mapped_nents) { 3664 dev_err(ctx->dev, "unable to DMA map source\n"); 3665 return -ENOMEM; 3666 } 3667 } else { 3668 mapped_nents = 0; 3669 } 3670 3671 /* allocate space for base edesc and link tables */ 3672 edesc = qi_cache_zalloc(GFP_DMA | flags); 3673 if (!edesc) { 3674 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3675 return -ENOMEM; 3676 } 3677 3678 edesc->src_nents = src_nents; 3679 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3680 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3681 sizeof(*sg_table); 3682 sg_table = &edesc->sgt[0]; 3683 3684 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3685 DMA_BIDIRECTIONAL); 3686 if (ret) 3687 goto unmap_ctx; 3688 3689 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3690 if (ret) 3691 goto unmap_ctx; 3692 3693 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3694 3695 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3696 DMA_TO_DEVICE); 3697 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3698 dev_err(ctx->dev, "unable to map S/G table\n"); 3699 ret = -ENOMEM; 3700 goto unmap_ctx; 3701 } 3702 edesc->qm_sg_bytes = qm_sg_bytes; 3703 3704 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3705 dpaa2_fl_set_final(in_fle, true); 3706 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3707 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3708 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3709 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3710 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3711 dpaa2_fl_set_len(out_fle, digestsize); 3712 3713 req_ctx->flc = &ctx->flc[FINALIZE]; 3714 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3715 req_ctx->cbk = ahash_done_ctx_src; 3716 req_ctx->ctx = &req->base; 3717 req_ctx->edesc = edesc; 3718 3719 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3720 if (ret == -EINPROGRESS || 3721 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3722 return ret; 3723 3724 unmap_ctx: 3725 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3726 qi_cache_free(edesc); 3727 return ret; 3728 } 3729 3730 static int ahash_digest(struct ahash_request *req) 3731 { 3732 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3733 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3734 struct caam_hash_state *state = ahash_request_ctx(req); 3735 struct caam_request *req_ctx = &state->caam_req; 3736 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3737 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3738 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3739 GFP_KERNEL : GFP_ATOMIC; 3740 int digestsize = crypto_ahash_digestsize(ahash); 3741 int src_nents, mapped_nents; 3742 struct ahash_edesc *edesc; 3743 int ret = -ENOMEM; 3744 3745 state->buf_dma = 0; 3746 3747 src_nents = sg_nents_for_len(req->src, req->nbytes); 3748 if (src_nents < 0) { 3749 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3750 return src_nents; 3751 } 3752 3753 if (src_nents) { 3754 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3755 DMA_TO_DEVICE); 3756 if (!mapped_nents) { 3757 dev_err(ctx->dev, "unable to map source for DMA\n"); 3758 return ret; 3759 } 3760 } else { 3761 mapped_nents = 0; 3762 } 3763 3764 /* allocate space for base edesc and link tables */ 3765 edesc = qi_cache_zalloc(GFP_DMA | flags); 3766 if (!edesc) { 3767 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3768 return ret; 3769 } 3770 3771 edesc->src_nents = src_nents; 3772 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3773 3774 if (mapped_nents > 1) { 3775 int qm_sg_bytes; 3776 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3777 3778 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3779 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3780 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3781 qm_sg_bytes, DMA_TO_DEVICE); 3782 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3783 dev_err(ctx->dev, "unable to map S/G table\n"); 3784 goto unmap; 3785 } 3786 edesc->qm_sg_bytes = qm_sg_bytes; 3787 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3788 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3789 } else { 3790 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3791 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3792 } 3793 3794 state->ctx_dma_len = digestsize; 3795 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3796 DMA_FROM_DEVICE); 3797 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3798 dev_err(ctx->dev, "unable to map ctx\n"); 3799 state->ctx_dma = 0; 3800 goto unmap; 3801 } 3802 3803 dpaa2_fl_set_final(in_fle, true); 3804 dpaa2_fl_set_len(in_fle, req->nbytes); 3805 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3806 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3807 dpaa2_fl_set_len(out_fle, digestsize); 3808 3809 req_ctx->flc = &ctx->flc[DIGEST]; 3810 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3811 req_ctx->cbk = ahash_done; 3812 req_ctx->ctx = &req->base; 3813 req_ctx->edesc = edesc; 3814 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3815 if (ret == -EINPROGRESS || 3816 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3817 return ret; 3818 3819 unmap: 3820 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3821 qi_cache_free(edesc); 3822 return ret; 3823 } 3824 3825 static int ahash_final_no_ctx(struct ahash_request *req) 3826 { 3827 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3828 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3829 struct caam_hash_state *state = ahash_request_ctx(req); 3830 struct caam_request *req_ctx = &state->caam_req; 3831 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3832 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3833 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3834 GFP_KERNEL : GFP_ATOMIC; 3835 u8 *buf = state->buf; 3836 int buflen = state->buflen; 3837 int digestsize = crypto_ahash_digestsize(ahash); 3838 struct ahash_edesc *edesc; 3839 int ret = -ENOMEM; 3840 3841 /* allocate space for base edesc and link tables */ 3842 edesc = qi_cache_zalloc(GFP_DMA | flags); 3843 if (!edesc) 3844 return ret; 3845 3846 if (buflen) { 3847 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3848 DMA_TO_DEVICE); 3849 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3850 dev_err(ctx->dev, "unable to map src\n"); 3851 goto unmap; 3852 } 3853 } 3854 3855 state->ctx_dma_len = digestsize; 3856 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3857 DMA_FROM_DEVICE); 3858 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3859 dev_err(ctx->dev, "unable to map ctx\n"); 3860 state->ctx_dma = 0; 3861 goto unmap; 3862 } 3863 3864 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3865 dpaa2_fl_set_final(in_fle, true); 3866 /* 3867 * crypto engine requires the input entry to be present when 3868 * "frame list" FD is used. 3869 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3870 * in_fle zeroized (except for "Final" flag) is the best option. 3871 */ 3872 if (buflen) { 3873 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3874 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3875 dpaa2_fl_set_len(in_fle, buflen); 3876 } 3877 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3878 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3879 dpaa2_fl_set_len(out_fle, digestsize); 3880 3881 req_ctx->flc = &ctx->flc[DIGEST]; 3882 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3883 req_ctx->cbk = ahash_done; 3884 req_ctx->ctx = &req->base; 3885 req_ctx->edesc = edesc; 3886 3887 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3888 if (ret == -EINPROGRESS || 3889 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3890 return ret; 3891 3892 unmap: 3893 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3894 qi_cache_free(edesc); 3895 return ret; 3896 } 3897 3898 static int ahash_update_no_ctx(struct ahash_request *req) 3899 { 3900 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3901 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3902 struct caam_hash_state *state = ahash_request_ctx(req); 3903 struct caam_request *req_ctx = &state->caam_req; 3904 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3905 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3906 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3907 GFP_KERNEL : GFP_ATOMIC; 3908 u8 *buf = state->buf; 3909 int *buflen = &state->buflen; 3910 int *next_buflen = &state->next_buflen; 3911 int in_len = *buflen + req->nbytes, to_hash; 3912 int qm_sg_bytes, src_nents, mapped_nents; 3913 struct ahash_edesc *edesc; 3914 int ret = 0; 3915 3916 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3917 to_hash = in_len - *next_buflen; 3918 3919 if (to_hash) { 3920 struct dpaa2_sg_entry *sg_table; 3921 int src_len = req->nbytes - *next_buflen; 3922 3923 src_nents = sg_nents_for_len(req->src, src_len); 3924 if (src_nents < 0) { 3925 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3926 return src_nents; 3927 } 3928 3929 if (src_nents) { 3930 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3931 DMA_TO_DEVICE); 3932 if (!mapped_nents) { 3933 dev_err(ctx->dev, "unable to DMA map source\n"); 3934 return -ENOMEM; 3935 } 3936 } else { 3937 mapped_nents = 0; 3938 } 3939 3940 /* allocate space for base edesc and link tables */ 3941 edesc = qi_cache_zalloc(GFP_DMA | flags); 3942 if (!edesc) { 3943 dma_unmap_sg(ctx->dev, req->src, src_nents, 3944 DMA_TO_DEVICE); 3945 return -ENOMEM; 3946 } 3947 3948 edesc->src_nents = src_nents; 3949 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 3950 sizeof(*sg_table); 3951 sg_table = &edesc->sgt[0]; 3952 3953 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 3954 if (ret) 3955 goto unmap_ctx; 3956 3957 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 3958 3959 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3960 qm_sg_bytes, DMA_TO_DEVICE); 3961 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3962 dev_err(ctx->dev, "unable to map S/G table\n"); 3963 ret = -ENOMEM; 3964 goto unmap_ctx; 3965 } 3966 edesc->qm_sg_bytes = qm_sg_bytes; 3967 3968 state->ctx_dma_len = ctx->ctx_len; 3969 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 3970 ctx->ctx_len, DMA_FROM_DEVICE); 3971 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3972 dev_err(ctx->dev, "unable to map ctx\n"); 3973 state->ctx_dma = 0; 3974 ret = -ENOMEM; 3975 goto unmap_ctx; 3976 } 3977 3978 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3979 dpaa2_fl_set_final(in_fle, true); 3980 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3981 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3982 dpaa2_fl_set_len(in_fle, to_hash); 3983 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3984 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3985 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3986 3987 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 3988 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 3989 req_ctx->cbk = ahash_done_ctx_dst; 3990 req_ctx->ctx = &req->base; 3991 req_ctx->edesc = edesc; 3992 3993 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3994 if (ret != -EINPROGRESS && 3995 !(ret == -EBUSY && 3996 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3997 goto unmap_ctx; 3998 3999 state->update = ahash_update_ctx; 4000 state->finup = ahash_finup_ctx; 4001 state->final = ahash_final_ctx; 4002 } else if (*next_buflen) { 4003 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 4004 req->nbytes, 0); 4005 *buflen = *next_buflen; 4006 4007 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4008 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4009 *buflen, 1); 4010 } 4011 4012 return ret; 4013 unmap_ctx: 4014 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4015 qi_cache_free(edesc); 4016 return ret; 4017 } 4018 4019 static int ahash_finup_no_ctx(struct ahash_request *req) 4020 { 4021 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4022 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4023 struct caam_hash_state *state = ahash_request_ctx(req); 4024 struct caam_request *req_ctx = &state->caam_req; 4025 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4026 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4027 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4028 GFP_KERNEL : GFP_ATOMIC; 4029 int buflen = state->buflen; 4030 int qm_sg_bytes, src_nents, mapped_nents; 4031 int digestsize = crypto_ahash_digestsize(ahash); 4032 struct ahash_edesc *edesc; 4033 struct dpaa2_sg_entry *sg_table; 4034 int ret; 4035 4036 src_nents = sg_nents_for_len(req->src, req->nbytes); 4037 if (src_nents < 0) { 4038 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4039 return src_nents; 4040 } 4041 4042 if (src_nents) { 4043 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4044 DMA_TO_DEVICE); 4045 if (!mapped_nents) { 4046 dev_err(ctx->dev, "unable to DMA map source\n"); 4047 return -ENOMEM; 4048 } 4049 } else { 4050 mapped_nents = 0; 4051 } 4052 4053 /* allocate space for base edesc and link tables */ 4054 edesc = qi_cache_zalloc(GFP_DMA | flags); 4055 if (!edesc) { 4056 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4057 return -ENOMEM; 4058 } 4059 4060 edesc->src_nents = src_nents; 4061 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4062 sg_table = &edesc->sgt[0]; 4063 4064 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4065 if (ret) 4066 goto unmap; 4067 4068 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4069 4070 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4071 DMA_TO_DEVICE); 4072 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4073 dev_err(ctx->dev, "unable to map S/G table\n"); 4074 ret = -ENOMEM; 4075 goto unmap; 4076 } 4077 edesc->qm_sg_bytes = qm_sg_bytes; 4078 4079 state->ctx_dma_len = digestsize; 4080 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4081 DMA_FROM_DEVICE); 4082 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4083 dev_err(ctx->dev, "unable to map ctx\n"); 4084 state->ctx_dma = 0; 4085 ret = -ENOMEM; 4086 goto unmap; 4087 } 4088 4089 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4090 dpaa2_fl_set_final(in_fle, true); 4091 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4092 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4093 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4094 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4095 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4096 dpaa2_fl_set_len(out_fle, digestsize); 4097 4098 req_ctx->flc = &ctx->flc[DIGEST]; 4099 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4100 req_ctx->cbk = ahash_done; 4101 req_ctx->ctx = &req->base; 4102 req_ctx->edesc = edesc; 4103 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4104 if (ret != -EINPROGRESS && 4105 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4106 goto unmap; 4107 4108 return ret; 4109 unmap: 4110 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4111 qi_cache_free(edesc); 4112 return -ENOMEM; 4113 } 4114 4115 static int ahash_update_first(struct ahash_request *req) 4116 { 4117 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4118 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4119 struct caam_hash_state *state = ahash_request_ctx(req); 4120 struct caam_request *req_ctx = &state->caam_req; 4121 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4122 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4123 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4124 GFP_KERNEL : GFP_ATOMIC; 4125 u8 *buf = state->buf; 4126 int *buflen = &state->buflen; 4127 int *next_buflen = &state->next_buflen; 4128 int to_hash; 4129 int src_nents, mapped_nents; 4130 struct ahash_edesc *edesc; 4131 int ret = 0; 4132 4133 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4134 1); 4135 to_hash = req->nbytes - *next_buflen; 4136 4137 if (to_hash) { 4138 struct dpaa2_sg_entry *sg_table; 4139 int src_len = req->nbytes - *next_buflen; 4140 4141 src_nents = sg_nents_for_len(req->src, src_len); 4142 if (src_nents < 0) { 4143 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4144 return src_nents; 4145 } 4146 4147 if (src_nents) { 4148 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4149 DMA_TO_DEVICE); 4150 if (!mapped_nents) { 4151 dev_err(ctx->dev, "unable to map source for DMA\n"); 4152 return -ENOMEM; 4153 } 4154 } else { 4155 mapped_nents = 0; 4156 } 4157 4158 /* allocate space for base edesc and link tables */ 4159 edesc = qi_cache_zalloc(GFP_DMA | flags); 4160 if (!edesc) { 4161 dma_unmap_sg(ctx->dev, req->src, src_nents, 4162 DMA_TO_DEVICE); 4163 return -ENOMEM; 4164 } 4165 4166 edesc->src_nents = src_nents; 4167 sg_table = &edesc->sgt[0]; 4168 4169 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4170 dpaa2_fl_set_final(in_fle, true); 4171 dpaa2_fl_set_len(in_fle, to_hash); 4172 4173 if (mapped_nents > 1) { 4174 int qm_sg_bytes; 4175 4176 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4177 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4178 sizeof(*sg_table); 4179 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4180 qm_sg_bytes, 4181 DMA_TO_DEVICE); 4182 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4183 dev_err(ctx->dev, "unable to map S/G table\n"); 4184 ret = -ENOMEM; 4185 goto unmap_ctx; 4186 } 4187 edesc->qm_sg_bytes = qm_sg_bytes; 4188 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4189 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4190 } else { 4191 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4192 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4193 } 4194 4195 state->ctx_dma_len = ctx->ctx_len; 4196 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4197 ctx->ctx_len, DMA_FROM_DEVICE); 4198 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4199 dev_err(ctx->dev, "unable to map ctx\n"); 4200 state->ctx_dma = 0; 4201 ret = -ENOMEM; 4202 goto unmap_ctx; 4203 } 4204 4205 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4206 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4207 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4208 4209 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4210 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4211 req_ctx->cbk = ahash_done_ctx_dst; 4212 req_ctx->ctx = &req->base; 4213 req_ctx->edesc = edesc; 4214 4215 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4216 if (ret != -EINPROGRESS && 4217 !(ret == -EBUSY && req->base.flags & 4218 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4219 goto unmap_ctx; 4220 4221 state->update = ahash_update_ctx; 4222 state->finup = ahash_finup_ctx; 4223 state->final = ahash_final_ctx; 4224 } else if (*next_buflen) { 4225 state->update = ahash_update_no_ctx; 4226 state->finup = ahash_finup_no_ctx; 4227 state->final = ahash_final_no_ctx; 4228 scatterwalk_map_and_copy(buf, req->src, 0, 4229 req->nbytes, 0); 4230 *buflen = *next_buflen; 4231 4232 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4233 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4234 *buflen, 1); 4235 } 4236 4237 return ret; 4238 unmap_ctx: 4239 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4240 qi_cache_free(edesc); 4241 return ret; 4242 } 4243 4244 static int ahash_finup_first(struct ahash_request *req) 4245 { 4246 return ahash_digest(req); 4247 } 4248 4249 static int ahash_init(struct ahash_request *req) 4250 { 4251 struct caam_hash_state *state = ahash_request_ctx(req); 4252 4253 state->update = ahash_update_first; 4254 state->finup = ahash_finup_first; 4255 state->final = ahash_final_no_ctx; 4256 4257 state->ctx_dma = 0; 4258 state->ctx_dma_len = 0; 4259 state->buf_dma = 0; 4260 state->buflen = 0; 4261 state->next_buflen = 0; 4262 4263 return 0; 4264 } 4265 4266 static int ahash_update(struct ahash_request *req) 4267 { 4268 struct caam_hash_state *state = ahash_request_ctx(req); 4269 4270 return state->update(req); 4271 } 4272 4273 static int ahash_finup(struct ahash_request *req) 4274 { 4275 struct caam_hash_state *state = ahash_request_ctx(req); 4276 4277 return state->finup(req); 4278 } 4279 4280 static int ahash_final(struct ahash_request *req) 4281 { 4282 struct caam_hash_state *state = ahash_request_ctx(req); 4283 4284 return state->final(req); 4285 } 4286 4287 static int ahash_export(struct ahash_request *req, void *out) 4288 { 4289 struct caam_hash_state *state = ahash_request_ctx(req); 4290 struct caam_export_state *export = out; 4291 u8 *buf = state->buf; 4292 int len = state->buflen; 4293 4294 memcpy(export->buf, buf, len); 4295 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4296 export->buflen = len; 4297 export->update = state->update; 4298 export->final = state->final; 4299 export->finup = state->finup; 4300 4301 return 0; 4302 } 4303 4304 static int ahash_import(struct ahash_request *req, const void *in) 4305 { 4306 struct caam_hash_state *state = ahash_request_ctx(req); 4307 const struct caam_export_state *export = in; 4308 4309 memset(state, 0, sizeof(*state)); 4310 memcpy(state->buf, export->buf, export->buflen); 4311 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4312 state->buflen = export->buflen; 4313 state->update = export->update; 4314 state->final = export->final; 4315 state->finup = export->finup; 4316 4317 return 0; 4318 } 4319 4320 struct caam_hash_template { 4321 char name[CRYPTO_MAX_ALG_NAME]; 4322 char driver_name[CRYPTO_MAX_ALG_NAME]; 4323 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4324 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4325 unsigned int blocksize; 4326 struct ahash_alg template_ahash; 4327 u32 alg_type; 4328 }; 4329 4330 /* ahash descriptors */ 4331 static struct caam_hash_template driver_hash[] = { 4332 { 4333 .name = "sha1", 4334 .driver_name = "sha1-caam-qi2", 4335 .hmac_name = "hmac(sha1)", 4336 .hmac_driver_name = "hmac-sha1-caam-qi2", 4337 .blocksize = SHA1_BLOCK_SIZE, 4338 .template_ahash = { 4339 .init = ahash_init, 4340 .update = ahash_update, 4341 .final = ahash_final, 4342 .finup = ahash_finup, 4343 .digest = ahash_digest, 4344 .export = ahash_export, 4345 .import = ahash_import, 4346 .setkey = ahash_setkey, 4347 .halg = { 4348 .digestsize = SHA1_DIGEST_SIZE, 4349 .statesize = sizeof(struct caam_export_state), 4350 }, 4351 }, 4352 .alg_type = OP_ALG_ALGSEL_SHA1, 4353 }, { 4354 .name = "sha224", 4355 .driver_name = "sha224-caam-qi2", 4356 .hmac_name = "hmac(sha224)", 4357 .hmac_driver_name = "hmac-sha224-caam-qi2", 4358 .blocksize = SHA224_BLOCK_SIZE, 4359 .template_ahash = { 4360 .init = ahash_init, 4361 .update = ahash_update, 4362 .final = ahash_final, 4363 .finup = ahash_finup, 4364 .digest = ahash_digest, 4365 .export = ahash_export, 4366 .import = ahash_import, 4367 .setkey = ahash_setkey, 4368 .halg = { 4369 .digestsize = SHA224_DIGEST_SIZE, 4370 .statesize = sizeof(struct caam_export_state), 4371 }, 4372 }, 4373 .alg_type = OP_ALG_ALGSEL_SHA224, 4374 }, { 4375 .name = "sha256", 4376 .driver_name = "sha256-caam-qi2", 4377 .hmac_name = "hmac(sha256)", 4378 .hmac_driver_name = "hmac-sha256-caam-qi2", 4379 .blocksize = SHA256_BLOCK_SIZE, 4380 .template_ahash = { 4381 .init = ahash_init, 4382 .update = ahash_update, 4383 .final = ahash_final, 4384 .finup = ahash_finup, 4385 .digest = ahash_digest, 4386 .export = ahash_export, 4387 .import = ahash_import, 4388 .setkey = ahash_setkey, 4389 .halg = { 4390 .digestsize = SHA256_DIGEST_SIZE, 4391 .statesize = sizeof(struct caam_export_state), 4392 }, 4393 }, 4394 .alg_type = OP_ALG_ALGSEL_SHA256, 4395 }, { 4396 .name = "sha384", 4397 .driver_name = "sha384-caam-qi2", 4398 .hmac_name = "hmac(sha384)", 4399 .hmac_driver_name = "hmac-sha384-caam-qi2", 4400 .blocksize = SHA384_BLOCK_SIZE, 4401 .template_ahash = { 4402 .init = ahash_init, 4403 .update = ahash_update, 4404 .final = ahash_final, 4405 .finup = ahash_finup, 4406 .digest = ahash_digest, 4407 .export = ahash_export, 4408 .import = ahash_import, 4409 .setkey = ahash_setkey, 4410 .halg = { 4411 .digestsize = SHA384_DIGEST_SIZE, 4412 .statesize = sizeof(struct caam_export_state), 4413 }, 4414 }, 4415 .alg_type = OP_ALG_ALGSEL_SHA384, 4416 }, { 4417 .name = "sha512", 4418 .driver_name = "sha512-caam-qi2", 4419 .hmac_name = "hmac(sha512)", 4420 .hmac_driver_name = "hmac-sha512-caam-qi2", 4421 .blocksize = SHA512_BLOCK_SIZE, 4422 .template_ahash = { 4423 .init = ahash_init, 4424 .update = ahash_update, 4425 .final = ahash_final, 4426 .finup = ahash_finup, 4427 .digest = ahash_digest, 4428 .export = ahash_export, 4429 .import = ahash_import, 4430 .setkey = ahash_setkey, 4431 .halg = { 4432 .digestsize = SHA512_DIGEST_SIZE, 4433 .statesize = sizeof(struct caam_export_state), 4434 }, 4435 }, 4436 .alg_type = OP_ALG_ALGSEL_SHA512, 4437 }, { 4438 .name = "md5", 4439 .driver_name = "md5-caam-qi2", 4440 .hmac_name = "hmac(md5)", 4441 .hmac_driver_name = "hmac-md5-caam-qi2", 4442 .blocksize = MD5_BLOCK_WORDS * 4, 4443 .template_ahash = { 4444 .init = ahash_init, 4445 .update = ahash_update, 4446 .final = ahash_final, 4447 .finup = ahash_finup, 4448 .digest = ahash_digest, 4449 .export = ahash_export, 4450 .import = ahash_import, 4451 .setkey = ahash_setkey, 4452 .halg = { 4453 .digestsize = MD5_DIGEST_SIZE, 4454 .statesize = sizeof(struct caam_export_state), 4455 }, 4456 }, 4457 .alg_type = OP_ALG_ALGSEL_MD5, 4458 } 4459 }; 4460 4461 struct caam_hash_alg { 4462 struct list_head entry; 4463 struct device *dev; 4464 int alg_type; 4465 struct ahash_alg ahash_alg; 4466 }; 4467 4468 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4469 { 4470 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4471 struct crypto_alg *base = tfm->__crt_alg; 4472 struct hash_alg_common *halg = 4473 container_of(base, struct hash_alg_common, base); 4474 struct ahash_alg *alg = 4475 container_of(halg, struct ahash_alg, halg); 4476 struct caam_hash_alg *caam_hash = 4477 container_of(alg, struct caam_hash_alg, ahash_alg); 4478 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4479 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4480 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4481 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4482 HASH_MSG_LEN + 32, 4483 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4484 HASH_MSG_LEN + 64, 4485 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4486 dma_addr_t dma_addr; 4487 int i; 4488 4489 ctx->dev = caam_hash->dev; 4490 4491 if (alg->setkey) { 4492 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4493 ARRAY_SIZE(ctx->key), 4494 DMA_TO_DEVICE, 4495 DMA_ATTR_SKIP_CPU_SYNC); 4496 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4497 dev_err(ctx->dev, "unable to map key\n"); 4498 return -ENOMEM; 4499 } 4500 } 4501 4502 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4503 DMA_BIDIRECTIONAL, 4504 DMA_ATTR_SKIP_CPU_SYNC); 4505 if (dma_mapping_error(ctx->dev, dma_addr)) { 4506 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4507 if (ctx->adata.key_dma) 4508 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4509 ARRAY_SIZE(ctx->key), 4510 DMA_TO_DEVICE, 4511 DMA_ATTR_SKIP_CPU_SYNC); 4512 return -ENOMEM; 4513 } 4514 4515 for (i = 0; i < HASH_NUM_OP; i++) 4516 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4517 4518 /* copy descriptor header template value */ 4519 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4520 4521 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4522 OP_ALG_ALGSEL_SUBMASK) >> 4523 OP_ALG_ALGSEL_SHIFT]; 4524 4525 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 4526 sizeof(struct caam_hash_state)); 4527 4528 return ahash_set_sh_desc(ahash); 4529 } 4530 4531 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4532 { 4533 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4534 4535 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4536 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4537 if (ctx->adata.key_dma) 4538 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4539 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4540 DMA_ATTR_SKIP_CPU_SYNC); 4541 } 4542 4543 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4544 struct caam_hash_template *template, bool keyed) 4545 { 4546 struct caam_hash_alg *t_alg; 4547 struct ahash_alg *halg; 4548 struct crypto_alg *alg; 4549 4550 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4551 if (!t_alg) 4552 return ERR_PTR(-ENOMEM); 4553 4554 t_alg->ahash_alg = template->template_ahash; 4555 halg = &t_alg->ahash_alg; 4556 alg = &halg->halg.base; 4557 4558 if (keyed) { 4559 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4560 template->hmac_name); 4561 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4562 template->hmac_driver_name); 4563 } else { 4564 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4565 template->name); 4566 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4567 template->driver_name); 4568 t_alg->ahash_alg.setkey = NULL; 4569 } 4570 alg->cra_module = THIS_MODULE; 4571 alg->cra_init = caam_hash_cra_init; 4572 alg->cra_exit = caam_hash_cra_exit; 4573 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 4574 alg->cra_priority = CAAM_CRA_PRIORITY; 4575 alg->cra_blocksize = template->blocksize; 4576 alg->cra_alignmask = 0; 4577 alg->cra_flags = CRYPTO_ALG_ASYNC; 4578 4579 t_alg->alg_type = template->alg_type; 4580 t_alg->dev = dev; 4581 4582 return t_alg; 4583 } 4584 4585 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4586 { 4587 struct dpaa2_caam_priv_per_cpu *ppriv; 4588 4589 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4590 napi_schedule_irqoff(&ppriv->napi); 4591 } 4592 4593 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4594 { 4595 struct device *dev = priv->dev; 4596 struct dpaa2_io_notification_ctx *nctx; 4597 struct dpaa2_caam_priv_per_cpu *ppriv; 4598 int err, i = 0, cpu; 4599 4600 for_each_online_cpu(cpu) { 4601 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4602 ppriv->priv = priv; 4603 nctx = &ppriv->nctx; 4604 nctx->is_cdan = 0; 4605 nctx->id = ppriv->rsp_fqid; 4606 nctx->desired_cpu = cpu; 4607 nctx->cb = dpaa2_caam_fqdan_cb; 4608 4609 /* Register notification callbacks */ 4610 ppriv->dpio = dpaa2_io_service_select(cpu); 4611 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4612 if (unlikely(err)) { 4613 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4614 nctx->cb = NULL; 4615 /* 4616 * If no affine DPIO for this core, there's probably 4617 * none available for next cores either. Signal we want 4618 * to retry later, in case the DPIO devices weren't 4619 * probed yet. 4620 */ 4621 err = -EPROBE_DEFER; 4622 goto err; 4623 } 4624 4625 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4626 dev); 4627 if (unlikely(!ppriv->store)) { 4628 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4629 err = -ENOMEM; 4630 goto err; 4631 } 4632 4633 if (++i == priv->num_pairs) 4634 break; 4635 } 4636 4637 return 0; 4638 4639 err: 4640 for_each_online_cpu(cpu) { 4641 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4642 if (!ppriv->nctx.cb) 4643 break; 4644 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4645 } 4646 4647 for_each_online_cpu(cpu) { 4648 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4649 if (!ppriv->store) 4650 break; 4651 dpaa2_io_store_destroy(ppriv->store); 4652 } 4653 4654 return err; 4655 } 4656 4657 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4658 { 4659 struct dpaa2_caam_priv_per_cpu *ppriv; 4660 int i = 0, cpu; 4661 4662 for_each_online_cpu(cpu) { 4663 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4664 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4665 priv->dev); 4666 dpaa2_io_store_destroy(ppriv->store); 4667 4668 if (++i == priv->num_pairs) 4669 return; 4670 } 4671 } 4672 4673 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4674 { 4675 struct dpseci_rx_queue_cfg rx_queue_cfg; 4676 struct device *dev = priv->dev; 4677 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4678 struct dpaa2_caam_priv_per_cpu *ppriv; 4679 int err = 0, i = 0, cpu; 4680 4681 /* Configure Rx queues */ 4682 for_each_online_cpu(cpu) { 4683 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4684 4685 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4686 DPSECI_QUEUE_OPT_USER_CTX; 4687 rx_queue_cfg.order_preservation_en = 0; 4688 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4689 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4690 /* 4691 * Rx priority (WQ) doesn't really matter, since we use 4692 * pull mode, i.e. volatile dequeues from specific FQs 4693 */ 4694 rx_queue_cfg.dest_cfg.priority = 0; 4695 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4696 4697 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4698 &rx_queue_cfg); 4699 if (err) { 4700 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4701 err); 4702 return err; 4703 } 4704 4705 if (++i == priv->num_pairs) 4706 break; 4707 } 4708 4709 return err; 4710 } 4711 4712 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4713 { 4714 struct device *dev = priv->dev; 4715 4716 if (!priv->cscn_mem) 4717 return; 4718 4719 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4720 kfree(priv->cscn_mem); 4721 } 4722 4723 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4724 { 4725 struct device *dev = priv->dev; 4726 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4727 4728 dpaa2_dpseci_congestion_free(priv); 4729 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4730 } 4731 4732 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4733 const struct dpaa2_fd *fd) 4734 { 4735 struct caam_request *req; 4736 u32 fd_err; 4737 4738 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4739 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4740 return; 4741 } 4742 4743 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4744 if (unlikely(fd_err)) 4745 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4746 4747 /* 4748 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4749 * in FD[ERR] or FD[FRC]. 4750 */ 4751 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4752 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4753 DMA_BIDIRECTIONAL); 4754 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4755 } 4756 4757 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4758 { 4759 int err; 4760 4761 /* Retry while portal is busy */ 4762 do { 4763 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4764 ppriv->store); 4765 } while (err == -EBUSY); 4766 4767 if (unlikely(err)) 4768 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4769 4770 return err; 4771 } 4772 4773 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4774 { 4775 struct dpaa2_dq *dq; 4776 int cleaned = 0, is_last; 4777 4778 do { 4779 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4780 if (unlikely(!dq)) { 4781 if (unlikely(!is_last)) { 4782 dev_dbg(ppriv->priv->dev, 4783 "FQ %d returned no valid frames\n", 4784 ppriv->rsp_fqid); 4785 /* 4786 * MUST retry until we get some sort of 4787 * valid response token (be it "empty dequeue" 4788 * or a valid frame). 4789 */ 4790 continue; 4791 } 4792 break; 4793 } 4794 4795 /* Process FD */ 4796 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4797 cleaned++; 4798 } while (!is_last); 4799 4800 return cleaned; 4801 } 4802 4803 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4804 { 4805 struct dpaa2_caam_priv_per_cpu *ppriv; 4806 struct dpaa2_caam_priv *priv; 4807 int err, cleaned = 0, store_cleaned; 4808 4809 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4810 priv = ppriv->priv; 4811 4812 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4813 return 0; 4814 4815 do { 4816 store_cleaned = dpaa2_caam_store_consume(ppriv); 4817 cleaned += store_cleaned; 4818 4819 if (store_cleaned == 0 || 4820 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4821 break; 4822 4823 /* Try to dequeue some more */ 4824 err = dpaa2_caam_pull_fq(ppriv); 4825 if (unlikely(err)) 4826 break; 4827 } while (1); 4828 4829 if (cleaned < budget) { 4830 napi_complete_done(napi, cleaned); 4831 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4832 if (unlikely(err)) 4833 dev_err(priv->dev, "Notification rearm failed: %d\n", 4834 err); 4835 } 4836 4837 return cleaned; 4838 } 4839 4840 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4841 u16 token) 4842 { 4843 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4844 struct device *dev = priv->dev; 4845 int err; 4846 4847 /* 4848 * Congestion group feature supported starting with DPSECI API v5.1 4849 * and only when object has been created with this capability. 4850 */ 4851 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4852 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4853 return 0; 4854 4855 priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, 4856 GFP_KERNEL | GFP_DMA); 4857 if (!priv->cscn_mem) 4858 return -ENOMEM; 4859 4860 priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); 4861 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, 4862 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4863 if (dma_mapping_error(dev, priv->cscn_dma)) { 4864 dev_err(dev, "Error mapping CSCN memory area\n"); 4865 err = -ENOMEM; 4866 goto err_dma_map; 4867 } 4868 4869 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4870 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4871 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4872 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4873 cong_notif_cfg.message_iova = priv->cscn_dma; 4874 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4875 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4876 DPSECI_CGN_MODE_COHERENT_WRITE; 4877 4878 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4879 &cong_notif_cfg); 4880 if (err) { 4881 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4882 goto err_set_cong; 4883 } 4884 4885 return 0; 4886 4887 err_set_cong: 4888 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4889 err_dma_map: 4890 kfree(priv->cscn_mem); 4891 4892 return err; 4893 } 4894 4895 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4896 { 4897 struct device *dev = &ls_dev->dev; 4898 struct dpaa2_caam_priv *priv; 4899 struct dpaa2_caam_priv_per_cpu *ppriv; 4900 int err, cpu; 4901 u8 i; 4902 4903 priv = dev_get_drvdata(dev); 4904 4905 priv->dev = dev; 4906 priv->dpsec_id = ls_dev->obj_desc.id; 4907 4908 /* Get a handle for the DPSECI this interface is associate with */ 4909 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 4910 if (err) { 4911 dev_err(dev, "dpseci_open() failed: %d\n", err); 4912 goto err_open; 4913 } 4914 4915 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 4916 &priv->minor_ver); 4917 if (err) { 4918 dev_err(dev, "dpseci_get_api_version() failed\n"); 4919 goto err_get_vers; 4920 } 4921 4922 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 4923 4924 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 4925 &priv->dpseci_attr); 4926 if (err) { 4927 dev_err(dev, "dpseci_get_attributes() failed\n"); 4928 goto err_get_vers; 4929 } 4930 4931 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 4932 &priv->sec_attr); 4933 if (err) { 4934 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 4935 goto err_get_vers; 4936 } 4937 4938 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 4939 if (err) { 4940 dev_err(dev, "setup_congestion() failed\n"); 4941 goto err_get_vers; 4942 } 4943 4944 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 4945 priv->dpseci_attr.num_tx_queues); 4946 if (priv->num_pairs > num_online_cpus()) { 4947 dev_warn(dev, "%d queues won't be used\n", 4948 priv->num_pairs - num_online_cpus()); 4949 priv->num_pairs = num_online_cpus(); 4950 } 4951 4952 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 4953 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4954 &priv->rx_queue_attr[i]); 4955 if (err) { 4956 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 4957 goto err_get_rx_queue; 4958 } 4959 } 4960 4961 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 4962 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4963 &priv->tx_queue_attr[i]); 4964 if (err) { 4965 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 4966 goto err_get_rx_queue; 4967 } 4968 } 4969 4970 i = 0; 4971 for_each_online_cpu(cpu) { 4972 u8 j; 4973 4974 j = i % priv->num_pairs; 4975 4976 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4977 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 4978 4979 /* 4980 * Allow all cores to enqueue, while only some of them 4981 * will take part in dequeuing. 4982 */ 4983 if (++i > priv->num_pairs) 4984 continue; 4985 4986 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 4987 ppriv->prio = j; 4988 4989 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 4990 priv->rx_queue_attr[j].fqid, 4991 priv->tx_queue_attr[j].fqid); 4992 4993 ppriv->net_dev.dev = *dev; 4994 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 4995 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, 4996 DPAA2_CAAM_NAPI_WEIGHT); 4997 } 4998 4999 return 0; 5000 5001 err_get_rx_queue: 5002 dpaa2_dpseci_congestion_free(priv); 5003 err_get_vers: 5004 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5005 err_open: 5006 return err; 5007 } 5008 5009 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 5010 { 5011 struct device *dev = priv->dev; 5012 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5013 struct dpaa2_caam_priv_per_cpu *ppriv; 5014 int i; 5015 5016 for (i = 0; i < priv->num_pairs; i++) { 5017 ppriv = per_cpu_ptr(priv->ppriv, i); 5018 napi_enable(&ppriv->napi); 5019 } 5020 5021 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 5022 } 5023 5024 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 5025 { 5026 struct device *dev = priv->dev; 5027 struct dpaa2_caam_priv_per_cpu *ppriv; 5028 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5029 int i, err = 0, enabled; 5030 5031 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5032 if (err) { 5033 dev_err(dev, "dpseci_disable() failed\n"); 5034 return err; 5035 } 5036 5037 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5038 if (err) { 5039 dev_err(dev, "dpseci_is_enabled() failed\n"); 5040 return err; 5041 } 5042 5043 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 5044 5045 for (i = 0; i < priv->num_pairs; i++) { 5046 ppriv = per_cpu_ptr(priv->ppriv, i); 5047 napi_disable(&ppriv->napi); 5048 netif_napi_del(&ppriv->napi); 5049 } 5050 5051 return 0; 5052 } 5053 5054 static struct list_head hash_list; 5055 5056 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5057 { 5058 struct device *dev; 5059 struct dpaa2_caam_priv *priv; 5060 int i, err = 0; 5061 bool registered = false; 5062 5063 /* 5064 * There is no way to get CAAM endianness - there is no direct register 5065 * space access and MC f/w does not provide this attribute. 5066 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5067 * property. 5068 */ 5069 caam_little_end = true; 5070 5071 caam_imx = false; 5072 5073 dev = &dpseci_dev->dev; 5074 5075 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5076 if (!priv) 5077 return -ENOMEM; 5078 5079 dev_set_drvdata(dev, priv); 5080 5081 priv->domain = iommu_get_domain_for_dev(dev); 5082 5083 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5084 0, SLAB_CACHE_DMA, NULL); 5085 if (!qi_cache) { 5086 dev_err(dev, "Can't allocate SEC cache\n"); 5087 return -ENOMEM; 5088 } 5089 5090 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5091 if (err) { 5092 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5093 goto err_dma_mask; 5094 } 5095 5096 /* Obtain a MC portal */ 5097 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5098 if (err) { 5099 if (err == -ENXIO) 5100 err = -EPROBE_DEFER; 5101 else 5102 dev_err(dev, "MC portal allocation failed\n"); 5103 5104 goto err_dma_mask; 5105 } 5106 5107 priv->ppriv = alloc_percpu(*priv->ppriv); 5108 if (!priv->ppriv) { 5109 dev_err(dev, "alloc_percpu() failed\n"); 5110 err = -ENOMEM; 5111 goto err_alloc_ppriv; 5112 } 5113 5114 /* DPSECI initialization */ 5115 err = dpaa2_dpseci_setup(dpseci_dev); 5116 if (err) { 5117 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5118 goto err_dpseci_setup; 5119 } 5120 5121 /* DPIO */ 5122 err = dpaa2_dpseci_dpio_setup(priv); 5123 if (err) { 5124 if (err != -EPROBE_DEFER) 5125 dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); 5126 goto err_dpio_setup; 5127 } 5128 5129 /* DPSECI binding to DPIO */ 5130 err = dpaa2_dpseci_bind(priv); 5131 if (err) { 5132 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5133 goto err_bind; 5134 } 5135 5136 /* DPSECI enable */ 5137 err = dpaa2_dpseci_enable(priv); 5138 if (err) { 5139 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5140 goto err_bind; 5141 } 5142 5143 dpaa2_dpseci_debugfs_init(priv); 5144 5145 /* register crypto algorithms the device supports */ 5146 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5147 struct caam_skcipher_alg *t_alg = driver_algs + i; 5148 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5149 5150 /* Skip DES algorithms if not supported by device */ 5151 if (!priv->sec_attr.des_acc_num && 5152 (alg_sel == OP_ALG_ALGSEL_3DES || 5153 alg_sel == OP_ALG_ALGSEL_DES)) 5154 continue; 5155 5156 /* Skip AES algorithms if not supported by device */ 5157 if (!priv->sec_attr.aes_acc_num && 5158 alg_sel == OP_ALG_ALGSEL_AES) 5159 continue; 5160 5161 /* Skip CHACHA20 algorithms if not supported by device */ 5162 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5163 !priv->sec_attr.ccha_acc_num) 5164 continue; 5165 5166 t_alg->caam.dev = dev; 5167 caam_skcipher_alg_init(t_alg); 5168 5169 err = crypto_register_skcipher(&t_alg->skcipher); 5170 if (err) { 5171 dev_warn(dev, "%s alg registration failed: %d\n", 5172 t_alg->skcipher.base.cra_driver_name, err); 5173 continue; 5174 } 5175 5176 t_alg->registered = true; 5177 registered = true; 5178 } 5179 5180 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5181 struct caam_aead_alg *t_alg = driver_aeads + i; 5182 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5183 OP_ALG_ALGSEL_MASK; 5184 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5185 OP_ALG_ALGSEL_MASK; 5186 5187 /* Skip DES algorithms if not supported by device */ 5188 if (!priv->sec_attr.des_acc_num && 5189 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5190 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5191 continue; 5192 5193 /* Skip AES algorithms if not supported by device */ 5194 if (!priv->sec_attr.aes_acc_num && 5195 c1_alg_sel == OP_ALG_ALGSEL_AES) 5196 continue; 5197 5198 /* Skip CHACHA20 algorithms if not supported by device */ 5199 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5200 !priv->sec_attr.ccha_acc_num) 5201 continue; 5202 5203 /* Skip POLY1305 algorithms if not supported by device */ 5204 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5205 !priv->sec_attr.ptha_acc_num) 5206 continue; 5207 5208 /* 5209 * Skip algorithms requiring message digests 5210 * if MD not supported by device. 5211 */ 5212 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5213 !priv->sec_attr.md_acc_num) 5214 continue; 5215 5216 t_alg->caam.dev = dev; 5217 caam_aead_alg_init(t_alg); 5218 5219 err = crypto_register_aead(&t_alg->aead); 5220 if (err) { 5221 dev_warn(dev, "%s alg registration failed: %d\n", 5222 t_alg->aead.base.cra_driver_name, err); 5223 continue; 5224 } 5225 5226 t_alg->registered = true; 5227 registered = true; 5228 } 5229 if (registered) 5230 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5231 5232 /* register hash algorithms the device supports */ 5233 INIT_LIST_HEAD(&hash_list); 5234 5235 /* 5236 * Skip registration of any hashing algorithms if MD block 5237 * is not present. 5238 */ 5239 if (!priv->sec_attr.md_acc_num) 5240 return 0; 5241 5242 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5243 struct caam_hash_alg *t_alg; 5244 struct caam_hash_template *alg = driver_hash + i; 5245 5246 /* register hmac version */ 5247 t_alg = caam_hash_alloc(dev, alg, true); 5248 if (IS_ERR(t_alg)) { 5249 err = PTR_ERR(t_alg); 5250 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5251 alg->driver_name, err); 5252 continue; 5253 } 5254 5255 err = crypto_register_ahash(&t_alg->ahash_alg); 5256 if (err) { 5257 dev_warn(dev, "%s alg registration failed: %d\n", 5258 t_alg->ahash_alg.halg.base.cra_driver_name, 5259 err); 5260 kfree(t_alg); 5261 } else { 5262 list_add_tail(&t_alg->entry, &hash_list); 5263 } 5264 5265 /* register unkeyed version */ 5266 t_alg = caam_hash_alloc(dev, alg, false); 5267 if (IS_ERR(t_alg)) { 5268 err = PTR_ERR(t_alg); 5269 dev_warn(dev, "%s alg allocation failed: %d\n", 5270 alg->driver_name, err); 5271 continue; 5272 } 5273 5274 err = crypto_register_ahash(&t_alg->ahash_alg); 5275 if (err) { 5276 dev_warn(dev, "%s alg registration failed: %d\n", 5277 t_alg->ahash_alg.halg.base.cra_driver_name, 5278 err); 5279 kfree(t_alg); 5280 } else { 5281 list_add_tail(&t_alg->entry, &hash_list); 5282 } 5283 } 5284 if (!list_empty(&hash_list)) 5285 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5286 5287 return err; 5288 5289 err_bind: 5290 dpaa2_dpseci_dpio_free(priv); 5291 err_dpio_setup: 5292 dpaa2_dpseci_free(priv); 5293 err_dpseci_setup: 5294 free_percpu(priv->ppriv); 5295 err_alloc_ppriv: 5296 fsl_mc_portal_free(priv->mc_io); 5297 err_dma_mask: 5298 kmem_cache_destroy(qi_cache); 5299 5300 return err; 5301 } 5302 5303 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5304 { 5305 struct device *dev; 5306 struct dpaa2_caam_priv *priv; 5307 int i; 5308 5309 dev = &ls_dev->dev; 5310 priv = dev_get_drvdata(dev); 5311 5312 dpaa2_dpseci_debugfs_exit(priv); 5313 5314 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5315 struct caam_aead_alg *t_alg = driver_aeads + i; 5316 5317 if (t_alg->registered) 5318 crypto_unregister_aead(&t_alg->aead); 5319 } 5320 5321 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5322 struct caam_skcipher_alg *t_alg = driver_algs + i; 5323 5324 if (t_alg->registered) 5325 crypto_unregister_skcipher(&t_alg->skcipher); 5326 } 5327 5328 if (hash_list.next) { 5329 struct caam_hash_alg *t_hash_alg, *p; 5330 5331 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5332 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5333 list_del(&t_hash_alg->entry); 5334 kfree(t_hash_alg); 5335 } 5336 } 5337 5338 dpaa2_dpseci_disable(priv); 5339 dpaa2_dpseci_dpio_free(priv); 5340 dpaa2_dpseci_free(priv); 5341 free_percpu(priv->ppriv); 5342 fsl_mc_portal_free(priv->mc_io); 5343 kmem_cache_destroy(qi_cache); 5344 5345 return 0; 5346 } 5347 5348 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5349 { 5350 struct dpaa2_fd fd; 5351 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5352 struct dpaa2_caam_priv_per_cpu *ppriv; 5353 int err = 0, i; 5354 5355 if (IS_ERR(req)) 5356 return PTR_ERR(req); 5357 5358 if (priv->cscn_mem) { 5359 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5360 DPAA2_CSCN_SIZE, 5361 DMA_FROM_DEVICE); 5362 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { 5363 dev_dbg_ratelimited(dev, "Dropping request\n"); 5364 return -EBUSY; 5365 } 5366 } 5367 5368 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5369 5370 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5371 DMA_BIDIRECTIONAL); 5372 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5373 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5374 goto err_out; 5375 } 5376 5377 memset(&fd, 0, sizeof(fd)); 5378 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5379 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5380 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5381 dpaa2_fd_set_flc(&fd, req->flc_dma); 5382 5383 ppriv = this_cpu_ptr(priv->ppriv); 5384 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5385 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5386 &fd); 5387 if (err != -EBUSY) 5388 break; 5389 5390 cpu_relax(); 5391 } 5392 5393 if (unlikely(err)) { 5394 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5395 goto err_out; 5396 } 5397 5398 return -EINPROGRESS; 5399 5400 err_out: 5401 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5402 DMA_BIDIRECTIONAL); 5403 return -EIO; 5404 } 5405 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5406 5407 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5408 { 5409 .vendor = FSL_MC_VENDOR_FREESCALE, 5410 .obj_type = "dpseci", 5411 }, 5412 { .vendor = 0x0 } 5413 }; 5414 5415 static struct fsl_mc_driver dpaa2_caam_driver = { 5416 .driver = { 5417 .name = KBUILD_MODNAME, 5418 .owner = THIS_MODULE, 5419 }, 5420 .probe = dpaa2_caam_probe, 5421 .remove = dpaa2_caam_remove, 5422 .match_id_table = dpaa2_caam_match_id_table 5423 }; 5424 5425 MODULE_LICENSE("Dual BSD/GPL"); 5426 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5427 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5428 5429 module_fsl_mc_driver(dpaa2_caam_driver); 5430