1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include "dpseci-debugfs.h" 19 #include <linux/fsl/mc.h> 20 #include <soc/fsl/dpaa2-io.h> 21 #include <soc/fsl/dpaa2-fd.h> 22 #include <crypto/xts.h> 23 #include <asm/unaligned.h> 24 25 #define CAAM_CRA_PRIORITY 2000 26 27 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 28 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 29 SHA512_DIGEST_SIZE * 2) 30 31 /* 32 * This is a a cache of buffers, from which the users of CAAM QI driver 33 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 34 * NOTE: A more elegant solution would be to have some headroom in the frames 35 * being processed. This can be added by the dpaa2-eth driver. This would 36 * pose a problem for userspace application processing which cannot 37 * know of this limitation. So for now, this will work. 38 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 39 */ 40 static struct kmem_cache *qi_cache; 41 42 struct caam_alg_entry { 43 struct device *dev; 44 int class1_alg_type; 45 int class2_alg_type; 46 bool rfc3686; 47 bool geniv; 48 bool nodkp; 49 }; 50 51 struct caam_aead_alg { 52 struct aead_alg aead; 53 struct caam_alg_entry caam; 54 bool registered; 55 }; 56 57 struct caam_skcipher_alg { 58 struct skcipher_alg skcipher; 59 struct caam_alg_entry caam; 60 bool registered; 61 }; 62 63 /** 64 * struct caam_ctx - per-session context 65 * @flc: Flow Contexts array 66 * @key: [authentication key], encryption key 67 * @flc_dma: I/O virtual addresses of the Flow Contexts 68 * @key_dma: I/O virtual address of the key 69 * @dir: DMA direction for mapping key and Flow Contexts 70 * @dev: dpseci device 71 * @adata: authentication algorithm details 72 * @cdata: encryption algorithm details 73 * @authsize: authentication tag (a.k.a. ICV / MAC) size 74 */ 75 struct caam_ctx { 76 struct caam_flc flc[NUM_OP]; 77 u8 key[CAAM_MAX_KEY_SIZE]; 78 dma_addr_t flc_dma[NUM_OP]; 79 dma_addr_t key_dma; 80 enum dma_data_direction dir; 81 struct device *dev; 82 struct alginfo adata; 83 struct alginfo cdata; 84 unsigned int authsize; 85 bool xts_key_fallback; 86 struct crypto_skcipher *fallback; 87 }; 88 89 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 90 dma_addr_t iova_addr) 91 { 92 phys_addr_t phys_addr; 93 94 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 95 iova_addr; 96 97 return phys_to_virt(phys_addr); 98 } 99 100 /* 101 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 102 * 103 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 104 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 105 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 106 * hosting 16 SG entries. 107 * 108 * @flags - flags that would be used for the equivalent kmalloc(..) call 109 * 110 * Returns a pointer to a retrieved buffer on success or NULL on failure. 111 */ 112 static inline void *qi_cache_zalloc(gfp_t flags) 113 { 114 return kmem_cache_zalloc(qi_cache, flags); 115 } 116 117 /* 118 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 119 * 120 * @obj - buffer previously allocated by qi_cache_zalloc 121 * 122 * No checking is being done, the call is a passthrough call to 123 * kmem_cache_free(...) 124 */ 125 static inline void qi_cache_free(void *obj) 126 { 127 kmem_cache_free(qi_cache, obj); 128 } 129 130 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 131 { 132 switch (crypto_tfm_alg_type(areq->tfm)) { 133 case CRYPTO_ALG_TYPE_SKCIPHER: 134 return skcipher_request_ctx(skcipher_request_cast(areq)); 135 case CRYPTO_ALG_TYPE_AEAD: 136 return aead_request_ctx(container_of(areq, struct aead_request, 137 base)); 138 case CRYPTO_ALG_TYPE_AHASH: 139 return ahash_request_ctx(ahash_request_cast(areq)); 140 default: 141 return ERR_PTR(-EINVAL); 142 } 143 } 144 145 static void caam_unmap(struct device *dev, struct scatterlist *src, 146 struct scatterlist *dst, int src_nents, 147 int dst_nents, dma_addr_t iv_dma, int ivsize, 148 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 149 int qm_sg_bytes) 150 { 151 if (dst != src) { 152 if (src_nents) 153 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 154 if (dst_nents) 155 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 156 } else { 157 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 158 } 159 160 if (iv_dma) 161 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 162 163 if (qm_sg_bytes) 164 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 165 } 166 167 static int aead_set_sh_desc(struct crypto_aead *aead) 168 { 169 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 170 typeof(*alg), aead); 171 struct caam_ctx *ctx = crypto_aead_ctx(aead); 172 unsigned int ivsize = crypto_aead_ivsize(aead); 173 struct device *dev = ctx->dev; 174 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 175 struct caam_flc *flc; 176 u32 *desc; 177 u32 ctx1_iv_off = 0; 178 u32 *nonce = NULL; 179 unsigned int data_len[2]; 180 u32 inl_mask; 181 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 182 OP_ALG_AAI_CTR_MOD128); 183 const bool is_rfc3686 = alg->caam.rfc3686; 184 185 if (!ctx->cdata.keylen || !ctx->authsize) 186 return 0; 187 188 /* 189 * AES-CTR needs to load IV in CONTEXT1 reg 190 * at an offset of 128bits (16bytes) 191 * CONTEXT1[255:128] = IV 192 */ 193 if (ctr_mode) 194 ctx1_iv_off = 16; 195 196 /* 197 * RFC3686 specific: 198 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 199 */ 200 if (is_rfc3686) { 201 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 202 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 203 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 204 } 205 206 /* 207 * In case |user key| > |derived key|, using DKP<imm,imm> would result 208 * in invalid opcodes (last bytes of user key) in the resulting 209 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 210 * addresses are needed. 211 */ 212 ctx->adata.key_virt = ctx->key; 213 ctx->adata.key_dma = ctx->key_dma; 214 215 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 216 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 217 218 data_len[0] = ctx->adata.keylen_pad; 219 data_len[1] = ctx->cdata.keylen; 220 221 /* aead_encrypt shared descriptor */ 222 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 223 DESC_QI_AEAD_ENC_LEN) + 224 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 225 DESC_JOB_IO_LEN, data_len, &inl_mask, 226 ARRAY_SIZE(data_len)) < 0) 227 return -EINVAL; 228 229 ctx->adata.key_inline = !!(inl_mask & 1); 230 ctx->cdata.key_inline = !!(inl_mask & 2); 231 232 flc = &ctx->flc[ENCRYPT]; 233 desc = flc->sh_desc; 234 235 if (alg->caam.geniv) 236 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 237 ivsize, ctx->authsize, is_rfc3686, 238 nonce, ctx1_iv_off, true, 239 priv->sec_attr.era); 240 else 241 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 242 ivsize, ctx->authsize, is_rfc3686, nonce, 243 ctx1_iv_off, true, priv->sec_attr.era); 244 245 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 246 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 247 sizeof(flc->flc) + desc_bytes(desc), 248 ctx->dir); 249 250 /* aead_decrypt shared descriptor */ 251 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 252 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 253 DESC_JOB_IO_LEN, data_len, &inl_mask, 254 ARRAY_SIZE(data_len)) < 0) 255 return -EINVAL; 256 257 ctx->adata.key_inline = !!(inl_mask & 1); 258 ctx->cdata.key_inline = !!(inl_mask & 2); 259 260 flc = &ctx->flc[DECRYPT]; 261 desc = flc->sh_desc; 262 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 263 ivsize, ctx->authsize, alg->caam.geniv, 264 is_rfc3686, nonce, ctx1_iv_off, true, 265 priv->sec_attr.era); 266 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 267 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 268 sizeof(flc->flc) + desc_bytes(desc), 269 ctx->dir); 270 271 return 0; 272 } 273 274 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 275 { 276 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 277 278 ctx->authsize = authsize; 279 aead_set_sh_desc(authenc); 280 281 return 0; 282 } 283 284 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 285 unsigned int keylen) 286 { 287 struct caam_ctx *ctx = crypto_aead_ctx(aead); 288 struct device *dev = ctx->dev; 289 struct crypto_authenc_keys keys; 290 291 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 292 goto badkey; 293 294 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 295 keys.authkeylen + keys.enckeylen, keys.enckeylen, 296 keys.authkeylen); 297 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 298 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 299 300 ctx->adata.keylen = keys.authkeylen; 301 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 302 OP_ALG_ALGSEL_MASK); 303 304 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 305 goto badkey; 306 307 memcpy(ctx->key, keys.authkey, keys.authkeylen); 308 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 309 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 310 keys.enckeylen, ctx->dir); 311 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 312 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 313 ctx->adata.keylen_pad + keys.enckeylen, 1); 314 315 ctx->cdata.keylen = keys.enckeylen; 316 317 memzero_explicit(&keys, sizeof(keys)); 318 return aead_set_sh_desc(aead); 319 badkey: 320 memzero_explicit(&keys, sizeof(keys)); 321 return -EINVAL; 322 } 323 324 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 325 unsigned int keylen) 326 { 327 struct crypto_authenc_keys keys; 328 int err; 329 330 err = crypto_authenc_extractkeys(&keys, key, keylen); 331 if (unlikely(err)) 332 goto out; 333 334 err = -EINVAL; 335 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 336 goto out; 337 338 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 339 aead_setkey(aead, key, keylen); 340 341 out: 342 memzero_explicit(&keys, sizeof(keys)); 343 return err; 344 } 345 346 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 347 bool encrypt) 348 { 349 struct crypto_aead *aead = crypto_aead_reqtfm(req); 350 struct caam_request *req_ctx = aead_request_ctx(req); 351 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 352 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 353 struct caam_ctx *ctx = crypto_aead_ctx(aead); 354 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 355 typeof(*alg), aead); 356 struct device *dev = ctx->dev; 357 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 358 GFP_KERNEL : GFP_ATOMIC; 359 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 360 int src_len, dst_len = 0; 361 struct aead_edesc *edesc; 362 dma_addr_t qm_sg_dma, iv_dma = 0; 363 int ivsize = 0; 364 unsigned int authsize = ctx->authsize; 365 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 366 int in_len, out_len; 367 struct dpaa2_sg_entry *sg_table; 368 369 /* allocate space for base edesc, link tables and IV */ 370 edesc = qi_cache_zalloc(GFP_DMA | flags); 371 if (unlikely(!edesc)) { 372 dev_err(dev, "could not allocate extended descriptor\n"); 373 return ERR_PTR(-ENOMEM); 374 } 375 376 if (unlikely(req->dst != req->src)) { 377 src_len = req->assoclen + req->cryptlen; 378 dst_len = src_len + (encrypt ? authsize : (-authsize)); 379 380 src_nents = sg_nents_for_len(req->src, src_len); 381 if (unlikely(src_nents < 0)) { 382 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 383 src_len); 384 qi_cache_free(edesc); 385 return ERR_PTR(src_nents); 386 } 387 388 dst_nents = sg_nents_for_len(req->dst, dst_len); 389 if (unlikely(dst_nents < 0)) { 390 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 391 dst_len); 392 qi_cache_free(edesc); 393 return ERR_PTR(dst_nents); 394 } 395 396 if (src_nents) { 397 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 398 DMA_TO_DEVICE); 399 if (unlikely(!mapped_src_nents)) { 400 dev_err(dev, "unable to map source\n"); 401 qi_cache_free(edesc); 402 return ERR_PTR(-ENOMEM); 403 } 404 } else { 405 mapped_src_nents = 0; 406 } 407 408 if (dst_nents) { 409 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 410 DMA_FROM_DEVICE); 411 if (unlikely(!mapped_dst_nents)) { 412 dev_err(dev, "unable to map destination\n"); 413 dma_unmap_sg(dev, req->src, src_nents, 414 DMA_TO_DEVICE); 415 qi_cache_free(edesc); 416 return ERR_PTR(-ENOMEM); 417 } 418 } else { 419 mapped_dst_nents = 0; 420 } 421 } else { 422 src_len = req->assoclen + req->cryptlen + 423 (encrypt ? authsize : 0); 424 425 src_nents = sg_nents_for_len(req->src, src_len); 426 if (unlikely(src_nents < 0)) { 427 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 428 src_len); 429 qi_cache_free(edesc); 430 return ERR_PTR(src_nents); 431 } 432 433 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 434 DMA_BIDIRECTIONAL); 435 if (unlikely(!mapped_src_nents)) { 436 dev_err(dev, "unable to map source\n"); 437 qi_cache_free(edesc); 438 return ERR_PTR(-ENOMEM); 439 } 440 } 441 442 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 443 ivsize = crypto_aead_ivsize(aead); 444 445 /* 446 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 447 * Input is not contiguous. 448 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 449 * the end of the table by allocating more S/G entries. Logic: 450 * if (src != dst && output S/G) 451 * pad output S/G, if needed 452 * else if (src == dst && S/G) 453 * overlapping S/Gs; pad one of them 454 * else if (input S/G) ... 455 * pad input S/G, if needed 456 */ 457 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 458 if (mapped_dst_nents > 1) 459 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 460 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 461 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 462 1 + !!ivsize + 463 pad_sg_nents(mapped_src_nents)); 464 else 465 qm_sg_nents = pad_sg_nents(qm_sg_nents); 466 467 sg_table = &edesc->sgt[0]; 468 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 469 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 470 CAAM_QI_MEMCACHE_SIZE)) { 471 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 472 qm_sg_nents, ivsize); 473 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 474 0, DMA_NONE, 0, 0); 475 qi_cache_free(edesc); 476 return ERR_PTR(-ENOMEM); 477 } 478 479 if (ivsize) { 480 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 481 482 /* Make sure IV is located in a DMAable area */ 483 memcpy(iv, req->iv, ivsize); 484 485 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 486 if (dma_mapping_error(dev, iv_dma)) { 487 dev_err(dev, "unable to map IV\n"); 488 caam_unmap(dev, req->src, req->dst, src_nents, 489 dst_nents, 0, 0, DMA_NONE, 0, 0); 490 qi_cache_free(edesc); 491 return ERR_PTR(-ENOMEM); 492 } 493 } 494 495 edesc->src_nents = src_nents; 496 edesc->dst_nents = dst_nents; 497 edesc->iv_dma = iv_dma; 498 499 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 500 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 501 /* 502 * The associated data comes already with the IV but we need 503 * to skip it when we authenticate or encrypt... 504 */ 505 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 506 else 507 edesc->assoclen = cpu_to_caam32(req->assoclen); 508 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 509 DMA_TO_DEVICE); 510 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 511 dev_err(dev, "unable to map assoclen\n"); 512 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 513 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 514 qi_cache_free(edesc); 515 return ERR_PTR(-ENOMEM); 516 } 517 518 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 519 qm_sg_index++; 520 if (ivsize) { 521 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 522 qm_sg_index++; 523 } 524 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 525 qm_sg_index += mapped_src_nents; 526 527 if (mapped_dst_nents > 1) 528 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 529 530 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 531 if (dma_mapping_error(dev, qm_sg_dma)) { 532 dev_err(dev, "unable to map S/G table\n"); 533 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 534 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 535 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 536 qi_cache_free(edesc); 537 return ERR_PTR(-ENOMEM); 538 } 539 540 edesc->qm_sg_dma = qm_sg_dma; 541 edesc->qm_sg_bytes = qm_sg_bytes; 542 543 out_len = req->assoclen + req->cryptlen + 544 (encrypt ? ctx->authsize : (-ctx->authsize)); 545 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 546 547 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 548 dpaa2_fl_set_final(in_fle, true); 549 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 550 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 551 dpaa2_fl_set_len(in_fle, in_len); 552 553 if (req->dst == req->src) { 554 if (mapped_src_nents == 1) { 555 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 556 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 557 } else { 558 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 559 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 560 (1 + !!ivsize) * sizeof(*sg_table)); 561 } 562 } else if (!mapped_dst_nents) { 563 /* 564 * crypto engine requires the output entry to be present when 565 * "frame list" FD is used. 566 * Since engine does not support FMT=2'b11 (unused entry type), 567 * leaving out_fle zeroized is the best option. 568 */ 569 goto skip_out_fle; 570 } else if (mapped_dst_nents == 1) { 571 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 572 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 573 } else { 574 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 575 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 576 sizeof(*sg_table)); 577 } 578 579 dpaa2_fl_set_len(out_fle, out_len); 580 581 skip_out_fle: 582 return edesc; 583 } 584 585 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 586 { 587 struct caam_ctx *ctx = crypto_aead_ctx(aead); 588 unsigned int ivsize = crypto_aead_ivsize(aead); 589 struct device *dev = ctx->dev; 590 struct caam_flc *flc; 591 u32 *desc; 592 593 if (!ctx->cdata.keylen || !ctx->authsize) 594 return 0; 595 596 flc = &ctx->flc[ENCRYPT]; 597 desc = flc->sh_desc; 598 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 599 ctx->authsize, true, true); 600 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 601 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 602 sizeof(flc->flc) + desc_bytes(desc), 603 ctx->dir); 604 605 flc = &ctx->flc[DECRYPT]; 606 desc = flc->sh_desc; 607 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 608 ctx->authsize, false, true); 609 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 610 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 611 sizeof(flc->flc) + desc_bytes(desc), 612 ctx->dir); 613 614 return 0; 615 } 616 617 static int chachapoly_setauthsize(struct crypto_aead *aead, 618 unsigned int authsize) 619 { 620 struct caam_ctx *ctx = crypto_aead_ctx(aead); 621 622 if (authsize != POLY1305_DIGEST_SIZE) 623 return -EINVAL; 624 625 ctx->authsize = authsize; 626 return chachapoly_set_sh_desc(aead); 627 } 628 629 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 630 unsigned int keylen) 631 { 632 struct caam_ctx *ctx = crypto_aead_ctx(aead); 633 unsigned int ivsize = crypto_aead_ivsize(aead); 634 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 635 636 if (keylen != CHACHA_KEY_SIZE + saltlen) 637 return -EINVAL; 638 639 ctx->cdata.key_virt = key; 640 ctx->cdata.keylen = keylen - saltlen; 641 642 return chachapoly_set_sh_desc(aead); 643 } 644 645 static int gcm_set_sh_desc(struct crypto_aead *aead) 646 { 647 struct caam_ctx *ctx = crypto_aead_ctx(aead); 648 struct device *dev = ctx->dev; 649 unsigned int ivsize = crypto_aead_ivsize(aead); 650 struct caam_flc *flc; 651 u32 *desc; 652 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 653 ctx->cdata.keylen; 654 655 if (!ctx->cdata.keylen || !ctx->authsize) 656 return 0; 657 658 /* 659 * AES GCM encrypt shared descriptor 660 * Job Descriptor and Shared Descriptor 661 * must fit into the 64-word Descriptor h/w Buffer 662 */ 663 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 664 ctx->cdata.key_inline = true; 665 ctx->cdata.key_virt = ctx->key; 666 } else { 667 ctx->cdata.key_inline = false; 668 ctx->cdata.key_dma = ctx->key_dma; 669 } 670 671 flc = &ctx->flc[ENCRYPT]; 672 desc = flc->sh_desc; 673 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 674 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 675 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 676 sizeof(flc->flc) + desc_bytes(desc), 677 ctx->dir); 678 679 /* 680 * Job Descriptor and Shared Descriptors 681 * must all fit into the 64-word Descriptor h/w Buffer 682 */ 683 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 684 ctx->cdata.key_inline = true; 685 ctx->cdata.key_virt = ctx->key; 686 } else { 687 ctx->cdata.key_inline = false; 688 ctx->cdata.key_dma = ctx->key_dma; 689 } 690 691 flc = &ctx->flc[DECRYPT]; 692 desc = flc->sh_desc; 693 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 694 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 695 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 696 sizeof(flc->flc) + desc_bytes(desc), 697 ctx->dir); 698 699 return 0; 700 } 701 702 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 703 { 704 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 705 int err; 706 707 err = crypto_gcm_check_authsize(authsize); 708 if (err) 709 return err; 710 711 ctx->authsize = authsize; 712 gcm_set_sh_desc(authenc); 713 714 return 0; 715 } 716 717 static int gcm_setkey(struct crypto_aead *aead, 718 const u8 *key, unsigned int keylen) 719 { 720 struct caam_ctx *ctx = crypto_aead_ctx(aead); 721 struct device *dev = ctx->dev; 722 int ret; 723 724 ret = aes_check_keylen(keylen); 725 if (ret) 726 return ret; 727 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 728 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 729 730 memcpy(ctx->key, key, keylen); 731 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 732 ctx->cdata.keylen = keylen; 733 734 return gcm_set_sh_desc(aead); 735 } 736 737 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 738 { 739 struct caam_ctx *ctx = crypto_aead_ctx(aead); 740 struct device *dev = ctx->dev; 741 unsigned int ivsize = crypto_aead_ivsize(aead); 742 struct caam_flc *flc; 743 u32 *desc; 744 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 745 ctx->cdata.keylen; 746 747 if (!ctx->cdata.keylen || !ctx->authsize) 748 return 0; 749 750 ctx->cdata.key_virt = ctx->key; 751 752 /* 753 * RFC4106 encrypt shared descriptor 754 * Job Descriptor and Shared Descriptor 755 * must fit into the 64-word Descriptor h/w Buffer 756 */ 757 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 758 ctx->cdata.key_inline = true; 759 } else { 760 ctx->cdata.key_inline = false; 761 ctx->cdata.key_dma = ctx->key_dma; 762 } 763 764 flc = &ctx->flc[ENCRYPT]; 765 desc = flc->sh_desc; 766 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 767 true); 768 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 769 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 770 sizeof(flc->flc) + desc_bytes(desc), 771 ctx->dir); 772 773 /* 774 * Job Descriptor and Shared Descriptors 775 * must all fit into the 64-word Descriptor h/w Buffer 776 */ 777 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 778 ctx->cdata.key_inline = true; 779 } else { 780 ctx->cdata.key_inline = false; 781 ctx->cdata.key_dma = ctx->key_dma; 782 } 783 784 flc = &ctx->flc[DECRYPT]; 785 desc = flc->sh_desc; 786 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 787 true); 788 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 789 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 790 sizeof(flc->flc) + desc_bytes(desc), 791 ctx->dir); 792 793 return 0; 794 } 795 796 static int rfc4106_setauthsize(struct crypto_aead *authenc, 797 unsigned int authsize) 798 { 799 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 800 int err; 801 802 err = crypto_rfc4106_check_authsize(authsize); 803 if (err) 804 return err; 805 806 ctx->authsize = authsize; 807 rfc4106_set_sh_desc(authenc); 808 809 return 0; 810 } 811 812 static int rfc4106_setkey(struct crypto_aead *aead, 813 const u8 *key, unsigned int keylen) 814 { 815 struct caam_ctx *ctx = crypto_aead_ctx(aead); 816 struct device *dev = ctx->dev; 817 int ret; 818 819 ret = aes_check_keylen(keylen - 4); 820 if (ret) 821 return ret; 822 823 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 824 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 825 826 memcpy(ctx->key, key, keylen); 827 /* 828 * The last four bytes of the key material are used as the salt value 829 * in the nonce. Update the AES key length. 830 */ 831 ctx->cdata.keylen = keylen - 4; 832 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 833 ctx->dir); 834 835 return rfc4106_set_sh_desc(aead); 836 } 837 838 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 839 { 840 struct caam_ctx *ctx = crypto_aead_ctx(aead); 841 struct device *dev = ctx->dev; 842 unsigned int ivsize = crypto_aead_ivsize(aead); 843 struct caam_flc *flc; 844 u32 *desc; 845 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 846 ctx->cdata.keylen; 847 848 if (!ctx->cdata.keylen || !ctx->authsize) 849 return 0; 850 851 ctx->cdata.key_virt = ctx->key; 852 853 /* 854 * RFC4543 encrypt shared descriptor 855 * Job Descriptor and Shared Descriptor 856 * must fit into the 64-word Descriptor h/w Buffer 857 */ 858 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 859 ctx->cdata.key_inline = true; 860 } else { 861 ctx->cdata.key_inline = false; 862 ctx->cdata.key_dma = ctx->key_dma; 863 } 864 865 flc = &ctx->flc[ENCRYPT]; 866 desc = flc->sh_desc; 867 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 868 true); 869 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 870 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 871 sizeof(flc->flc) + desc_bytes(desc), 872 ctx->dir); 873 874 /* 875 * Job Descriptor and Shared Descriptors 876 * must all fit into the 64-word Descriptor h/w Buffer 877 */ 878 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 879 ctx->cdata.key_inline = true; 880 } else { 881 ctx->cdata.key_inline = false; 882 ctx->cdata.key_dma = ctx->key_dma; 883 } 884 885 flc = &ctx->flc[DECRYPT]; 886 desc = flc->sh_desc; 887 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 888 true); 889 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 890 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 891 sizeof(flc->flc) + desc_bytes(desc), 892 ctx->dir); 893 894 return 0; 895 } 896 897 static int rfc4543_setauthsize(struct crypto_aead *authenc, 898 unsigned int authsize) 899 { 900 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 901 902 if (authsize != 16) 903 return -EINVAL; 904 905 ctx->authsize = authsize; 906 rfc4543_set_sh_desc(authenc); 907 908 return 0; 909 } 910 911 static int rfc4543_setkey(struct crypto_aead *aead, 912 const u8 *key, unsigned int keylen) 913 { 914 struct caam_ctx *ctx = crypto_aead_ctx(aead); 915 struct device *dev = ctx->dev; 916 int ret; 917 918 ret = aes_check_keylen(keylen - 4); 919 if (ret) 920 return ret; 921 922 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 923 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 924 925 memcpy(ctx->key, key, keylen); 926 /* 927 * The last four bytes of the key material are used as the salt value 928 * in the nonce. Update the AES key length. 929 */ 930 ctx->cdata.keylen = keylen - 4; 931 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 932 ctx->dir); 933 934 return rfc4543_set_sh_desc(aead); 935 } 936 937 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 938 unsigned int keylen, const u32 ctx1_iv_off) 939 { 940 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 941 struct caam_skcipher_alg *alg = 942 container_of(crypto_skcipher_alg(skcipher), 943 struct caam_skcipher_alg, skcipher); 944 struct device *dev = ctx->dev; 945 struct caam_flc *flc; 946 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 947 u32 *desc; 948 const bool is_rfc3686 = alg->caam.rfc3686; 949 950 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 951 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 952 953 ctx->cdata.keylen = keylen; 954 ctx->cdata.key_virt = key; 955 ctx->cdata.key_inline = true; 956 957 /* skcipher_encrypt shared descriptor */ 958 flc = &ctx->flc[ENCRYPT]; 959 desc = flc->sh_desc; 960 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 961 ctx1_iv_off); 962 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 963 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 964 sizeof(flc->flc) + desc_bytes(desc), 965 ctx->dir); 966 967 /* skcipher_decrypt shared descriptor */ 968 flc = &ctx->flc[DECRYPT]; 969 desc = flc->sh_desc; 970 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 971 ctx1_iv_off); 972 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 973 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 974 sizeof(flc->flc) + desc_bytes(desc), 975 ctx->dir); 976 977 return 0; 978 } 979 980 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 981 const u8 *key, unsigned int keylen) 982 { 983 int err; 984 985 err = aes_check_keylen(keylen); 986 if (err) 987 return err; 988 989 return skcipher_setkey(skcipher, key, keylen, 0); 990 } 991 992 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 993 const u8 *key, unsigned int keylen) 994 { 995 u32 ctx1_iv_off; 996 int err; 997 998 /* 999 * RFC3686 specific: 1000 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1001 * | *key = {KEY, NONCE} 1002 */ 1003 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1004 keylen -= CTR_RFC3686_NONCE_SIZE; 1005 1006 err = aes_check_keylen(keylen); 1007 if (err) 1008 return err; 1009 1010 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1011 } 1012 1013 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1014 const u8 *key, unsigned int keylen) 1015 { 1016 u32 ctx1_iv_off; 1017 int err; 1018 1019 /* 1020 * AES-CTR needs to load IV in CONTEXT1 reg 1021 * at an offset of 128bits (16bytes) 1022 * CONTEXT1[255:128] = IV 1023 */ 1024 ctx1_iv_off = 16; 1025 1026 err = aes_check_keylen(keylen); 1027 if (err) 1028 return err; 1029 1030 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1031 } 1032 1033 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1034 const u8 *key, unsigned int keylen) 1035 { 1036 if (keylen != CHACHA_KEY_SIZE) 1037 return -EINVAL; 1038 1039 return skcipher_setkey(skcipher, key, keylen, 0); 1040 } 1041 1042 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1043 const u8 *key, unsigned int keylen) 1044 { 1045 return verify_skcipher_des_key(skcipher, key) ?: 1046 skcipher_setkey(skcipher, key, keylen, 0); 1047 } 1048 1049 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1050 const u8 *key, unsigned int keylen) 1051 { 1052 return verify_skcipher_des3_key(skcipher, key) ?: 1053 skcipher_setkey(skcipher, key, keylen, 0); 1054 } 1055 1056 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1057 unsigned int keylen) 1058 { 1059 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1060 struct device *dev = ctx->dev; 1061 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 1062 struct caam_flc *flc; 1063 u32 *desc; 1064 int err; 1065 1066 err = xts_verify_key(skcipher, key, keylen); 1067 if (err) { 1068 dev_dbg(dev, "key size mismatch\n"); 1069 return err; 1070 } 1071 1072 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 1073 ctx->xts_key_fallback = true; 1074 1075 if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { 1076 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 1077 if (err) 1078 return err; 1079 } 1080 1081 ctx->cdata.keylen = keylen; 1082 ctx->cdata.key_virt = key; 1083 ctx->cdata.key_inline = true; 1084 1085 /* xts_skcipher_encrypt shared descriptor */ 1086 flc = &ctx->flc[ENCRYPT]; 1087 desc = flc->sh_desc; 1088 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1089 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1090 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1091 sizeof(flc->flc) + desc_bytes(desc), 1092 ctx->dir); 1093 1094 /* xts_skcipher_decrypt shared descriptor */ 1095 flc = &ctx->flc[DECRYPT]; 1096 desc = flc->sh_desc; 1097 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1098 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1099 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1100 sizeof(flc->flc) + desc_bytes(desc), 1101 ctx->dir); 1102 1103 return 0; 1104 } 1105 1106 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1107 { 1108 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1109 struct caam_request *req_ctx = skcipher_request_ctx(req); 1110 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1111 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1112 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1113 struct device *dev = ctx->dev; 1114 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1115 GFP_KERNEL : GFP_ATOMIC; 1116 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1117 struct skcipher_edesc *edesc; 1118 dma_addr_t iv_dma; 1119 u8 *iv; 1120 int ivsize = crypto_skcipher_ivsize(skcipher); 1121 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1122 struct dpaa2_sg_entry *sg_table; 1123 1124 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1125 if (unlikely(src_nents < 0)) { 1126 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1127 req->cryptlen); 1128 return ERR_PTR(src_nents); 1129 } 1130 1131 if (unlikely(req->dst != req->src)) { 1132 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1133 if (unlikely(dst_nents < 0)) { 1134 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1135 req->cryptlen); 1136 return ERR_PTR(dst_nents); 1137 } 1138 1139 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1140 DMA_TO_DEVICE); 1141 if (unlikely(!mapped_src_nents)) { 1142 dev_err(dev, "unable to map source\n"); 1143 return ERR_PTR(-ENOMEM); 1144 } 1145 1146 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1147 DMA_FROM_DEVICE); 1148 if (unlikely(!mapped_dst_nents)) { 1149 dev_err(dev, "unable to map destination\n"); 1150 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1151 return ERR_PTR(-ENOMEM); 1152 } 1153 } else { 1154 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1155 DMA_BIDIRECTIONAL); 1156 if (unlikely(!mapped_src_nents)) { 1157 dev_err(dev, "unable to map source\n"); 1158 return ERR_PTR(-ENOMEM); 1159 } 1160 } 1161 1162 qm_sg_ents = 1 + mapped_src_nents; 1163 dst_sg_idx = qm_sg_ents; 1164 1165 /* 1166 * Input, output HW S/G tables: [IV, src][dst, IV] 1167 * IV entries point to the same buffer 1168 * If src == dst, S/G entries are reused (S/G tables overlap) 1169 * 1170 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1171 * the end of the table by allocating more S/G entries. 1172 */ 1173 if (req->src != req->dst) 1174 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1175 else 1176 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1177 1178 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1179 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1180 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1181 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1182 qm_sg_ents, ivsize); 1183 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1184 0, DMA_NONE, 0, 0); 1185 return ERR_PTR(-ENOMEM); 1186 } 1187 1188 /* allocate space for base edesc, link tables and IV */ 1189 edesc = qi_cache_zalloc(GFP_DMA | flags); 1190 if (unlikely(!edesc)) { 1191 dev_err(dev, "could not allocate extended descriptor\n"); 1192 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1193 0, DMA_NONE, 0, 0); 1194 return ERR_PTR(-ENOMEM); 1195 } 1196 1197 /* Make sure IV is located in a DMAable area */ 1198 sg_table = &edesc->sgt[0]; 1199 iv = (u8 *)(sg_table + qm_sg_ents); 1200 memcpy(iv, req->iv, ivsize); 1201 1202 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1203 if (dma_mapping_error(dev, iv_dma)) { 1204 dev_err(dev, "unable to map IV\n"); 1205 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1206 0, DMA_NONE, 0, 0); 1207 qi_cache_free(edesc); 1208 return ERR_PTR(-ENOMEM); 1209 } 1210 1211 edesc->src_nents = src_nents; 1212 edesc->dst_nents = dst_nents; 1213 edesc->iv_dma = iv_dma; 1214 edesc->qm_sg_bytes = qm_sg_bytes; 1215 1216 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1217 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1218 1219 if (req->src != req->dst) 1220 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1221 1222 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1223 ivsize, 0); 1224 1225 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1226 DMA_TO_DEVICE); 1227 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1228 dev_err(dev, "unable to map S/G table\n"); 1229 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1230 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1231 qi_cache_free(edesc); 1232 return ERR_PTR(-ENOMEM); 1233 } 1234 1235 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1236 dpaa2_fl_set_final(in_fle, true); 1237 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1238 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1239 1240 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1241 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1242 1243 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1244 1245 if (req->src == req->dst) 1246 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1247 sizeof(*sg_table)); 1248 else 1249 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1250 sizeof(*sg_table)); 1251 1252 return edesc; 1253 } 1254 1255 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1256 struct aead_request *req) 1257 { 1258 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1259 int ivsize = crypto_aead_ivsize(aead); 1260 1261 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1262 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1263 edesc->qm_sg_bytes); 1264 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1265 } 1266 1267 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1268 struct skcipher_request *req) 1269 { 1270 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1271 int ivsize = crypto_skcipher_ivsize(skcipher); 1272 1273 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1274 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1275 edesc->qm_sg_bytes); 1276 } 1277 1278 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1279 { 1280 struct crypto_async_request *areq = cbk_ctx; 1281 struct aead_request *req = container_of(areq, struct aead_request, 1282 base); 1283 struct caam_request *req_ctx = to_caam_req(areq); 1284 struct aead_edesc *edesc = req_ctx->edesc; 1285 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1286 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1287 int ecode = 0; 1288 1289 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1290 1291 if (unlikely(status)) 1292 ecode = caam_qi2_strstatus(ctx->dev, status); 1293 1294 aead_unmap(ctx->dev, edesc, req); 1295 qi_cache_free(edesc); 1296 aead_request_complete(req, ecode); 1297 } 1298 1299 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1300 { 1301 struct crypto_async_request *areq = cbk_ctx; 1302 struct aead_request *req = container_of(areq, struct aead_request, 1303 base); 1304 struct caam_request *req_ctx = to_caam_req(areq); 1305 struct aead_edesc *edesc = req_ctx->edesc; 1306 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1307 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1308 int ecode = 0; 1309 1310 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1311 1312 if (unlikely(status)) 1313 ecode = caam_qi2_strstatus(ctx->dev, status); 1314 1315 aead_unmap(ctx->dev, edesc, req); 1316 qi_cache_free(edesc); 1317 aead_request_complete(req, ecode); 1318 } 1319 1320 static int aead_encrypt(struct aead_request *req) 1321 { 1322 struct aead_edesc *edesc; 1323 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1324 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1325 struct caam_request *caam_req = aead_request_ctx(req); 1326 int ret; 1327 1328 /* allocate extended descriptor */ 1329 edesc = aead_edesc_alloc(req, true); 1330 if (IS_ERR(edesc)) 1331 return PTR_ERR(edesc); 1332 1333 caam_req->flc = &ctx->flc[ENCRYPT]; 1334 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1335 caam_req->cbk = aead_encrypt_done; 1336 caam_req->ctx = &req->base; 1337 caam_req->edesc = edesc; 1338 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1339 if (ret != -EINPROGRESS && 1340 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1341 aead_unmap(ctx->dev, edesc, req); 1342 qi_cache_free(edesc); 1343 } 1344 1345 return ret; 1346 } 1347 1348 static int aead_decrypt(struct aead_request *req) 1349 { 1350 struct aead_edesc *edesc; 1351 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1352 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1353 struct caam_request *caam_req = aead_request_ctx(req); 1354 int ret; 1355 1356 /* allocate extended descriptor */ 1357 edesc = aead_edesc_alloc(req, false); 1358 if (IS_ERR(edesc)) 1359 return PTR_ERR(edesc); 1360 1361 caam_req->flc = &ctx->flc[DECRYPT]; 1362 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1363 caam_req->cbk = aead_decrypt_done; 1364 caam_req->ctx = &req->base; 1365 caam_req->edesc = edesc; 1366 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1367 if (ret != -EINPROGRESS && 1368 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1369 aead_unmap(ctx->dev, edesc, req); 1370 qi_cache_free(edesc); 1371 } 1372 1373 return ret; 1374 } 1375 1376 static int ipsec_gcm_encrypt(struct aead_request *req) 1377 { 1378 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1379 } 1380 1381 static int ipsec_gcm_decrypt(struct aead_request *req) 1382 { 1383 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1384 } 1385 1386 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1387 { 1388 struct crypto_async_request *areq = cbk_ctx; 1389 struct skcipher_request *req = skcipher_request_cast(areq); 1390 struct caam_request *req_ctx = to_caam_req(areq); 1391 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1392 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1393 struct skcipher_edesc *edesc = req_ctx->edesc; 1394 int ecode = 0; 1395 int ivsize = crypto_skcipher_ivsize(skcipher); 1396 1397 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1398 1399 if (unlikely(status)) 1400 ecode = caam_qi2_strstatus(ctx->dev, status); 1401 1402 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1403 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1404 edesc->src_nents > 1 ? 100 : ivsize, 1); 1405 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1406 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1407 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1408 1409 skcipher_unmap(ctx->dev, edesc, req); 1410 1411 /* 1412 * The crypto API expects us to set the IV (req->iv) to the last 1413 * ciphertext block (CBC mode) or last counter (CTR mode). 1414 * This is used e.g. by the CTS mode. 1415 */ 1416 if (!ecode) 1417 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1418 ivsize); 1419 1420 qi_cache_free(edesc); 1421 skcipher_request_complete(req, ecode); 1422 } 1423 1424 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1425 { 1426 struct crypto_async_request *areq = cbk_ctx; 1427 struct skcipher_request *req = skcipher_request_cast(areq); 1428 struct caam_request *req_ctx = to_caam_req(areq); 1429 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1430 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1431 struct skcipher_edesc *edesc = req_ctx->edesc; 1432 int ecode = 0; 1433 int ivsize = crypto_skcipher_ivsize(skcipher); 1434 1435 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1436 1437 if (unlikely(status)) 1438 ecode = caam_qi2_strstatus(ctx->dev, status); 1439 1440 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1441 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1442 edesc->src_nents > 1 ? 100 : ivsize, 1); 1443 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1444 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1445 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1446 1447 skcipher_unmap(ctx->dev, edesc, req); 1448 1449 /* 1450 * The crypto API expects us to set the IV (req->iv) to the last 1451 * ciphertext block (CBC mode) or last counter (CTR mode). 1452 * This is used e.g. by the CTS mode. 1453 */ 1454 if (!ecode) 1455 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1456 ivsize); 1457 1458 qi_cache_free(edesc); 1459 skcipher_request_complete(req, ecode); 1460 } 1461 1462 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1463 { 1464 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1465 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1466 1467 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1468 } 1469 1470 static int skcipher_encrypt(struct skcipher_request *req) 1471 { 1472 struct skcipher_edesc *edesc; 1473 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1474 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1475 struct caam_request *caam_req = skcipher_request_ctx(req); 1476 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1477 int ret; 1478 1479 /* 1480 * XTS is expected to return an error even for input length = 0 1481 * Note that the case input length < block size will be caught during 1482 * HW offloading and return an error. 1483 */ 1484 if (!req->cryptlen && !ctx->fallback) 1485 return 0; 1486 1487 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1488 ctx->xts_key_fallback)) { 1489 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1490 skcipher_request_set_callback(&caam_req->fallback_req, 1491 req->base.flags, 1492 req->base.complete, 1493 req->base.data); 1494 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1495 req->dst, req->cryptlen, req->iv); 1496 1497 return crypto_skcipher_encrypt(&caam_req->fallback_req); 1498 } 1499 1500 /* allocate extended descriptor */ 1501 edesc = skcipher_edesc_alloc(req); 1502 if (IS_ERR(edesc)) 1503 return PTR_ERR(edesc); 1504 1505 caam_req->flc = &ctx->flc[ENCRYPT]; 1506 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1507 caam_req->cbk = skcipher_encrypt_done; 1508 caam_req->ctx = &req->base; 1509 caam_req->edesc = edesc; 1510 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1511 if (ret != -EINPROGRESS && 1512 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1513 skcipher_unmap(ctx->dev, edesc, req); 1514 qi_cache_free(edesc); 1515 } 1516 1517 return ret; 1518 } 1519 1520 static int skcipher_decrypt(struct skcipher_request *req) 1521 { 1522 struct skcipher_edesc *edesc; 1523 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1524 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1525 struct caam_request *caam_req = skcipher_request_ctx(req); 1526 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1527 int ret; 1528 1529 /* 1530 * XTS is expected to return an error even for input length = 0 1531 * Note that the case input length < block size will be caught during 1532 * HW offloading and return an error. 1533 */ 1534 if (!req->cryptlen && !ctx->fallback) 1535 return 0; 1536 1537 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1538 ctx->xts_key_fallback)) { 1539 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1540 skcipher_request_set_callback(&caam_req->fallback_req, 1541 req->base.flags, 1542 req->base.complete, 1543 req->base.data); 1544 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1545 req->dst, req->cryptlen, req->iv); 1546 1547 return crypto_skcipher_decrypt(&caam_req->fallback_req); 1548 } 1549 1550 /* allocate extended descriptor */ 1551 edesc = skcipher_edesc_alloc(req); 1552 if (IS_ERR(edesc)) 1553 return PTR_ERR(edesc); 1554 1555 caam_req->flc = &ctx->flc[DECRYPT]; 1556 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1557 caam_req->cbk = skcipher_decrypt_done; 1558 caam_req->ctx = &req->base; 1559 caam_req->edesc = edesc; 1560 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1561 if (ret != -EINPROGRESS && 1562 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1563 skcipher_unmap(ctx->dev, edesc, req); 1564 qi_cache_free(edesc); 1565 } 1566 1567 return ret; 1568 } 1569 1570 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1571 bool uses_dkp) 1572 { 1573 dma_addr_t dma_addr; 1574 int i; 1575 1576 /* copy descriptor header template value */ 1577 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1578 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1579 1580 ctx->dev = caam->dev; 1581 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1582 1583 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1584 offsetof(struct caam_ctx, flc_dma), 1585 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1586 if (dma_mapping_error(ctx->dev, dma_addr)) { 1587 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1588 return -ENOMEM; 1589 } 1590 1591 for (i = 0; i < NUM_OP; i++) 1592 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1593 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1594 1595 return 0; 1596 } 1597 1598 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1599 { 1600 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1601 struct caam_skcipher_alg *caam_alg = 1602 container_of(alg, typeof(*caam_alg), skcipher); 1603 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 1604 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 1605 int ret = 0; 1606 1607 if (alg_aai == OP_ALG_AAI_XTS) { 1608 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1609 struct crypto_skcipher *fallback; 1610 1611 fallback = crypto_alloc_skcipher(tfm_name, 0, 1612 CRYPTO_ALG_NEED_FALLBACK); 1613 if (IS_ERR(fallback)) { 1614 dev_err(caam_alg->caam.dev, 1615 "Failed to allocate %s fallback: %ld\n", 1616 tfm_name, PTR_ERR(fallback)); 1617 return PTR_ERR(fallback); 1618 } 1619 1620 ctx->fallback = fallback; 1621 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request) + 1622 crypto_skcipher_reqsize(fallback)); 1623 } else { 1624 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); 1625 } 1626 1627 ret = caam_cra_init(ctx, &caam_alg->caam, false); 1628 if (ret && ctx->fallback) 1629 crypto_free_skcipher(ctx->fallback); 1630 1631 return ret; 1632 } 1633 1634 static int caam_cra_init_aead(struct crypto_aead *tfm) 1635 { 1636 struct aead_alg *alg = crypto_aead_alg(tfm); 1637 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1638 aead); 1639 1640 crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); 1641 return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam, 1642 !caam_alg->caam.nodkp); 1643 } 1644 1645 static void caam_exit_common(struct caam_ctx *ctx) 1646 { 1647 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1648 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1649 DMA_ATTR_SKIP_CPU_SYNC); 1650 } 1651 1652 static void caam_cra_exit(struct crypto_skcipher *tfm) 1653 { 1654 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 1655 1656 if (ctx->fallback) 1657 crypto_free_skcipher(ctx->fallback); 1658 caam_exit_common(ctx); 1659 } 1660 1661 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1662 { 1663 caam_exit_common(crypto_aead_ctx(tfm)); 1664 } 1665 1666 static struct caam_skcipher_alg driver_algs[] = { 1667 { 1668 .skcipher = { 1669 .base = { 1670 .cra_name = "cbc(aes)", 1671 .cra_driver_name = "cbc-aes-caam-qi2", 1672 .cra_blocksize = AES_BLOCK_SIZE, 1673 }, 1674 .setkey = aes_skcipher_setkey, 1675 .encrypt = skcipher_encrypt, 1676 .decrypt = skcipher_decrypt, 1677 .min_keysize = AES_MIN_KEY_SIZE, 1678 .max_keysize = AES_MAX_KEY_SIZE, 1679 .ivsize = AES_BLOCK_SIZE, 1680 }, 1681 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1682 }, 1683 { 1684 .skcipher = { 1685 .base = { 1686 .cra_name = "cbc(des3_ede)", 1687 .cra_driver_name = "cbc-3des-caam-qi2", 1688 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1689 }, 1690 .setkey = des3_skcipher_setkey, 1691 .encrypt = skcipher_encrypt, 1692 .decrypt = skcipher_decrypt, 1693 .min_keysize = DES3_EDE_KEY_SIZE, 1694 .max_keysize = DES3_EDE_KEY_SIZE, 1695 .ivsize = DES3_EDE_BLOCK_SIZE, 1696 }, 1697 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1698 }, 1699 { 1700 .skcipher = { 1701 .base = { 1702 .cra_name = "cbc(des)", 1703 .cra_driver_name = "cbc-des-caam-qi2", 1704 .cra_blocksize = DES_BLOCK_SIZE, 1705 }, 1706 .setkey = des_skcipher_setkey, 1707 .encrypt = skcipher_encrypt, 1708 .decrypt = skcipher_decrypt, 1709 .min_keysize = DES_KEY_SIZE, 1710 .max_keysize = DES_KEY_SIZE, 1711 .ivsize = DES_BLOCK_SIZE, 1712 }, 1713 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1714 }, 1715 { 1716 .skcipher = { 1717 .base = { 1718 .cra_name = "ctr(aes)", 1719 .cra_driver_name = "ctr-aes-caam-qi2", 1720 .cra_blocksize = 1, 1721 }, 1722 .setkey = ctr_skcipher_setkey, 1723 .encrypt = skcipher_encrypt, 1724 .decrypt = skcipher_decrypt, 1725 .min_keysize = AES_MIN_KEY_SIZE, 1726 .max_keysize = AES_MAX_KEY_SIZE, 1727 .ivsize = AES_BLOCK_SIZE, 1728 .chunksize = AES_BLOCK_SIZE, 1729 }, 1730 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1731 OP_ALG_AAI_CTR_MOD128, 1732 }, 1733 { 1734 .skcipher = { 1735 .base = { 1736 .cra_name = "rfc3686(ctr(aes))", 1737 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1738 .cra_blocksize = 1, 1739 }, 1740 .setkey = rfc3686_skcipher_setkey, 1741 .encrypt = skcipher_encrypt, 1742 .decrypt = skcipher_decrypt, 1743 .min_keysize = AES_MIN_KEY_SIZE + 1744 CTR_RFC3686_NONCE_SIZE, 1745 .max_keysize = AES_MAX_KEY_SIZE + 1746 CTR_RFC3686_NONCE_SIZE, 1747 .ivsize = CTR_RFC3686_IV_SIZE, 1748 .chunksize = AES_BLOCK_SIZE, 1749 }, 1750 .caam = { 1751 .class1_alg_type = OP_ALG_ALGSEL_AES | 1752 OP_ALG_AAI_CTR_MOD128, 1753 .rfc3686 = true, 1754 }, 1755 }, 1756 { 1757 .skcipher = { 1758 .base = { 1759 .cra_name = "xts(aes)", 1760 .cra_driver_name = "xts-aes-caam-qi2", 1761 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1762 .cra_blocksize = AES_BLOCK_SIZE, 1763 }, 1764 .setkey = xts_skcipher_setkey, 1765 .encrypt = skcipher_encrypt, 1766 .decrypt = skcipher_decrypt, 1767 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1768 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1769 .ivsize = AES_BLOCK_SIZE, 1770 }, 1771 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1772 }, 1773 { 1774 .skcipher = { 1775 .base = { 1776 .cra_name = "chacha20", 1777 .cra_driver_name = "chacha20-caam-qi2", 1778 .cra_blocksize = 1, 1779 }, 1780 .setkey = chacha20_skcipher_setkey, 1781 .encrypt = skcipher_encrypt, 1782 .decrypt = skcipher_decrypt, 1783 .min_keysize = CHACHA_KEY_SIZE, 1784 .max_keysize = CHACHA_KEY_SIZE, 1785 .ivsize = CHACHA_IV_SIZE, 1786 }, 1787 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1788 }, 1789 }; 1790 1791 static struct caam_aead_alg driver_aeads[] = { 1792 { 1793 .aead = { 1794 .base = { 1795 .cra_name = "rfc4106(gcm(aes))", 1796 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1797 .cra_blocksize = 1, 1798 }, 1799 .setkey = rfc4106_setkey, 1800 .setauthsize = rfc4106_setauthsize, 1801 .encrypt = ipsec_gcm_encrypt, 1802 .decrypt = ipsec_gcm_decrypt, 1803 .ivsize = 8, 1804 .maxauthsize = AES_BLOCK_SIZE, 1805 }, 1806 .caam = { 1807 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1808 .nodkp = true, 1809 }, 1810 }, 1811 { 1812 .aead = { 1813 .base = { 1814 .cra_name = "rfc4543(gcm(aes))", 1815 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1816 .cra_blocksize = 1, 1817 }, 1818 .setkey = rfc4543_setkey, 1819 .setauthsize = rfc4543_setauthsize, 1820 .encrypt = ipsec_gcm_encrypt, 1821 .decrypt = ipsec_gcm_decrypt, 1822 .ivsize = 8, 1823 .maxauthsize = AES_BLOCK_SIZE, 1824 }, 1825 .caam = { 1826 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1827 .nodkp = true, 1828 }, 1829 }, 1830 /* Galois Counter Mode */ 1831 { 1832 .aead = { 1833 .base = { 1834 .cra_name = "gcm(aes)", 1835 .cra_driver_name = "gcm-aes-caam-qi2", 1836 .cra_blocksize = 1, 1837 }, 1838 .setkey = gcm_setkey, 1839 .setauthsize = gcm_setauthsize, 1840 .encrypt = aead_encrypt, 1841 .decrypt = aead_decrypt, 1842 .ivsize = 12, 1843 .maxauthsize = AES_BLOCK_SIZE, 1844 }, 1845 .caam = { 1846 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1847 .nodkp = true, 1848 } 1849 }, 1850 /* single-pass ipsec_esp descriptor */ 1851 { 1852 .aead = { 1853 .base = { 1854 .cra_name = "authenc(hmac(md5),cbc(aes))", 1855 .cra_driver_name = "authenc-hmac-md5-" 1856 "cbc-aes-caam-qi2", 1857 .cra_blocksize = AES_BLOCK_SIZE, 1858 }, 1859 .setkey = aead_setkey, 1860 .setauthsize = aead_setauthsize, 1861 .encrypt = aead_encrypt, 1862 .decrypt = aead_decrypt, 1863 .ivsize = AES_BLOCK_SIZE, 1864 .maxauthsize = MD5_DIGEST_SIZE, 1865 }, 1866 .caam = { 1867 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1868 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1869 OP_ALG_AAI_HMAC_PRECOMP, 1870 } 1871 }, 1872 { 1873 .aead = { 1874 .base = { 1875 .cra_name = "echainiv(authenc(hmac(md5)," 1876 "cbc(aes)))", 1877 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1878 "cbc-aes-caam-qi2", 1879 .cra_blocksize = AES_BLOCK_SIZE, 1880 }, 1881 .setkey = aead_setkey, 1882 .setauthsize = aead_setauthsize, 1883 .encrypt = aead_encrypt, 1884 .decrypt = aead_decrypt, 1885 .ivsize = AES_BLOCK_SIZE, 1886 .maxauthsize = MD5_DIGEST_SIZE, 1887 }, 1888 .caam = { 1889 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1890 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1891 OP_ALG_AAI_HMAC_PRECOMP, 1892 .geniv = true, 1893 } 1894 }, 1895 { 1896 .aead = { 1897 .base = { 1898 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1899 .cra_driver_name = "authenc-hmac-sha1-" 1900 "cbc-aes-caam-qi2", 1901 .cra_blocksize = AES_BLOCK_SIZE, 1902 }, 1903 .setkey = aead_setkey, 1904 .setauthsize = aead_setauthsize, 1905 .encrypt = aead_encrypt, 1906 .decrypt = aead_decrypt, 1907 .ivsize = AES_BLOCK_SIZE, 1908 .maxauthsize = SHA1_DIGEST_SIZE, 1909 }, 1910 .caam = { 1911 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1912 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1913 OP_ALG_AAI_HMAC_PRECOMP, 1914 } 1915 }, 1916 { 1917 .aead = { 1918 .base = { 1919 .cra_name = "echainiv(authenc(hmac(sha1)," 1920 "cbc(aes)))", 1921 .cra_driver_name = "echainiv-authenc-" 1922 "hmac-sha1-cbc-aes-caam-qi2", 1923 .cra_blocksize = AES_BLOCK_SIZE, 1924 }, 1925 .setkey = aead_setkey, 1926 .setauthsize = aead_setauthsize, 1927 .encrypt = aead_encrypt, 1928 .decrypt = aead_decrypt, 1929 .ivsize = AES_BLOCK_SIZE, 1930 .maxauthsize = SHA1_DIGEST_SIZE, 1931 }, 1932 .caam = { 1933 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1934 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1935 OP_ALG_AAI_HMAC_PRECOMP, 1936 .geniv = true, 1937 }, 1938 }, 1939 { 1940 .aead = { 1941 .base = { 1942 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1943 .cra_driver_name = "authenc-hmac-sha224-" 1944 "cbc-aes-caam-qi2", 1945 .cra_blocksize = AES_BLOCK_SIZE, 1946 }, 1947 .setkey = aead_setkey, 1948 .setauthsize = aead_setauthsize, 1949 .encrypt = aead_encrypt, 1950 .decrypt = aead_decrypt, 1951 .ivsize = AES_BLOCK_SIZE, 1952 .maxauthsize = SHA224_DIGEST_SIZE, 1953 }, 1954 .caam = { 1955 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1956 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1957 OP_ALG_AAI_HMAC_PRECOMP, 1958 } 1959 }, 1960 { 1961 .aead = { 1962 .base = { 1963 .cra_name = "echainiv(authenc(hmac(sha224)," 1964 "cbc(aes)))", 1965 .cra_driver_name = "echainiv-authenc-" 1966 "hmac-sha224-cbc-aes-caam-qi2", 1967 .cra_blocksize = AES_BLOCK_SIZE, 1968 }, 1969 .setkey = aead_setkey, 1970 .setauthsize = aead_setauthsize, 1971 .encrypt = aead_encrypt, 1972 .decrypt = aead_decrypt, 1973 .ivsize = AES_BLOCK_SIZE, 1974 .maxauthsize = SHA224_DIGEST_SIZE, 1975 }, 1976 .caam = { 1977 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1978 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1979 OP_ALG_AAI_HMAC_PRECOMP, 1980 .geniv = true, 1981 } 1982 }, 1983 { 1984 .aead = { 1985 .base = { 1986 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1987 .cra_driver_name = "authenc-hmac-sha256-" 1988 "cbc-aes-caam-qi2", 1989 .cra_blocksize = AES_BLOCK_SIZE, 1990 }, 1991 .setkey = aead_setkey, 1992 .setauthsize = aead_setauthsize, 1993 .encrypt = aead_encrypt, 1994 .decrypt = aead_decrypt, 1995 .ivsize = AES_BLOCK_SIZE, 1996 .maxauthsize = SHA256_DIGEST_SIZE, 1997 }, 1998 .caam = { 1999 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2000 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2001 OP_ALG_AAI_HMAC_PRECOMP, 2002 } 2003 }, 2004 { 2005 .aead = { 2006 .base = { 2007 .cra_name = "echainiv(authenc(hmac(sha256)," 2008 "cbc(aes)))", 2009 .cra_driver_name = "echainiv-authenc-" 2010 "hmac-sha256-cbc-aes-" 2011 "caam-qi2", 2012 .cra_blocksize = AES_BLOCK_SIZE, 2013 }, 2014 .setkey = aead_setkey, 2015 .setauthsize = aead_setauthsize, 2016 .encrypt = aead_encrypt, 2017 .decrypt = aead_decrypt, 2018 .ivsize = AES_BLOCK_SIZE, 2019 .maxauthsize = SHA256_DIGEST_SIZE, 2020 }, 2021 .caam = { 2022 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2023 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2024 OP_ALG_AAI_HMAC_PRECOMP, 2025 .geniv = true, 2026 } 2027 }, 2028 { 2029 .aead = { 2030 .base = { 2031 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2032 .cra_driver_name = "authenc-hmac-sha384-" 2033 "cbc-aes-caam-qi2", 2034 .cra_blocksize = AES_BLOCK_SIZE, 2035 }, 2036 .setkey = aead_setkey, 2037 .setauthsize = aead_setauthsize, 2038 .encrypt = aead_encrypt, 2039 .decrypt = aead_decrypt, 2040 .ivsize = AES_BLOCK_SIZE, 2041 .maxauthsize = SHA384_DIGEST_SIZE, 2042 }, 2043 .caam = { 2044 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2045 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2046 OP_ALG_AAI_HMAC_PRECOMP, 2047 } 2048 }, 2049 { 2050 .aead = { 2051 .base = { 2052 .cra_name = "echainiv(authenc(hmac(sha384)," 2053 "cbc(aes)))", 2054 .cra_driver_name = "echainiv-authenc-" 2055 "hmac-sha384-cbc-aes-" 2056 "caam-qi2", 2057 .cra_blocksize = AES_BLOCK_SIZE, 2058 }, 2059 .setkey = aead_setkey, 2060 .setauthsize = aead_setauthsize, 2061 .encrypt = aead_encrypt, 2062 .decrypt = aead_decrypt, 2063 .ivsize = AES_BLOCK_SIZE, 2064 .maxauthsize = SHA384_DIGEST_SIZE, 2065 }, 2066 .caam = { 2067 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2068 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2069 OP_ALG_AAI_HMAC_PRECOMP, 2070 .geniv = true, 2071 } 2072 }, 2073 { 2074 .aead = { 2075 .base = { 2076 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2077 .cra_driver_name = "authenc-hmac-sha512-" 2078 "cbc-aes-caam-qi2", 2079 .cra_blocksize = AES_BLOCK_SIZE, 2080 }, 2081 .setkey = aead_setkey, 2082 .setauthsize = aead_setauthsize, 2083 .encrypt = aead_encrypt, 2084 .decrypt = aead_decrypt, 2085 .ivsize = AES_BLOCK_SIZE, 2086 .maxauthsize = SHA512_DIGEST_SIZE, 2087 }, 2088 .caam = { 2089 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2090 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2091 OP_ALG_AAI_HMAC_PRECOMP, 2092 } 2093 }, 2094 { 2095 .aead = { 2096 .base = { 2097 .cra_name = "echainiv(authenc(hmac(sha512)," 2098 "cbc(aes)))", 2099 .cra_driver_name = "echainiv-authenc-" 2100 "hmac-sha512-cbc-aes-" 2101 "caam-qi2", 2102 .cra_blocksize = AES_BLOCK_SIZE, 2103 }, 2104 .setkey = aead_setkey, 2105 .setauthsize = aead_setauthsize, 2106 .encrypt = aead_encrypt, 2107 .decrypt = aead_decrypt, 2108 .ivsize = AES_BLOCK_SIZE, 2109 .maxauthsize = SHA512_DIGEST_SIZE, 2110 }, 2111 .caam = { 2112 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2113 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2114 OP_ALG_AAI_HMAC_PRECOMP, 2115 .geniv = true, 2116 } 2117 }, 2118 { 2119 .aead = { 2120 .base = { 2121 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2122 .cra_driver_name = "authenc-hmac-md5-" 2123 "cbc-des3_ede-caam-qi2", 2124 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2125 }, 2126 .setkey = des3_aead_setkey, 2127 .setauthsize = aead_setauthsize, 2128 .encrypt = aead_encrypt, 2129 .decrypt = aead_decrypt, 2130 .ivsize = DES3_EDE_BLOCK_SIZE, 2131 .maxauthsize = MD5_DIGEST_SIZE, 2132 }, 2133 .caam = { 2134 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2135 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2136 OP_ALG_AAI_HMAC_PRECOMP, 2137 } 2138 }, 2139 { 2140 .aead = { 2141 .base = { 2142 .cra_name = "echainiv(authenc(hmac(md5)," 2143 "cbc(des3_ede)))", 2144 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2145 "cbc-des3_ede-caam-qi2", 2146 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2147 }, 2148 .setkey = des3_aead_setkey, 2149 .setauthsize = aead_setauthsize, 2150 .encrypt = aead_encrypt, 2151 .decrypt = aead_decrypt, 2152 .ivsize = DES3_EDE_BLOCK_SIZE, 2153 .maxauthsize = MD5_DIGEST_SIZE, 2154 }, 2155 .caam = { 2156 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2157 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2158 OP_ALG_AAI_HMAC_PRECOMP, 2159 .geniv = true, 2160 } 2161 }, 2162 { 2163 .aead = { 2164 .base = { 2165 .cra_name = "authenc(hmac(sha1)," 2166 "cbc(des3_ede))", 2167 .cra_driver_name = "authenc-hmac-sha1-" 2168 "cbc-des3_ede-caam-qi2", 2169 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2170 }, 2171 .setkey = des3_aead_setkey, 2172 .setauthsize = aead_setauthsize, 2173 .encrypt = aead_encrypt, 2174 .decrypt = aead_decrypt, 2175 .ivsize = DES3_EDE_BLOCK_SIZE, 2176 .maxauthsize = SHA1_DIGEST_SIZE, 2177 }, 2178 .caam = { 2179 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2180 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2181 OP_ALG_AAI_HMAC_PRECOMP, 2182 }, 2183 }, 2184 { 2185 .aead = { 2186 .base = { 2187 .cra_name = "echainiv(authenc(hmac(sha1)," 2188 "cbc(des3_ede)))", 2189 .cra_driver_name = "echainiv-authenc-" 2190 "hmac-sha1-" 2191 "cbc-des3_ede-caam-qi2", 2192 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2193 }, 2194 .setkey = des3_aead_setkey, 2195 .setauthsize = aead_setauthsize, 2196 .encrypt = aead_encrypt, 2197 .decrypt = aead_decrypt, 2198 .ivsize = DES3_EDE_BLOCK_SIZE, 2199 .maxauthsize = SHA1_DIGEST_SIZE, 2200 }, 2201 .caam = { 2202 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2203 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2204 OP_ALG_AAI_HMAC_PRECOMP, 2205 .geniv = true, 2206 } 2207 }, 2208 { 2209 .aead = { 2210 .base = { 2211 .cra_name = "authenc(hmac(sha224)," 2212 "cbc(des3_ede))", 2213 .cra_driver_name = "authenc-hmac-sha224-" 2214 "cbc-des3_ede-caam-qi2", 2215 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2216 }, 2217 .setkey = des3_aead_setkey, 2218 .setauthsize = aead_setauthsize, 2219 .encrypt = aead_encrypt, 2220 .decrypt = aead_decrypt, 2221 .ivsize = DES3_EDE_BLOCK_SIZE, 2222 .maxauthsize = SHA224_DIGEST_SIZE, 2223 }, 2224 .caam = { 2225 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2226 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2227 OP_ALG_AAI_HMAC_PRECOMP, 2228 }, 2229 }, 2230 { 2231 .aead = { 2232 .base = { 2233 .cra_name = "echainiv(authenc(hmac(sha224)," 2234 "cbc(des3_ede)))", 2235 .cra_driver_name = "echainiv-authenc-" 2236 "hmac-sha224-" 2237 "cbc-des3_ede-caam-qi2", 2238 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2239 }, 2240 .setkey = des3_aead_setkey, 2241 .setauthsize = aead_setauthsize, 2242 .encrypt = aead_encrypt, 2243 .decrypt = aead_decrypt, 2244 .ivsize = DES3_EDE_BLOCK_SIZE, 2245 .maxauthsize = SHA224_DIGEST_SIZE, 2246 }, 2247 .caam = { 2248 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2249 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2250 OP_ALG_AAI_HMAC_PRECOMP, 2251 .geniv = true, 2252 } 2253 }, 2254 { 2255 .aead = { 2256 .base = { 2257 .cra_name = "authenc(hmac(sha256)," 2258 "cbc(des3_ede))", 2259 .cra_driver_name = "authenc-hmac-sha256-" 2260 "cbc-des3_ede-caam-qi2", 2261 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2262 }, 2263 .setkey = des3_aead_setkey, 2264 .setauthsize = aead_setauthsize, 2265 .encrypt = aead_encrypt, 2266 .decrypt = aead_decrypt, 2267 .ivsize = DES3_EDE_BLOCK_SIZE, 2268 .maxauthsize = SHA256_DIGEST_SIZE, 2269 }, 2270 .caam = { 2271 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2272 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2273 OP_ALG_AAI_HMAC_PRECOMP, 2274 }, 2275 }, 2276 { 2277 .aead = { 2278 .base = { 2279 .cra_name = "echainiv(authenc(hmac(sha256)," 2280 "cbc(des3_ede)))", 2281 .cra_driver_name = "echainiv-authenc-" 2282 "hmac-sha256-" 2283 "cbc-des3_ede-caam-qi2", 2284 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2285 }, 2286 .setkey = des3_aead_setkey, 2287 .setauthsize = aead_setauthsize, 2288 .encrypt = aead_encrypt, 2289 .decrypt = aead_decrypt, 2290 .ivsize = DES3_EDE_BLOCK_SIZE, 2291 .maxauthsize = SHA256_DIGEST_SIZE, 2292 }, 2293 .caam = { 2294 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2295 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2296 OP_ALG_AAI_HMAC_PRECOMP, 2297 .geniv = true, 2298 } 2299 }, 2300 { 2301 .aead = { 2302 .base = { 2303 .cra_name = "authenc(hmac(sha384)," 2304 "cbc(des3_ede))", 2305 .cra_driver_name = "authenc-hmac-sha384-" 2306 "cbc-des3_ede-caam-qi2", 2307 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2308 }, 2309 .setkey = des3_aead_setkey, 2310 .setauthsize = aead_setauthsize, 2311 .encrypt = aead_encrypt, 2312 .decrypt = aead_decrypt, 2313 .ivsize = DES3_EDE_BLOCK_SIZE, 2314 .maxauthsize = SHA384_DIGEST_SIZE, 2315 }, 2316 .caam = { 2317 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2318 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2319 OP_ALG_AAI_HMAC_PRECOMP, 2320 }, 2321 }, 2322 { 2323 .aead = { 2324 .base = { 2325 .cra_name = "echainiv(authenc(hmac(sha384)," 2326 "cbc(des3_ede)))", 2327 .cra_driver_name = "echainiv-authenc-" 2328 "hmac-sha384-" 2329 "cbc-des3_ede-caam-qi2", 2330 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2331 }, 2332 .setkey = des3_aead_setkey, 2333 .setauthsize = aead_setauthsize, 2334 .encrypt = aead_encrypt, 2335 .decrypt = aead_decrypt, 2336 .ivsize = DES3_EDE_BLOCK_SIZE, 2337 .maxauthsize = SHA384_DIGEST_SIZE, 2338 }, 2339 .caam = { 2340 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2341 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2342 OP_ALG_AAI_HMAC_PRECOMP, 2343 .geniv = true, 2344 } 2345 }, 2346 { 2347 .aead = { 2348 .base = { 2349 .cra_name = "authenc(hmac(sha512)," 2350 "cbc(des3_ede))", 2351 .cra_driver_name = "authenc-hmac-sha512-" 2352 "cbc-des3_ede-caam-qi2", 2353 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2354 }, 2355 .setkey = des3_aead_setkey, 2356 .setauthsize = aead_setauthsize, 2357 .encrypt = aead_encrypt, 2358 .decrypt = aead_decrypt, 2359 .ivsize = DES3_EDE_BLOCK_SIZE, 2360 .maxauthsize = SHA512_DIGEST_SIZE, 2361 }, 2362 .caam = { 2363 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2364 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2365 OP_ALG_AAI_HMAC_PRECOMP, 2366 }, 2367 }, 2368 { 2369 .aead = { 2370 .base = { 2371 .cra_name = "echainiv(authenc(hmac(sha512)," 2372 "cbc(des3_ede)))", 2373 .cra_driver_name = "echainiv-authenc-" 2374 "hmac-sha512-" 2375 "cbc-des3_ede-caam-qi2", 2376 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2377 }, 2378 .setkey = des3_aead_setkey, 2379 .setauthsize = aead_setauthsize, 2380 .encrypt = aead_encrypt, 2381 .decrypt = aead_decrypt, 2382 .ivsize = DES3_EDE_BLOCK_SIZE, 2383 .maxauthsize = SHA512_DIGEST_SIZE, 2384 }, 2385 .caam = { 2386 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2387 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2388 OP_ALG_AAI_HMAC_PRECOMP, 2389 .geniv = true, 2390 } 2391 }, 2392 { 2393 .aead = { 2394 .base = { 2395 .cra_name = "authenc(hmac(md5),cbc(des))", 2396 .cra_driver_name = "authenc-hmac-md5-" 2397 "cbc-des-caam-qi2", 2398 .cra_blocksize = DES_BLOCK_SIZE, 2399 }, 2400 .setkey = aead_setkey, 2401 .setauthsize = aead_setauthsize, 2402 .encrypt = aead_encrypt, 2403 .decrypt = aead_decrypt, 2404 .ivsize = DES_BLOCK_SIZE, 2405 .maxauthsize = MD5_DIGEST_SIZE, 2406 }, 2407 .caam = { 2408 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2409 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2410 OP_ALG_AAI_HMAC_PRECOMP, 2411 }, 2412 }, 2413 { 2414 .aead = { 2415 .base = { 2416 .cra_name = "echainiv(authenc(hmac(md5)," 2417 "cbc(des)))", 2418 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2419 "cbc-des-caam-qi2", 2420 .cra_blocksize = DES_BLOCK_SIZE, 2421 }, 2422 .setkey = aead_setkey, 2423 .setauthsize = aead_setauthsize, 2424 .encrypt = aead_encrypt, 2425 .decrypt = aead_decrypt, 2426 .ivsize = DES_BLOCK_SIZE, 2427 .maxauthsize = MD5_DIGEST_SIZE, 2428 }, 2429 .caam = { 2430 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2431 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2432 OP_ALG_AAI_HMAC_PRECOMP, 2433 .geniv = true, 2434 } 2435 }, 2436 { 2437 .aead = { 2438 .base = { 2439 .cra_name = "authenc(hmac(sha1),cbc(des))", 2440 .cra_driver_name = "authenc-hmac-sha1-" 2441 "cbc-des-caam-qi2", 2442 .cra_blocksize = DES_BLOCK_SIZE, 2443 }, 2444 .setkey = aead_setkey, 2445 .setauthsize = aead_setauthsize, 2446 .encrypt = aead_encrypt, 2447 .decrypt = aead_decrypt, 2448 .ivsize = DES_BLOCK_SIZE, 2449 .maxauthsize = SHA1_DIGEST_SIZE, 2450 }, 2451 .caam = { 2452 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2453 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2454 OP_ALG_AAI_HMAC_PRECOMP, 2455 }, 2456 }, 2457 { 2458 .aead = { 2459 .base = { 2460 .cra_name = "echainiv(authenc(hmac(sha1)," 2461 "cbc(des)))", 2462 .cra_driver_name = "echainiv-authenc-" 2463 "hmac-sha1-cbc-des-caam-qi2", 2464 .cra_blocksize = DES_BLOCK_SIZE, 2465 }, 2466 .setkey = aead_setkey, 2467 .setauthsize = aead_setauthsize, 2468 .encrypt = aead_encrypt, 2469 .decrypt = aead_decrypt, 2470 .ivsize = DES_BLOCK_SIZE, 2471 .maxauthsize = SHA1_DIGEST_SIZE, 2472 }, 2473 .caam = { 2474 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2475 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2476 OP_ALG_AAI_HMAC_PRECOMP, 2477 .geniv = true, 2478 } 2479 }, 2480 { 2481 .aead = { 2482 .base = { 2483 .cra_name = "authenc(hmac(sha224),cbc(des))", 2484 .cra_driver_name = "authenc-hmac-sha224-" 2485 "cbc-des-caam-qi2", 2486 .cra_blocksize = DES_BLOCK_SIZE, 2487 }, 2488 .setkey = aead_setkey, 2489 .setauthsize = aead_setauthsize, 2490 .encrypt = aead_encrypt, 2491 .decrypt = aead_decrypt, 2492 .ivsize = DES_BLOCK_SIZE, 2493 .maxauthsize = SHA224_DIGEST_SIZE, 2494 }, 2495 .caam = { 2496 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2497 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2498 OP_ALG_AAI_HMAC_PRECOMP, 2499 }, 2500 }, 2501 { 2502 .aead = { 2503 .base = { 2504 .cra_name = "echainiv(authenc(hmac(sha224)," 2505 "cbc(des)))", 2506 .cra_driver_name = "echainiv-authenc-" 2507 "hmac-sha224-cbc-des-" 2508 "caam-qi2", 2509 .cra_blocksize = DES_BLOCK_SIZE, 2510 }, 2511 .setkey = aead_setkey, 2512 .setauthsize = aead_setauthsize, 2513 .encrypt = aead_encrypt, 2514 .decrypt = aead_decrypt, 2515 .ivsize = DES_BLOCK_SIZE, 2516 .maxauthsize = SHA224_DIGEST_SIZE, 2517 }, 2518 .caam = { 2519 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2520 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2521 OP_ALG_AAI_HMAC_PRECOMP, 2522 .geniv = true, 2523 } 2524 }, 2525 { 2526 .aead = { 2527 .base = { 2528 .cra_name = "authenc(hmac(sha256),cbc(des))", 2529 .cra_driver_name = "authenc-hmac-sha256-" 2530 "cbc-des-caam-qi2", 2531 .cra_blocksize = DES_BLOCK_SIZE, 2532 }, 2533 .setkey = aead_setkey, 2534 .setauthsize = aead_setauthsize, 2535 .encrypt = aead_encrypt, 2536 .decrypt = aead_decrypt, 2537 .ivsize = DES_BLOCK_SIZE, 2538 .maxauthsize = SHA256_DIGEST_SIZE, 2539 }, 2540 .caam = { 2541 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2542 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2543 OP_ALG_AAI_HMAC_PRECOMP, 2544 }, 2545 }, 2546 { 2547 .aead = { 2548 .base = { 2549 .cra_name = "echainiv(authenc(hmac(sha256)," 2550 "cbc(des)))", 2551 .cra_driver_name = "echainiv-authenc-" 2552 "hmac-sha256-cbc-des-" 2553 "caam-qi2", 2554 .cra_blocksize = DES_BLOCK_SIZE, 2555 }, 2556 .setkey = aead_setkey, 2557 .setauthsize = aead_setauthsize, 2558 .encrypt = aead_encrypt, 2559 .decrypt = aead_decrypt, 2560 .ivsize = DES_BLOCK_SIZE, 2561 .maxauthsize = SHA256_DIGEST_SIZE, 2562 }, 2563 .caam = { 2564 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2565 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2566 OP_ALG_AAI_HMAC_PRECOMP, 2567 .geniv = true, 2568 }, 2569 }, 2570 { 2571 .aead = { 2572 .base = { 2573 .cra_name = "authenc(hmac(sha384),cbc(des))", 2574 .cra_driver_name = "authenc-hmac-sha384-" 2575 "cbc-des-caam-qi2", 2576 .cra_blocksize = DES_BLOCK_SIZE, 2577 }, 2578 .setkey = aead_setkey, 2579 .setauthsize = aead_setauthsize, 2580 .encrypt = aead_encrypt, 2581 .decrypt = aead_decrypt, 2582 .ivsize = DES_BLOCK_SIZE, 2583 .maxauthsize = SHA384_DIGEST_SIZE, 2584 }, 2585 .caam = { 2586 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2587 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2588 OP_ALG_AAI_HMAC_PRECOMP, 2589 }, 2590 }, 2591 { 2592 .aead = { 2593 .base = { 2594 .cra_name = "echainiv(authenc(hmac(sha384)," 2595 "cbc(des)))", 2596 .cra_driver_name = "echainiv-authenc-" 2597 "hmac-sha384-cbc-des-" 2598 "caam-qi2", 2599 .cra_blocksize = DES_BLOCK_SIZE, 2600 }, 2601 .setkey = aead_setkey, 2602 .setauthsize = aead_setauthsize, 2603 .encrypt = aead_encrypt, 2604 .decrypt = aead_decrypt, 2605 .ivsize = DES_BLOCK_SIZE, 2606 .maxauthsize = SHA384_DIGEST_SIZE, 2607 }, 2608 .caam = { 2609 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2610 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2611 OP_ALG_AAI_HMAC_PRECOMP, 2612 .geniv = true, 2613 } 2614 }, 2615 { 2616 .aead = { 2617 .base = { 2618 .cra_name = "authenc(hmac(sha512),cbc(des))", 2619 .cra_driver_name = "authenc-hmac-sha512-" 2620 "cbc-des-caam-qi2", 2621 .cra_blocksize = DES_BLOCK_SIZE, 2622 }, 2623 .setkey = aead_setkey, 2624 .setauthsize = aead_setauthsize, 2625 .encrypt = aead_encrypt, 2626 .decrypt = aead_decrypt, 2627 .ivsize = DES_BLOCK_SIZE, 2628 .maxauthsize = SHA512_DIGEST_SIZE, 2629 }, 2630 .caam = { 2631 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2632 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2633 OP_ALG_AAI_HMAC_PRECOMP, 2634 } 2635 }, 2636 { 2637 .aead = { 2638 .base = { 2639 .cra_name = "echainiv(authenc(hmac(sha512)," 2640 "cbc(des)))", 2641 .cra_driver_name = "echainiv-authenc-" 2642 "hmac-sha512-cbc-des-" 2643 "caam-qi2", 2644 .cra_blocksize = DES_BLOCK_SIZE, 2645 }, 2646 .setkey = aead_setkey, 2647 .setauthsize = aead_setauthsize, 2648 .encrypt = aead_encrypt, 2649 .decrypt = aead_decrypt, 2650 .ivsize = DES_BLOCK_SIZE, 2651 .maxauthsize = SHA512_DIGEST_SIZE, 2652 }, 2653 .caam = { 2654 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2655 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2656 OP_ALG_AAI_HMAC_PRECOMP, 2657 .geniv = true, 2658 } 2659 }, 2660 { 2661 .aead = { 2662 .base = { 2663 .cra_name = "authenc(hmac(md5)," 2664 "rfc3686(ctr(aes)))", 2665 .cra_driver_name = "authenc-hmac-md5-" 2666 "rfc3686-ctr-aes-caam-qi2", 2667 .cra_blocksize = 1, 2668 }, 2669 .setkey = aead_setkey, 2670 .setauthsize = aead_setauthsize, 2671 .encrypt = aead_encrypt, 2672 .decrypt = aead_decrypt, 2673 .ivsize = CTR_RFC3686_IV_SIZE, 2674 .maxauthsize = MD5_DIGEST_SIZE, 2675 }, 2676 .caam = { 2677 .class1_alg_type = OP_ALG_ALGSEL_AES | 2678 OP_ALG_AAI_CTR_MOD128, 2679 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2680 OP_ALG_AAI_HMAC_PRECOMP, 2681 .rfc3686 = true, 2682 }, 2683 }, 2684 { 2685 .aead = { 2686 .base = { 2687 .cra_name = "seqiv(authenc(" 2688 "hmac(md5),rfc3686(ctr(aes))))", 2689 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2690 "rfc3686-ctr-aes-caam-qi2", 2691 .cra_blocksize = 1, 2692 }, 2693 .setkey = aead_setkey, 2694 .setauthsize = aead_setauthsize, 2695 .encrypt = aead_encrypt, 2696 .decrypt = aead_decrypt, 2697 .ivsize = CTR_RFC3686_IV_SIZE, 2698 .maxauthsize = MD5_DIGEST_SIZE, 2699 }, 2700 .caam = { 2701 .class1_alg_type = OP_ALG_ALGSEL_AES | 2702 OP_ALG_AAI_CTR_MOD128, 2703 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2704 OP_ALG_AAI_HMAC_PRECOMP, 2705 .rfc3686 = true, 2706 .geniv = true, 2707 }, 2708 }, 2709 { 2710 .aead = { 2711 .base = { 2712 .cra_name = "authenc(hmac(sha1)," 2713 "rfc3686(ctr(aes)))", 2714 .cra_driver_name = "authenc-hmac-sha1-" 2715 "rfc3686-ctr-aes-caam-qi2", 2716 .cra_blocksize = 1, 2717 }, 2718 .setkey = aead_setkey, 2719 .setauthsize = aead_setauthsize, 2720 .encrypt = aead_encrypt, 2721 .decrypt = aead_decrypt, 2722 .ivsize = CTR_RFC3686_IV_SIZE, 2723 .maxauthsize = SHA1_DIGEST_SIZE, 2724 }, 2725 .caam = { 2726 .class1_alg_type = OP_ALG_ALGSEL_AES | 2727 OP_ALG_AAI_CTR_MOD128, 2728 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2729 OP_ALG_AAI_HMAC_PRECOMP, 2730 .rfc3686 = true, 2731 }, 2732 }, 2733 { 2734 .aead = { 2735 .base = { 2736 .cra_name = "seqiv(authenc(" 2737 "hmac(sha1),rfc3686(ctr(aes))))", 2738 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2739 "rfc3686-ctr-aes-caam-qi2", 2740 .cra_blocksize = 1, 2741 }, 2742 .setkey = aead_setkey, 2743 .setauthsize = aead_setauthsize, 2744 .encrypt = aead_encrypt, 2745 .decrypt = aead_decrypt, 2746 .ivsize = CTR_RFC3686_IV_SIZE, 2747 .maxauthsize = SHA1_DIGEST_SIZE, 2748 }, 2749 .caam = { 2750 .class1_alg_type = OP_ALG_ALGSEL_AES | 2751 OP_ALG_AAI_CTR_MOD128, 2752 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2753 OP_ALG_AAI_HMAC_PRECOMP, 2754 .rfc3686 = true, 2755 .geniv = true, 2756 }, 2757 }, 2758 { 2759 .aead = { 2760 .base = { 2761 .cra_name = "authenc(hmac(sha224)," 2762 "rfc3686(ctr(aes)))", 2763 .cra_driver_name = "authenc-hmac-sha224-" 2764 "rfc3686-ctr-aes-caam-qi2", 2765 .cra_blocksize = 1, 2766 }, 2767 .setkey = aead_setkey, 2768 .setauthsize = aead_setauthsize, 2769 .encrypt = aead_encrypt, 2770 .decrypt = aead_decrypt, 2771 .ivsize = CTR_RFC3686_IV_SIZE, 2772 .maxauthsize = SHA224_DIGEST_SIZE, 2773 }, 2774 .caam = { 2775 .class1_alg_type = OP_ALG_ALGSEL_AES | 2776 OP_ALG_AAI_CTR_MOD128, 2777 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2778 OP_ALG_AAI_HMAC_PRECOMP, 2779 .rfc3686 = true, 2780 }, 2781 }, 2782 { 2783 .aead = { 2784 .base = { 2785 .cra_name = "seqiv(authenc(" 2786 "hmac(sha224),rfc3686(ctr(aes))))", 2787 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2788 "rfc3686-ctr-aes-caam-qi2", 2789 .cra_blocksize = 1, 2790 }, 2791 .setkey = aead_setkey, 2792 .setauthsize = aead_setauthsize, 2793 .encrypt = aead_encrypt, 2794 .decrypt = aead_decrypt, 2795 .ivsize = CTR_RFC3686_IV_SIZE, 2796 .maxauthsize = SHA224_DIGEST_SIZE, 2797 }, 2798 .caam = { 2799 .class1_alg_type = OP_ALG_ALGSEL_AES | 2800 OP_ALG_AAI_CTR_MOD128, 2801 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2802 OP_ALG_AAI_HMAC_PRECOMP, 2803 .rfc3686 = true, 2804 .geniv = true, 2805 }, 2806 }, 2807 { 2808 .aead = { 2809 .base = { 2810 .cra_name = "authenc(hmac(sha256)," 2811 "rfc3686(ctr(aes)))", 2812 .cra_driver_name = "authenc-hmac-sha256-" 2813 "rfc3686-ctr-aes-caam-qi2", 2814 .cra_blocksize = 1, 2815 }, 2816 .setkey = aead_setkey, 2817 .setauthsize = aead_setauthsize, 2818 .encrypt = aead_encrypt, 2819 .decrypt = aead_decrypt, 2820 .ivsize = CTR_RFC3686_IV_SIZE, 2821 .maxauthsize = SHA256_DIGEST_SIZE, 2822 }, 2823 .caam = { 2824 .class1_alg_type = OP_ALG_ALGSEL_AES | 2825 OP_ALG_AAI_CTR_MOD128, 2826 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2827 OP_ALG_AAI_HMAC_PRECOMP, 2828 .rfc3686 = true, 2829 }, 2830 }, 2831 { 2832 .aead = { 2833 .base = { 2834 .cra_name = "seqiv(authenc(hmac(sha256)," 2835 "rfc3686(ctr(aes))))", 2836 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2837 "rfc3686-ctr-aes-caam-qi2", 2838 .cra_blocksize = 1, 2839 }, 2840 .setkey = aead_setkey, 2841 .setauthsize = aead_setauthsize, 2842 .encrypt = aead_encrypt, 2843 .decrypt = aead_decrypt, 2844 .ivsize = CTR_RFC3686_IV_SIZE, 2845 .maxauthsize = SHA256_DIGEST_SIZE, 2846 }, 2847 .caam = { 2848 .class1_alg_type = OP_ALG_ALGSEL_AES | 2849 OP_ALG_AAI_CTR_MOD128, 2850 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2851 OP_ALG_AAI_HMAC_PRECOMP, 2852 .rfc3686 = true, 2853 .geniv = true, 2854 }, 2855 }, 2856 { 2857 .aead = { 2858 .base = { 2859 .cra_name = "authenc(hmac(sha384)," 2860 "rfc3686(ctr(aes)))", 2861 .cra_driver_name = "authenc-hmac-sha384-" 2862 "rfc3686-ctr-aes-caam-qi2", 2863 .cra_blocksize = 1, 2864 }, 2865 .setkey = aead_setkey, 2866 .setauthsize = aead_setauthsize, 2867 .encrypt = aead_encrypt, 2868 .decrypt = aead_decrypt, 2869 .ivsize = CTR_RFC3686_IV_SIZE, 2870 .maxauthsize = SHA384_DIGEST_SIZE, 2871 }, 2872 .caam = { 2873 .class1_alg_type = OP_ALG_ALGSEL_AES | 2874 OP_ALG_AAI_CTR_MOD128, 2875 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2876 OP_ALG_AAI_HMAC_PRECOMP, 2877 .rfc3686 = true, 2878 }, 2879 }, 2880 { 2881 .aead = { 2882 .base = { 2883 .cra_name = "seqiv(authenc(hmac(sha384)," 2884 "rfc3686(ctr(aes))))", 2885 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2886 "rfc3686-ctr-aes-caam-qi2", 2887 .cra_blocksize = 1, 2888 }, 2889 .setkey = aead_setkey, 2890 .setauthsize = aead_setauthsize, 2891 .encrypt = aead_encrypt, 2892 .decrypt = aead_decrypt, 2893 .ivsize = CTR_RFC3686_IV_SIZE, 2894 .maxauthsize = SHA384_DIGEST_SIZE, 2895 }, 2896 .caam = { 2897 .class1_alg_type = OP_ALG_ALGSEL_AES | 2898 OP_ALG_AAI_CTR_MOD128, 2899 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2900 OP_ALG_AAI_HMAC_PRECOMP, 2901 .rfc3686 = true, 2902 .geniv = true, 2903 }, 2904 }, 2905 { 2906 .aead = { 2907 .base = { 2908 .cra_name = "rfc7539(chacha20,poly1305)", 2909 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2910 "caam-qi2", 2911 .cra_blocksize = 1, 2912 }, 2913 .setkey = chachapoly_setkey, 2914 .setauthsize = chachapoly_setauthsize, 2915 .encrypt = aead_encrypt, 2916 .decrypt = aead_decrypt, 2917 .ivsize = CHACHAPOLY_IV_SIZE, 2918 .maxauthsize = POLY1305_DIGEST_SIZE, 2919 }, 2920 .caam = { 2921 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2922 OP_ALG_AAI_AEAD, 2923 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2924 OP_ALG_AAI_AEAD, 2925 .nodkp = true, 2926 }, 2927 }, 2928 { 2929 .aead = { 2930 .base = { 2931 .cra_name = "rfc7539esp(chacha20,poly1305)", 2932 .cra_driver_name = "rfc7539esp-chacha20-" 2933 "poly1305-caam-qi2", 2934 .cra_blocksize = 1, 2935 }, 2936 .setkey = chachapoly_setkey, 2937 .setauthsize = chachapoly_setauthsize, 2938 .encrypt = aead_encrypt, 2939 .decrypt = aead_decrypt, 2940 .ivsize = 8, 2941 .maxauthsize = POLY1305_DIGEST_SIZE, 2942 }, 2943 .caam = { 2944 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2945 OP_ALG_AAI_AEAD, 2946 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2947 OP_ALG_AAI_AEAD, 2948 .nodkp = true, 2949 }, 2950 }, 2951 { 2952 .aead = { 2953 .base = { 2954 .cra_name = "authenc(hmac(sha512)," 2955 "rfc3686(ctr(aes)))", 2956 .cra_driver_name = "authenc-hmac-sha512-" 2957 "rfc3686-ctr-aes-caam-qi2", 2958 .cra_blocksize = 1, 2959 }, 2960 .setkey = aead_setkey, 2961 .setauthsize = aead_setauthsize, 2962 .encrypt = aead_encrypt, 2963 .decrypt = aead_decrypt, 2964 .ivsize = CTR_RFC3686_IV_SIZE, 2965 .maxauthsize = SHA512_DIGEST_SIZE, 2966 }, 2967 .caam = { 2968 .class1_alg_type = OP_ALG_ALGSEL_AES | 2969 OP_ALG_AAI_CTR_MOD128, 2970 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2971 OP_ALG_AAI_HMAC_PRECOMP, 2972 .rfc3686 = true, 2973 }, 2974 }, 2975 { 2976 .aead = { 2977 .base = { 2978 .cra_name = "seqiv(authenc(hmac(sha512)," 2979 "rfc3686(ctr(aes))))", 2980 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2981 "rfc3686-ctr-aes-caam-qi2", 2982 .cra_blocksize = 1, 2983 }, 2984 .setkey = aead_setkey, 2985 .setauthsize = aead_setauthsize, 2986 .encrypt = aead_encrypt, 2987 .decrypt = aead_decrypt, 2988 .ivsize = CTR_RFC3686_IV_SIZE, 2989 .maxauthsize = SHA512_DIGEST_SIZE, 2990 }, 2991 .caam = { 2992 .class1_alg_type = OP_ALG_ALGSEL_AES | 2993 OP_ALG_AAI_CTR_MOD128, 2994 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2995 OP_ALG_AAI_HMAC_PRECOMP, 2996 .rfc3686 = true, 2997 .geniv = true, 2998 }, 2999 }, 3000 }; 3001 3002 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3003 { 3004 struct skcipher_alg *alg = &t_alg->skcipher; 3005 3006 alg->base.cra_module = THIS_MODULE; 3007 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3008 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3009 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3010 CRYPTO_ALG_KERN_DRIVER_ONLY); 3011 3012 alg->init = caam_cra_init_skcipher; 3013 alg->exit = caam_cra_exit; 3014 } 3015 3016 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3017 { 3018 struct aead_alg *alg = &t_alg->aead; 3019 3020 alg->base.cra_module = THIS_MODULE; 3021 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3022 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3023 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3024 CRYPTO_ALG_KERN_DRIVER_ONLY; 3025 3026 alg->init = caam_cra_init_aead; 3027 alg->exit = caam_cra_exit_aead; 3028 } 3029 3030 /* max hash key is max split key size */ 3031 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 3032 3033 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 3034 3035 /* caam context sizes for hashes: running digest + 8 */ 3036 #define HASH_MSG_LEN 8 3037 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 3038 3039 enum hash_optype { 3040 UPDATE = 0, 3041 UPDATE_FIRST, 3042 FINALIZE, 3043 DIGEST, 3044 HASH_NUM_OP 3045 }; 3046 3047 /** 3048 * struct caam_hash_ctx - ahash per-session context 3049 * @flc: Flow Contexts array 3050 * @key: authentication key 3051 * @flc_dma: I/O virtual addresses of the Flow Contexts 3052 * @dev: dpseci device 3053 * @ctx_len: size of Context Register 3054 * @adata: hashing algorithm details 3055 */ 3056 struct caam_hash_ctx { 3057 struct caam_flc flc[HASH_NUM_OP]; 3058 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3059 dma_addr_t flc_dma[HASH_NUM_OP]; 3060 struct device *dev; 3061 int ctx_len; 3062 struct alginfo adata; 3063 }; 3064 3065 /* ahash state */ 3066 struct caam_hash_state { 3067 struct caam_request caam_req; 3068 dma_addr_t buf_dma; 3069 dma_addr_t ctx_dma; 3070 int ctx_dma_len; 3071 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3072 int buflen; 3073 int next_buflen; 3074 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 3075 int (*update)(struct ahash_request *req); 3076 int (*final)(struct ahash_request *req); 3077 int (*finup)(struct ahash_request *req); 3078 }; 3079 3080 struct caam_export_state { 3081 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 3082 u8 caam_ctx[MAX_CTX_LEN]; 3083 int buflen; 3084 int (*update)(struct ahash_request *req); 3085 int (*final)(struct ahash_request *req); 3086 int (*finup)(struct ahash_request *req); 3087 }; 3088 3089 /* Map current buffer in state (if length > 0) and put it in link table */ 3090 static inline int buf_map_to_qm_sg(struct device *dev, 3091 struct dpaa2_sg_entry *qm_sg, 3092 struct caam_hash_state *state) 3093 { 3094 int buflen = state->buflen; 3095 3096 if (!buflen) 3097 return 0; 3098 3099 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3100 DMA_TO_DEVICE); 3101 if (dma_mapping_error(dev, state->buf_dma)) { 3102 dev_err(dev, "unable to map buf\n"); 3103 state->buf_dma = 0; 3104 return -ENOMEM; 3105 } 3106 3107 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3108 3109 return 0; 3110 } 3111 3112 /* Map state->caam_ctx, and add it to link table */ 3113 static inline int ctx_map_to_qm_sg(struct device *dev, 3114 struct caam_hash_state *state, int ctx_len, 3115 struct dpaa2_sg_entry *qm_sg, u32 flag) 3116 { 3117 state->ctx_dma_len = ctx_len; 3118 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3119 if (dma_mapping_error(dev, state->ctx_dma)) { 3120 dev_err(dev, "unable to map ctx\n"); 3121 state->ctx_dma = 0; 3122 return -ENOMEM; 3123 } 3124 3125 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3126 3127 return 0; 3128 } 3129 3130 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3131 { 3132 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3133 int digestsize = crypto_ahash_digestsize(ahash); 3134 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3135 struct caam_flc *flc; 3136 u32 *desc; 3137 3138 /* ahash_update shared descriptor */ 3139 flc = &ctx->flc[UPDATE]; 3140 desc = flc->sh_desc; 3141 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3142 ctx->ctx_len, true, priv->sec_attr.era); 3143 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3144 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3145 desc_bytes(desc), DMA_BIDIRECTIONAL); 3146 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3147 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3148 1); 3149 3150 /* ahash_update_first shared descriptor */ 3151 flc = &ctx->flc[UPDATE_FIRST]; 3152 desc = flc->sh_desc; 3153 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3154 ctx->ctx_len, false, priv->sec_attr.era); 3155 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3156 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3157 desc_bytes(desc), DMA_BIDIRECTIONAL); 3158 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3159 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3160 1); 3161 3162 /* ahash_final shared descriptor */ 3163 flc = &ctx->flc[FINALIZE]; 3164 desc = flc->sh_desc; 3165 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3166 ctx->ctx_len, true, priv->sec_attr.era); 3167 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3168 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3169 desc_bytes(desc), DMA_BIDIRECTIONAL); 3170 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3171 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3172 1); 3173 3174 /* ahash_digest shared descriptor */ 3175 flc = &ctx->flc[DIGEST]; 3176 desc = flc->sh_desc; 3177 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3178 ctx->ctx_len, false, priv->sec_attr.era); 3179 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3180 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3181 desc_bytes(desc), DMA_BIDIRECTIONAL); 3182 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3183 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3184 1); 3185 3186 return 0; 3187 } 3188 3189 struct split_key_sh_result { 3190 struct completion completion; 3191 int err; 3192 struct device *dev; 3193 }; 3194 3195 static void split_key_sh_done(void *cbk_ctx, u32 err) 3196 { 3197 struct split_key_sh_result *res = cbk_ctx; 3198 3199 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3200 3201 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3202 complete(&res->completion); 3203 } 3204 3205 /* Digest hash size if it is too large */ 3206 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3207 u32 digestsize) 3208 { 3209 struct caam_request *req_ctx; 3210 u32 *desc; 3211 struct split_key_sh_result result; 3212 dma_addr_t key_dma; 3213 struct caam_flc *flc; 3214 dma_addr_t flc_dma; 3215 int ret = -ENOMEM; 3216 struct dpaa2_fl_entry *in_fle, *out_fle; 3217 3218 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); 3219 if (!req_ctx) 3220 return -ENOMEM; 3221 3222 in_fle = &req_ctx->fd_flt[1]; 3223 out_fle = &req_ctx->fd_flt[0]; 3224 3225 flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); 3226 if (!flc) 3227 goto err_flc; 3228 3229 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3230 if (dma_mapping_error(ctx->dev, key_dma)) { 3231 dev_err(ctx->dev, "unable to map key memory\n"); 3232 goto err_key_dma; 3233 } 3234 3235 desc = flc->sh_desc; 3236 3237 init_sh_desc(desc, 0); 3238 3239 /* descriptor to perform unkeyed hash on key_in */ 3240 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3241 OP_ALG_AS_INITFINAL); 3242 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3243 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3244 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3245 LDST_SRCDST_BYTE_CONTEXT); 3246 3247 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3248 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3249 desc_bytes(desc), DMA_TO_DEVICE); 3250 if (dma_mapping_error(ctx->dev, flc_dma)) { 3251 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3252 goto err_flc_dma; 3253 } 3254 3255 dpaa2_fl_set_final(in_fle, true); 3256 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3257 dpaa2_fl_set_addr(in_fle, key_dma); 3258 dpaa2_fl_set_len(in_fle, *keylen); 3259 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3260 dpaa2_fl_set_addr(out_fle, key_dma); 3261 dpaa2_fl_set_len(out_fle, digestsize); 3262 3263 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3264 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3265 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3266 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3267 1); 3268 3269 result.err = 0; 3270 init_completion(&result.completion); 3271 result.dev = ctx->dev; 3272 3273 req_ctx->flc = flc; 3274 req_ctx->flc_dma = flc_dma; 3275 req_ctx->cbk = split_key_sh_done; 3276 req_ctx->ctx = &result; 3277 3278 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3279 if (ret == -EINPROGRESS) { 3280 /* in progress */ 3281 wait_for_completion(&result.completion); 3282 ret = result.err; 3283 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3284 DUMP_PREFIX_ADDRESS, 16, 4, key, 3285 digestsize, 1); 3286 } 3287 3288 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3289 DMA_TO_DEVICE); 3290 err_flc_dma: 3291 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3292 err_key_dma: 3293 kfree(flc); 3294 err_flc: 3295 kfree(req_ctx); 3296 3297 *keylen = digestsize; 3298 3299 return ret; 3300 } 3301 3302 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3303 unsigned int keylen) 3304 { 3305 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3306 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3307 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3308 int ret; 3309 u8 *hashed_key = NULL; 3310 3311 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3312 3313 if (keylen > blocksize) { 3314 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 3315 if (!hashed_key) 3316 return -ENOMEM; 3317 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3318 if (ret) 3319 goto bad_free_key; 3320 key = hashed_key; 3321 } 3322 3323 ctx->adata.keylen = keylen; 3324 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3325 OP_ALG_ALGSEL_MASK); 3326 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3327 goto bad_free_key; 3328 3329 ctx->adata.key_virt = key; 3330 ctx->adata.key_inline = true; 3331 3332 /* 3333 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3334 * in invalid opcodes (last bytes of user key) in the resulting 3335 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3336 * addresses are needed. 3337 */ 3338 if (keylen > ctx->adata.keylen_pad) { 3339 memcpy(ctx->key, key, keylen); 3340 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3341 ctx->adata.keylen_pad, 3342 DMA_TO_DEVICE); 3343 } 3344 3345 ret = ahash_set_sh_desc(ahash); 3346 kfree(hashed_key); 3347 return ret; 3348 bad_free_key: 3349 kfree(hashed_key); 3350 return -EINVAL; 3351 } 3352 3353 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3354 struct ahash_request *req) 3355 { 3356 struct caam_hash_state *state = ahash_request_ctx(req); 3357 3358 if (edesc->src_nents) 3359 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3360 3361 if (edesc->qm_sg_bytes) 3362 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3363 DMA_TO_DEVICE); 3364 3365 if (state->buf_dma) { 3366 dma_unmap_single(dev, state->buf_dma, state->buflen, 3367 DMA_TO_DEVICE); 3368 state->buf_dma = 0; 3369 } 3370 } 3371 3372 static inline void ahash_unmap_ctx(struct device *dev, 3373 struct ahash_edesc *edesc, 3374 struct ahash_request *req, u32 flag) 3375 { 3376 struct caam_hash_state *state = ahash_request_ctx(req); 3377 3378 if (state->ctx_dma) { 3379 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3380 state->ctx_dma = 0; 3381 } 3382 ahash_unmap(dev, edesc, req); 3383 } 3384 3385 static void ahash_done(void *cbk_ctx, u32 status) 3386 { 3387 struct crypto_async_request *areq = cbk_ctx; 3388 struct ahash_request *req = ahash_request_cast(areq); 3389 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3390 struct caam_hash_state *state = ahash_request_ctx(req); 3391 struct ahash_edesc *edesc = state->caam_req.edesc; 3392 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3393 int digestsize = crypto_ahash_digestsize(ahash); 3394 int ecode = 0; 3395 3396 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3397 3398 if (unlikely(status)) 3399 ecode = caam_qi2_strstatus(ctx->dev, status); 3400 3401 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3402 memcpy(req->result, state->caam_ctx, digestsize); 3403 qi_cache_free(edesc); 3404 3405 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3406 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3407 ctx->ctx_len, 1); 3408 3409 req->base.complete(&req->base, ecode); 3410 } 3411 3412 static void ahash_done_bi(void *cbk_ctx, u32 status) 3413 { 3414 struct crypto_async_request *areq = cbk_ctx; 3415 struct ahash_request *req = ahash_request_cast(areq); 3416 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3417 struct caam_hash_state *state = ahash_request_ctx(req); 3418 struct ahash_edesc *edesc = state->caam_req.edesc; 3419 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3420 int ecode = 0; 3421 3422 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3423 3424 if (unlikely(status)) 3425 ecode = caam_qi2_strstatus(ctx->dev, status); 3426 3427 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3428 qi_cache_free(edesc); 3429 3430 scatterwalk_map_and_copy(state->buf, req->src, 3431 req->nbytes - state->next_buflen, 3432 state->next_buflen, 0); 3433 state->buflen = state->next_buflen; 3434 3435 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3436 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3437 state->buflen, 1); 3438 3439 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3440 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3441 ctx->ctx_len, 1); 3442 if (req->result) 3443 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3444 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3445 crypto_ahash_digestsize(ahash), 1); 3446 3447 req->base.complete(&req->base, ecode); 3448 } 3449 3450 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3451 { 3452 struct crypto_async_request *areq = cbk_ctx; 3453 struct ahash_request *req = ahash_request_cast(areq); 3454 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3455 struct caam_hash_state *state = ahash_request_ctx(req); 3456 struct ahash_edesc *edesc = state->caam_req.edesc; 3457 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3458 int digestsize = crypto_ahash_digestsize(ahash); 3459 int ecode = 0; 3460 3461 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3462 3463 if (unlikely(status)) 3464 ecode = caam_qi2_strstatus(ctx->dev, status); 3465 3466 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3467 memcpy(req->result, state->caam_ctx, digestsize); 3468 qi_cache_free(edesc); 3469 3470 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3471 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3472 ctx->ctx_len, 1); 3473 3474 req->base.complete(&req->base, ecode); 3475 } 3476 3477 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3478 { 3479 struct crypto_async_request *areq = cbk_ctx; 3480 struct ahash_request *req = ahash_request_cast(areq); 3481 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3482 struct caam_hash_state *state = ahash_request_ctx(req); 3483 struct ahash_edesc *edesc = state->caam_req.edesc; 3484 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3485 int ecode = 0; 3486 3487 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3488 3489 if (unlikely(status)) 3490 ecode = caam_qi2_strstatus(ctx->dev, status); 3491 3492 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3493 qi_cache_free(edesc); 3494 3495 scatterwalk_map_and_copy(state->buf, req->src, 3496 req->nbytes - state->next_buflen, 3497 state->next_buflen, 0); 3498 state->buflen = state->next_buflen; 3499 3500 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3501 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3502 state->buflen, 1); 3503 3504 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3505 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3506 ctx->ctx_len, 1); 3507 if (req->result) 3508 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3509 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3510 crypto_ahash_digestsize(ahash), 1); 3511 3512 req->base.complete(&req->base, ecode); 3513 } 3514 3515 static int ahash_update_ctx(struct ahash_request *req) 3516 { 3517 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3518 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3519 struct caam_hash_state *state = ahash_request_ctx(req); 3520 struct caam_request *req_ctx = &state->caam_req; 3521 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3522 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3523 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3524 GFP_KERNEL : GFP_ATOMIC; 3525 u8 *buf = state->buf; 3526 int *buflen = &state->buflen; 3527 int *next_buflen = &state->next_buflen; 3528 int in_len = *buflen + req->nbytes, to_hash; 3529 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3530 struct ahash_edesc *edesc; 3531 int ret = 0; 3532 3533 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3534 to_hash = in_len - *next_buflen; 3535 3536 if (to_hash) { 3537 struct dpaa2_sg_entry *sg_table; 3538 int src_len = req->nbytes - *next_buflen; 3539 3540 src_nents = sg_nents_for_len(req->src, src_len); 3541 if (src_nents < 0) { 3542 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3543 return src_nents; 3544 } 3545 3546 if (src_nents) { 3547 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3548 DMA_TO_DEVICE); 3549 if (!mapped_nents) { 3550 dev_err(ctx->dev, "unable to DMA map source\n"); 3551 return -ENOMEM; 3552 } 3553 } else { 3554 mapped_nents = 0; 3555 } 3556 3557 /* allocate space for base edesc and link tables */ 3558 edesc = qi_cache_zalloc(GFP_DMA | flags); 3559 if (!edesc) { 3560 dma_unmap_sg(ctx->dev, req->src, src_nents, 3561 DMA_TO_DEVICE); 3562 return -ENOMEM; 3563 } 3564 3565 edesc->src_nents = src_nents; 3566 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3567 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3568 sizeof(*sg_table); 3569 sg_table = &edesc->sgt[0]; 3570 3571 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3572 DMA_BIDIRECTIONAL); 3573 if (ret) 3574 goto unmap_ctx; 3575 3576 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3577 if (ret) 3578 goto unmap_ctx; 3579 3580 if (mapped_nents) { 3581 sg_to_qm_sg_last(req->src, src_len, 3582 sg_table + qm_sg_src_index, 0); 3583 } else { 3584 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3585 true); 3586 } 3587 3588 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3589 qm_sg_bytes, DMA_TO_DEVICE); 3590 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3591 dev_err(ctx->dev, "unable to map S/G table\n"); 3592 ret = -ENOMEM; 3593 goto unmap_ctx; 3594 } 3595 edesc->qm_sg_bytes = qm_sg_bytes; 3596 3597 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3598 dpaa2_fl_set_final(in_fle, true); 3599 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3600 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3601 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3602 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3603 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3604 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3605 3606 req_ctx->flc = &ctx->flc[UPDATE]; 3607 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3608 req_ctx->cbk = ahash_done_bi; 3609 req_ctx->ctx = &req->base; 3610 req_ctx->edesc = edesc; 3611 3612 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3613 if (ret != -EINPROGRESS && 3614 !(ret == -EBUSY && 3615 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3616 goto unmap_ctx; 3617 } else if (*next_buflen) { 3618 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3619 req->nbytes, 0); 3620 *buflen = *next_buflen; 3621 3622 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3623 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3624 *buflen, 1); 3625 } 3626 3627 return ret; 3628 unmap_ctx: 3629 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3630 qi_cache_free(edesc); 3631 return ret; 3632 } 3633 3634 static int ahash_final_ctx(struct ahash_request *req) 3635 { 3636 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3637 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3638 struct caam_hash_state *state = ahash_request_ctx(req); 3639 struct caam_request *req_ctx = &state->caam_req; 3640 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3641 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3642 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3643 GFP_KERNEL : GFP_ATOMIC; 3644 int buflen = state->buflen; 3645 int qm_sg_bytes; 3646 int digestsize = crypto_ahash_digestsize(ahash); 3647 struct ahash_edesc *edesc; 3648 struct dpaa2_sg_entry *sg_table; 3649 int ret; 3650 3651 /* allocate space for base edesc and link tables */ 3652 edesc = qi_cache_zalloc(GFP_DMA | flags); 3653 if (!edesc) 3654 return -ENOMEM; 3655 3656 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3657 sg_table = &edesc->sgt[0]; 3658 3659 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3660 DMA_BIDIRECTIONAL); 3661 if (ret) 3662 goto unmap_ctx; 3663 3664 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3665 if (ret) 3666 goto unmap_ctx; 3667 3668 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3669 3670 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3671 DMA_TO_DEVICE); 3672 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3673 dev_err(ctx->dev, "unable to map S/G table\n"); 3674 ret = -ENOMEM; 3675 goto unmap_ctx; 3676 } 3677 edesc->qm_sg_bytes = qm_sg_bytes; 3678 3679 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3680 dpaa2_fl_set_final(in_fle, true); 3681 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3682 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3683 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3684 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3685 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3686 dpaa2_fl_set_len(out_fle, digestsize); 3687 3688 req_ctx->flc = &ctx->flc[FINALIZE]; 3689 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3690 req_ctx->cbk = ahash_done_ctx_src; 3691 req_ctx->ctx = &req->base; 3692 req_ctx->edesc = edesc; 3693 3694 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3695 if (ret == -EINPROGRESS || 3696 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3697 return ret; 3698 3699 unmap_ctx: 3700 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3701 qi_cache_free(edesc); 3702 return ret; 3703 } 3704 3705 static int ahash_finup_ctx(struct ahash_request *req) 3706 { 3707 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3708 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3709 struct caam_hash_state *state = ahash_request_ctx(req); 3710 struct caam_request *req_ctx = &state->caam_req; 3711 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3712 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3713 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3714 GFP_KERNEL : GFP_ATOMIC; 3715 int buflen = state->buflen; 3716 int qm_sg_bytes, qm_sg_src_index; 3717 int src_nents, mapped_nents; 3718 int digestsize = crypto_ahash_digestsize(ahash); 3719 struct ahash_edesc *edesc; 3720 struct dpaa2_sg_entry *sg_table; 3721 int ret; 3722 3723 src_nents = sg_nents_for_len(req->src, req->nbytes); 3724 if (src_nents < 0) { 3725 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3726 return src_nents; 3727 } 3728 3729 if (src_nents) { 3730 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3731 DMA_TO_DEVICE); 3732 if (!mapped_nents) { 3733 dev_err(ctx->dev, "unable to DMA map source\n"); 3734 return -ENOMEM; 3735 } 3736 } else { 3737 mapped_nents = 0; 3738 } 3739 3740 /* allocate space for base edesc and link tables */ 3741 edesc = qi_cache_zalloc(GFP_DMA | flags); 3742 if (!edesc) { 3743 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3744 return -ENOMEM; 3745 } 3746 3747 edesc->src_nents = src_nents; 3748 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3749 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3750 sizeof(*sg_table); 3751 sg_table = &edesc->sgt[0]; 3752 3753 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3754 DMA_BIDIRECTIONAL); 3755 if (ret) 3756 goto unmap_ctx; 3757 3758 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3759 if (ret) 3760 goto unmap_ctx; 3761 3762 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3763 3764 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3765 DMA_TO_DEVICE); 3766 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3767 dev_err(ctx->dev, "unable to map S/G table\n"); 3768 ret = -ENOMEM; 3769 goto unmap_ctx; 3770 } 3771 edesc->qm_sg_bytes = qm_sg_bytes; 3772 3773 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3774 dpaa2_fl_set_final(in_fle, true); 3775 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3776 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3777 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3778 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3779 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3780 dpaa2_fl_set_len(out_fle, digestsize); 3781 3782 req_ctx->flc = &ctx->flc[FINALIZE]; 3783 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3784 req_ctx->cbk = ahash_done_ctx_src; 3785 req_ctx->ctx = &req->base; 3786 req_ctx->edesc = edesc; 3787 3788 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3789 if (ret == -EINPROGRESS || 3790 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3791 return ret; 3792 3793 unmap_ctx: 3794 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3795 qi_cache_free(edesc); 3796 return ret; 3797 } 3798 3799 static int ahash_digest(struct ahash_request *req) 3800 { 3801 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3802 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3803 struct caam_hash_state *state = ahash_request_ctx(req); 3804 struct caam_request *req_ctx = &state->caam_req; 3805 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3806 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3807 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3808 GFP_KERNEL : GFP_ATOMIC; 3809 int digestsize = crypto_ahash_digestsize(ahash); 3810 int src_nents, mapped_nents; 3811 struct ahash_edesc *edesc; 3812 int ret = -ENOMEM; 3813 3814 state->buf_dma = 0; 3815 3816 src_nents = sg_nents_for_len(req->src, req->nbytes); 3817 if (src_nents < 0) { 3818 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3819 return src_nents; 3820 } 3821 3822 if (src_nents) { 3823 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3824 DMA_TO_DEVICE); 3825 if (!mapped_nents) { 3826 dev_err(ctx->dev, "unable to map source for DMA\n"); 3827 return ret; 3828 } 3829 } else { 3830 mapped_nents = 0; 3831 } 3832 3833 /* allocate space for base edesc and link tables */ 3834 edesc = qi_cache_zalloc(GFP_DMA | flags); 3835 if (!edesc) { 3836 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3837 return ret; 3838 } 3839 3840 edesc->src_nents = src_nents; 3841 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3842 3843 if (mapped_nents > 1) { 3844 int qm_sg_bytes; 3845 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3846 3847 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3848 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3849 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3850 qm_sg_bytes, DMA_TO_DEVICE); 3851 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3852 dev_err(ctx->dev, "unable to map S/G table\n"); 3853 goto unmap; 3854 } 3855 edesc->qm_sg_bytes = qm_sg_bytes; 3856 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3857 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3858 } else { 3859 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3860 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3861 } 3862 3863 state->ctx_dma_len = digestsize; 3864 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3865 DMA_FROM_DEVICE); 3866 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3867 dev_err(ctx->dev, "unable to map ctx\n"); 3868 state->ctx_dma = 0; 3869 goto unmap; 3870 } 3871 3872 dpaa2_fl_set_final(in_fle, true); 3873 dpaa2_fl_set_len(in_fle, req->nbytes); 3874 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3875 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3876 dpaa2_fl_set_len(out_fle, digestsize); 3877 3878 req_ctx->flc = &ctx->flc[DIGEST]; 3879 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3880 req_ctx->cbk = ahash_done; 3881 req_ctx->ctx = &req->base; 3882 req_ctx->edesc = edesc; 3883 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3884 if (ret == -EINPROGRESS || 3885 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3886 return ret; 3887 3888 unmap: 3889 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3890 qi_cache_free(edesc); 3891 return ret; 3892 } 3893 3894 static int ahash_final_no_ctx(struct ahash_request *req) 3895 { 3896 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3897 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3898 struct caam_hash_state *state = ahash_request_ctx(req); 3899 struct caam_request *req_ctx = &state->caam_req; 3900 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3901 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3902 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3903 GFP_KERNEL : GFP_ATOMIC; 3904 u8 *buf = state->buf; 3905 int buflen = state->buflen; 3906 int digestsize = crypto_ahash_digestsize(ahash); 3907 struct ahash_edesc *edesc; 3908 int ret = -ENOMEM; 3909 3910 /* allocate space for base edesc and link tables */ 3911 edesc = qi_cache_zalloc(GFP_DMA | flags); 3912 if (!edesc) 3913 return ret; 3914 3915 if (buflen) { 3916 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3917 DMA_TO_DEVICE); 3918 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3919 dev_err(ctx->dev, "unable to map src\n"); 3920 goto unmap; 3921 } 3922 } 3923 3924 state->ctx_dma_len = digestsize; 3925 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3926 DMA_FROM_DEVICE); 3927 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3928 dev_err(ctx->dev, "unable to map ctx\n"); 3929 state->ctx_dma = 0; 3930 goto unmap; 3931 } 3932 3933 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3934 dpaa2_fl_set_final(in_fle, true); 3935 /* 3936 * crypto engine requires the input entry to be present when 3937 * "frame list" FD is used. 3938 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3939 * in_fle zeroized (except for "Final" flag) is the best option. 3940 */ 3941 if (buflen) { 3942 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3943 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3944 dpaa2_fl_set_len(in_fle, buflen); 3945 } 3946 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3947 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3948 dpaa2_fl_set_len(out_fle, digestsize); 3949 3950 req_ctx->flc = &ctx->flc[DIGEST]; 3951 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3952 req_ctx->cbk = ahash_done; 3953 req_ctx->ctx = &req->base; 3954 req_ctx->edesc = edesc; 3955 3956 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3957 if (ret == -EINPROGRESS || 3958 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3959 return ret; 3960 3961 unmap: 3962 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3963 qi_cache_free(edesc); 3964 return ret; 3965 } 3966 3967 static int ahash_update_no_ctx(struct ahash_request *req) 3968 { 3969 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3970 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3971 struct caam_hash_state *state = ahash_request_ctx(req); 3972 struct caam_request *req_ctx = &state->caam_req; 3973 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3974 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3975 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3976 GFP_KERNEL : GFP_ATOMIC; 3977 u8 *buf = state->buf; 3978 int *buflen = &state->buflen; 3979 int *next_buflen = &state->next_buflen; 3980 int in_len = *buflen + req->nbytes, to_hash; 3981 int qm_sg_bytes, src_nents, mapped_nents; 3982 struct ahash_edesc *edesc; 3983 int ret = 0; 3984 3985 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3986 to_hash = in_len - *next_buflen; 3987 3988 if (to_hash) { 3989 struct dpaa2_sg_entry *sg_table; 3990 int src_len = req->nbytes - *next_buflen; 3991 3992 src_nents = sg_nents_for_len(req->src, src_len); 3993 if (src_nents < 0) { 3994 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3995 return src_nents; 3996 } 3997 3998 if (src_nents) { 3999 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4000 DMA_TO_DEVICE); 4001 if (!mapped_nents) { 4002 dev_err(ctx->dev, "unable to DMA map source\n"); 4003 return -ENOMEM; 4004 } 4005 } else { 4006 mapped_nents = 0; 4007 } 4008 4009 /* allocate space for base edesc and link tables */ 4010 edesc = qi_cache_zalloc(GFP_DMA | flags); 4011 if (!edesc) { 4012 dma_unmap_sg(ctx->dev, req->src, src_nents, 4013 DMA_TO_DEVICE); 4014 return -ENOMEM; 4015 } 4016 4017 edesc->src_nents = src_nents; 4018 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 4019 sizeof(*sg_table); 4020 sg_table = &edesc->sgt[0]; 4021 4022 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4023 if (ret) 4024 goto unmap_ctx; 4025 4026 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 4027 4028 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4029 qm_sg_bytes, DMA_TO_DEVICE); 4030 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4031 dev_err(ctx->dev, "unable to map S/G table\n"); 4032 ret = -ENOMEM; 4033 goto unmap_ctx; 4034 } 4035 edesc->qm_sg_bytes = qm_sg_bytes; 4036 4037 state->ctx_dma_len = ctx->ctx_len; 4038 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4039 ctx->ctx_len, DMA_FROM_DEVICE); 4040 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4041 dev_err(ctx->dev, "unable to map ctx\n"); 4042 state->ctx_dma = 0; 4043 ret = -ENOMEM; 4044 goto unmap_ctx; 4045 } 4046 4047 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4048 dpaa2_fl_set_final(in_fle, true); 4049 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4050 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4051 dpaa2_fl_set_len(in_fle, to_hash); 4052 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4053 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4054 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4055 4056 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4057 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4058 req_ctx->cbk = ahash_done_ctx_dst; 4059 req_ctx->ctx = &req->base; 4060 req_ctx->edesc = edesc; 4061 4062 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4063 if (ret != -EINPROGRESS && 4064 !(ret == -EBUSY && 4065 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4066 goto unmap_ctx; 4067 4068 state->update = ahash_update_ctx; 4069 state->finup = ahash_finup_ctx; 4070 state->final = ahash_final_ctx; 4071 } else if (*next_buflen) { 4072 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 4073 req->nbytes, 0); 4074 *buflen = *next_buflen; 4075 4076 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4077 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4078 *buflen, 1); 4079 } 4080 4081 return ret; 4082 unmap_ctx: 4083 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4084 qi_cache_free(edesc); 4085 return ret; 4086 } 4087 4088 static int ahash_finup_no_ctx(struct ahash_request *req) 4089 { 4090 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4091 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4092 struct caam_hash_state *state = ahash_request_ctx(req); 4093 struct caam_request *req_ctx = &state->caam_req; 4094 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4095 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4096 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4097 GFP_KERNEL : GFP_ATOMIC; 4098 int buflen = state->buflen; 4099 int qm_sg_bytes, src_nents, mapped_nents; 4100 int digestsize = crypto_ahash_digestsize(ahash); 4101 struct ahash_edesc *edesc; 4102 struct dpaa2_sg_entry *sg_table; 4103 int ret = -ENOMEM; 4104 4105 src_nents = sg_nents_for_len(req->src, req->nbytes); 4106 if (src_nents < 0) { 4107 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4108 return src_nents; 4109 } 4110 4111 if (src_nents) { 4112 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4113 DMA_TO_DEVICE); 4114 if (!mapped_nents) { 4115 dev_err(ctx->dev, "unable to DMA map source\n"); 4116 return ret; 4117 } 4118 } else { 4119 mapped_nents = 0; 4120 } 4121 4122 /* allocate space for base edesc and link tables */ 4123 edesc = qi_cache_zalloc(GFP_DMA | flags); 4124 if (!edesc) { 4125 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4126 return ret; 4127 } 4128 4129 edesc->src_nents = src_nents; 4130 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4131 sg_table = &edesc->sgt[0]; 4132 4133 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4134 if (ret) 4135 goto unmap; 4136 4137 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4138 4139 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4140 DMA_TO_DEVICE); 4141 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4142 dev_err(ctx->dev, "unable to map S/G table\n"); 4143 ret = -ENOMEM; 4144 goto unmap; 4145 } 4146 edesc->qm_sg_bytes = qm_sg_bytes; 4147 4148 state->ctx_dma_len = digestsize; 4149 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4150 DMA_FROM_DEVICE); 4151 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4152 dev_err(ctx->dev, "unable to map ctx\n"); 4153 state->ctx_dma = 0; 4154 ret = -ENOMEM; 4155 goto unmap; 4156 } 4157 4158 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4159 dpaa2_fl_set_final(in_fle, true); 4160 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4161 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4162 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4163 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4164 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4165 dpaa2_fl_set_len(out_fle, digestsize); 4166 4167 req_ctx->flc = &ctx->flc[DIGEST]; 4168 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4169 req_ctx->cbk = ahash_done; 4170 req_ctx->ctx = &req->base; 4171 req_ctx->edesc = edesc; 4172 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4173 if (ret != -EINPROGRESS && 4174 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4175 goto unmap; 4176 4177 return ret; 4178 unmap: 4179 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4180 qi_cache_free(edesc); 4181 return ret; 4182 } 4183 4184 static int ahash_update_first(struct ahash_request *req) 4185 { 4186 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4187 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4188 struct caam_hash_state *state = ahash_request_ctx(req); 4189 struct caam_request *req_ctx = &state->caam_req; 4190 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4191 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4192 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4193 GFP_KERNEL : GFP_ATOMIC; 4194 u8 *buf = state->buf; 4195 int *buflen = &state->buflen; 4196 int *next_buflen = &state->next_buflen; 4197 int to_hash; 4198 int src_nents, mapped_nents; 4199 struct ahash_edesc *edesc; 4200 int ret = 0; 4201 4202 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4203 1); 4204 to_hash = req->nbytes - *next_buflen; 4205 4206 if (to_hash) { 4207 struct dpaa2_sg_entry *sg_table; 4208 int src_len = req->nbytes - *next_buflen; 4209 4210 src_nents = sg_nents_for_len(req->src, src_len); 4211 if (src_nents < 0) { 4212 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4213 return src_nents; 4214 } 4215 4216 if (src_nents) { 4217 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4218 DMA_TO_DEVICE); 4219 if (!mapped_nents) { 4220 dev_err(ctx->dev, "unable to map source for DMA\n"); 4221 return -ENOMEM; 4222 } 4223 } else { 4224 mapped_nents = 0; 4225 } 4226 4227 /* allocate space for base edesc and link tables */ 4228 edesc = qi_cache_zalloc(GFP_DMA | flags); 4229 if (!edesc) { 4230 dma_unmap_sg(ctx->dev, req->src, src_nents, 4231 DMA_TO_DEVICE); 4232 return -ENOMEM; 4233 } 4234 4235 edesc->src_nents = src_nents; 4236 sg_table = &edesc->sgt[0]; 4237 4238 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4239 dpaa2_fl_set_final(in_fle, true); 4240 dpaa2_fl_set_len(in_fle, to_hash); 4241 4242 if (mapped_nents > 1) { 4243 int qm_sg_bytes; 4244 4245 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4246 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4247 sizeof(*sg_table); 4248 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4249 qm_sg_bytes, 4250 DMA_TO_DEVICE); 4251 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4252 dev_err(ctx->dev, "unable to map S/G table\n"); 4253 ret = -ENOMEM; 4254 goto unmap_ctx; 4255 } 4256 edesc->qm_sg_bytes = qm_sg_bytes; 4257 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4258 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4259 } else { 4260 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4261 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4262 } 4263 4264 state->ctx_dma_len = ctx->ctx_len; 4265 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4266 ctx->ctx_len, DMA_FROM_DEVICE); 4267 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4268 dev_err(ctx->dev, "unable to map ctx\n"); 4269 state->ctx_dma = 0; 4270 ret = -ENOMEM; 4271 goto unmap_ctx; 4272 } 4273 4274 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4275 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4276 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4277 4278 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4279 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4280 req_ctx->cbk = ahash_done_ctx_dst; 4281 req_ctx->ctx = &req->base; 4282 req_ctx->edesc = edesc; 4283 4284 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4285 if (ret != -EINPROGRESS && 4286 !(ret == -EBUSY && req->base.flags & 4287 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4288 goto unmap_ctx; 4289 4290 state->update = ahash_update_ctx; 4291 state->finup = ahash_finup_ctx; 4292 state->final = ahash_final_ctx; 4293 } else if (*next_buflen) { 4294 state->update = ahash_update_no_ctx; 4295 state->finup = ahash_finup_no_ctx; 4296 state->final = ahash_final_no_ctx; 4297 scatterwalk_map_and_copy(buf, req->src, 0, 4298 req->nbytes, 0); 4299 *buflen = *next_buflen; 4300 4301 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4302 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4303 *buflen, 1); 4304 } 4305 4306 return ret; 4307 unmap_ctx: 4308 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4309 qi_cache_free(edesc); 4310 return ret; 4311 } 4312 4313 static int ahash_finup_first(struct ahash_request *req) 4314 { 4315 return ahash_digest(req); 4316 } 4317 4318 static int ahash_init(struct ahash_request *req) 4319 { 4320 struct caam_hash_state *state = ahash_request_ctx(req); 4321 4322 state->update = ahash_update_first; 4323 state->finup = ahash_finup_first; 4324 state->final = ahash_final_no_ctx; 4325 4326 state->ctx_dma = 0; 4327 state->ctx_dma_len = 0; 4328 state->buf_dma = 0; 4329 state->buflen = 0; 4330 state->next_buflen = 0; 4331 4332 return 0; 4333 } 4334 4335 static int ahash_update(struct ahash_request *req) 4336 { 4337 struct caam_hash_state *state = ahash_request_ctx(req); 4338 4339 return state->update(req); 4340 } 4341 4342 static int ahash_finup(struct ahash_request *req) 4343 { 4344 struct caam_hash_state *state = ahash_request_ctx(req); 4345 4346 return state->finup(req); 4347 } 4348 4349 static int ahash_final(struct ahash_request *req) 4350 { 4351 struct caam_hash_state *state = ahash_request_ctx(req); 4352 4353 return state->final(req); 4354 } 4355 4356 static int ahash_export(struct ahash_request *req, void *out) 4357 { 4358 struct caam_hash_state *state = ahash_request_ctx(req); 4359 struct caam_export_state *export = out; 4360 u8 *buf = state->buf; 4361 int len = state->buflen; 4362 4363 memcpy(export->buf, buf, len); 4364 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4365 export->buflen = len; 4366 export->update = state->update; 4367 export->final = state->final; 4368 export->finup = state->finup; 4369 4370 return 0; 4371 } 4372 4373 static int ahash_import(struct ahash_request *req, const void *in) 4374 { 4375 struct caam_hash_state *state = ahash_request_ctx(req); 4376 const struct caam_export_state *export = in; 4377 4378 memset(state, 0, sizeof(*state)); 4379 memcpy(state->buf, export->buf, export->buflen); 4380 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4381 state->buflen = export->buflen; 4382 state->update = export->update; 4383 state->final = export->final; 4384 state->finup = export->finup; 4385 4386 return 0; 4387 } 4388 4389 struct caam_hash_template { 4390 char name[CRYPTO_MAX_ALG_NAME]; 4391 char driver_name[CRYPTO_MAX_ALG_NAME]; 4392 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4393 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4394 unsigned int blocksize; 4395 struct ahash_alg template_ahash; 4396 u32 alg_type; 4397 }; 4398 4399 /* ahash descriptors */ 4400 static struct caam_hash_template driver_hash[] = { 4401 { 4402 .name = "sha1", 4403 .driver_name = "sha1-caam-qi2", 4404 .hmac_name = "hmac(sha1)", 4405 .hmac_driver_name = "hmac-sha1-caam-qi2", 4406 .blocksize = SHA1_BLOCK_SIZE, 4407 .template_ahash = { 4408 .init = ahash_init, 4409 .update = ahash_update, 4410 .final = ahash_final, 4411 .finup = ahash_finup, 4412 .digest = ahash_digest, 4413 .export = ahash_export, 4414 .import = ahash_import, 4415 .setkey = ahash_setkey, 4416 .halg = { 4417 .digestsize = SHA1_DIGEST_SIZE, 4418 .statesize = sizeof(struct caam_export_state), 4419 }, 4420 }, 4421 .alg_type = OP_ALG_ALGSEL_SHA1, 4422 }, { 4423 .name = "sha224", 4424 .driver_name = "sha224-caam-qi2", 4425 .hmac_name = "hmac(sha224)", 4426 .hmac_driver_name = "hmac-sha224-caam-qi2", 4427 .blocksize = SHA224_BLOCK_SIZE, 4428 .template_ahash = { 4429 .init = ahash_init, 4430 .update = ahash_update, 4431 .final = ahash_final, 4432 .finup = ahash_finup, 4433 .digest = ahash_digest, 4434 .export = ahash_export, 4435 .import = ahash_import, 4436 .setkey = ahash_setkey, 4437 .halg = { 4438 .digestsize = SHA224_DIGEST_SIZE, 4439 .statesize = sizeof(struct caam_export_state), 4440 }, 4441 }, 4442 .alg_type = OP_ALG_ALGSEL_SHA224, 4443 }, { 4444 .name = "sha256", 4445 .driver_name = "sha256-caam-qi2", 4446 .hmac_name = "hmac(sha256)", 4447 .hmac_driver_name = "hmac-sha256-caam-qi2", 4448 .blocksize = SHA256_BLOCK_SIZE, 4449 .template_ahash = { 4450 .init = ahash_init, 4451 .update = ahash_update, 4452 .final = ahash_final, 4453 .finup = ahash_finup, 4454 .digest = ahash_digest, 4455 .export = ahash_export, 4456 .import = ahash_import, 4457 .setkey = ahash_setkey, 4458 .halg = { 4459 .digestsize = SHA256_DIGEST_SIZE, 4460 .statesize = sizeof(struct caam_export_state), 4461 }, 4462 }, 4463 .alg_type = OP_ALG_ALGSEL_SHA256, 4464 }, { 4465 .name = "sha384", 4466 .driver_name = "sha384-caam-qi2", 4467 .hmac_name = "hmac(sha384)", 4468 .hmac_driver_name = "hmac-sha384-caam-qi2", 4469 .blocksize = SHA384_BLOCK_SIZE, 4470 .template_ahash = { 4471 .init = ahash_init, 4472 .update = ahash_update, 4473 .final = ahash_final, 4474 .finup = ahash_finup, 4475 .digest = ahash_digest, 4476 .export = ahash_export, 4477 .import = ahash_import, 4478 .setkey = ahash_setkey, 4479 .halg = { 4480 .digestsize = SHA384_DIGEST_SIZE, 4481 .statesize = sizeof(struct caam_export_state), 4482 }, 4483 }, 4484 .alg_type = OP_ALG_ALGSEL_SHA384, 4485 }, { 4486 .name = "sha512", 4487 .driver_name = "sha512-caam-qi2", 4488 .hmac_name = "hmac(sha512)", 4489 .hmac_driver_name = "hmac-sha512-caam-qi2", 4490 .blocksize = SHA512_BLOCK_SIZE, 4491 .template_ahash = { 4492 .init = ahash_init, 4493 .update = ahash_update, 4494 .final = ahash_final, 4495 .finup = ahash_finup, 4496 .digest = ahash_digest, 4497 .export = ahash_export, 4498 .import = ahash_import, 4499 .setkey = ahash_setkey, 4500 .halg = { 4501 .digestsize = SHA512_DIGEST_SIZE, 4502 .statesize = sizeof(struct caam_export_state), 4503 }, 4504 }, 4505 .alg_type = OP_ALG_ALGSEL_SHA512, 4506 }, { 4507 .name = "md5", 4508 .driver_name = "md5-caam-qi2", 4509 .hmac_name = "hmac(md5)", 4510 .hmac_driver_name = "hmac-md5-caam-qi2", 4511 .blocksize = MD5_BLOCK_WORDS * 4, 4512 .template_ahash = { 4513 .init = ahash_init, 4514 .update = ahash_update, 4515 .final = ahash_final, 4516 .finup = ahash_finup, 4517 .digest = ahash_digest, 4518 .export = ahash_export, 4519 .import = ahash_import, 4520 .setkey = ahash_setkey, 4521 .halg = { 4522 .digestsize = MD5_DIGEST_SIZE, 4523 .statesize = sizeof(struct caam_export_state), 4524 }, 4525 }, 4526 .alg_type = OP_ALG_ALGSEL_MD5, 4527 } 4528 }; 4529 4530 struct caam_hash_alg { 4531 struct list_head entry; 4532 struct device *dev; 4533 int alg_type; 4534 struct ahash_alg ahash_alg; 4535 }; 4536 4537 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4538 { 4539 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4540 struct crypto_alg *base = tfm->__crt_alg; 4541 struct hash_alg_common *halg = 4542 container_of(base, struct hash_alg_common, base); 4543 struct ahash_alg *alg = 4544 container_of(halg, struct ahash_alg, halg); 4545 struct caam_hash_alg *caam_hash = 4546 container_of(alg, struct caam_hash_alg, ahash_alg); 4547 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4548 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4549 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4550 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4551 HASH_MSG_LEN + 32, 4552 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4553 HASH_MSG_LEN + 64, 4554 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4555 dma_addr_t dma_addr; 4556 int i; 4557 4558 ctx->dev = caam_hash->dev; 4559 4560 if (alg->setkey) { 4561 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4562 ARRAY_SIZE(ctx->key), 4563 DMA_TO_DEVICE, 4564 DMA_ATTR_SKIP_CPU_SYNC); 4565 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4566 dev_err(ctx->dev, "unable to map key\n"); 4567 return -ENOMEM; 4568 } 4569 } 4570 4571 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4572 DMA_BIDIRECTIONAL, 4573 DMA_ATTR_SKIP_CPU_SYNC); 4574 if (dma_mapping_error(ctx->dev, dma_addr)) { 4575 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4576 if (ctx->adata.key_dma) 4577 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4578 ARRAY_SIZE(ctx->key), 4579 DMA_TO_DEVICE, 4580 DMA_ATTR_SKIP_CPU_SYNC); 4581 return -ENOMEM; 4582 } 4583 4584 for (i = 0; i < HASH_NUM_OP; i++) 4585 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4586 4587 /* copy descriptor header template value */ 4588 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4589 4590 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4591 OP_ALG_ALGSEL_SUBMASK) >> 4592 OP_ALG_ALGSEL_SHIFT]; 4593 4594 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 4595 sizeof(struct caam_hash_state)); 4596 4597 /* 4598 * For keyed hash algorithms shared descriptors 4599 * will be created later in setkey() callback 4600 */ 4601 return alg->setkey ? 0 : ahash_set_sh_desc(ahash); 4602 } 4603 4604 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4605 { 4606 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4607 4608 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4609 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4610 if (ctx->adata.key_dma) 4611 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4612 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4613 DMA_ATTR_SKIP_CPU_SYNC); 4614 } 4615 4616 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4617 struct caam_hash_template *template, bool keyed) 4618 { 4619 struct caam_hash_alg *t_alg; 4620 struct ahash_alg *halg; 4621 struct crypto_alg *alg; 4622 4623 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4624 if (!t_alg) 4625 return ERR_PTR(-ENOMEM); 4626 4627 t_alg->ahash_alg = template->template_ahash; 4628 halg = &t_alg->ahash_alg; 4629 alg = &halg->halg.base; 4630 4631 if (keyed) { 4632 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4633 template->hmac_name); 4634 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4635 template->hmac_driver_name); 4636 } else { 4637 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4638 template->name); 4639 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4640 template->driver_name); 4641 t_alg->ahash_alg.setkey = NULL; 4642 } 4643 alg->cra_module = THIS_MODULE; 4644 alg->cra_init = caam_hash_cra_init; 4645 alg->cra_exit = caam_hash_cra_exit; 4646 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 4647 alg->cra_priority = CAAM_CRA_PRIORITY; 4648 alg->cra_blocksize = template->blocksize; 4649 alg->cra_alignmask = 0; 4650 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4651 4652 t_alg->alg_type = template->alg_type; 4653 t_alg->dev = dev; 4654 4655 return t_alg; 4656 } 4657 4658 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4659 { 4660 struct dpaa2_caam_priv_per_cpu *ppriv; 4661 4662 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4663 napi_schedule_irqoff(&ppriv->napi); 4664 } 4665 4666 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4667 { 4668 struct device *dev = priv->dev; 4669 struct dpaa2_io_notification_ctx *nctx; 4670 struct dpaa2_caam_priv_per_cpu *ppriv; 4671 int err, i = 0, cpu; 4672 4673 for_each_online_cpu(cpu) { 4674 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4675 ppriv->priv = priv; 4676 nctx = &ppriv->nctx; 4677 nctx->is_cdan = 0; 4678 nctx->id = ppriv->rsp_fqid; 4679 nctx->desired_cpu = cpu; 4680 nctx->cb = dpaa2_caam_fqdan_cb; 4681 4682 /* Register notification callbacks */ 4683 ppriv->dpio = dpaa2_io_service_select(cpu); 4684 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4685 if (unlikely(err)) { 4686 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4687 nctx->cb = NULL; 4688 /* 4689 * If no affine DPIO for this core, there's probably 4690 * none available for next cores either. Signal we want 4691 * to retry later, in case the DPIO devices weren't 4692 * probed yet. 4693 */ 4694 err = -EPROBE_DEFER; 4695 goto err; 4696 } 4697 4698 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4699 dev); 4700 if (unlikely(!ppriv->store)) { 4701 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4702 err = -ENOMEM; 4703 goto err; 4704 } 4705 4706 if (++i == priv->num_pairs) 4707 break; 4708 } 4709 4710 return 0; 4711 4712 err: 4713 for_each_online_cpu(cpu) { 4714 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4715 if (!ppriv->nctx.cb) 4716 break; 4717 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4718 } 4719 4720 for_each_online_cpu(cpu) { 4721 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4722 if (!ppriv->store) 4723 break; 4724 dpaa2_io_store_destroy(ppriv->store); 4725 } 4726 4727 return err; 4728 } 4729 4730 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4731 { 4732 struct dpaa2_caam_priv_per_cpu *ppriv; 4733 int i = 0, cpu; 4734 4735 for_each_online_cpu(cpu) { 4736 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4737 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4738 priv->dev); 4739 dpaa2_io_store_destroy(ppriv->store); 4740 4741 if (++i == priv->num_pairs) 4742 return; 4743 } 4744 } 4745 4746 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4747 { 4748 struct dpseci_rx_queue_cfg rx_queue_cfg; 4749 struct device *dev = priv->dev; 4750 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4751 struct dpaa2_caam_priv_per_cpu *ppriv; 4752 int err = 0, i = 0, cpu; 4753 4754 /* Configure Rx queues */ 4755 for_each_online_cpu(cpu) { 4756 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4757 4758 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4759 DPSECI_QUEUE_OPT_USER_CTX; 4760 rx_queue_cfg.order_preservation_en = 0; 4761 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4762 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4763 /* 4764 * Rx priority (WQ) doesn't really matter, since we use 4765 * pull mode, i.e. volatile dequeues from specific FQs 4766 */ 4767 rx_queue_cfg.dest_cfg.priority = 0; 4768 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4769 4770 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4771 &rx_queue_cfg); 4772 if (err) { 4773 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4774 err); 4775 return err; 4776 } 4777 4778 if (++i == priv->num_pairs) 4779 break; 4780 } 4781 4782 return err; 4783 } 4784 4785 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4786 { 4787 struct device *dev = priv->dev; 4788 4789 if (!priv->cscn_mem) 4790 return; 4791 4792 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4793 kfree(priv->cscn_mem); 4794 } 4795 4796 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4797 { 4798 struct device *dev = priv->dev; 4799 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4800 int err; 4801 4802 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 4803 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 4804 if (err) 4805 dev_err(dev, "dpseci_reset() failed\n"); 4806 } 4807 4808 dpaa2_dpseci_congestion_free(priv); 4809 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4810 } 4811 4812 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4813 const struct dpaa2_fd *fd) 4814 { 4815 struct caam_request *req; 4816 u32 fd_err; 4817 4818 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4819 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4820 return; 4821 } 4822 4823 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4824 if (unlikely(fd_err)) 4825 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4826 4827 /* 4828 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4829 * in FD[ERR] or FD[FRC]. 4830 */ 4831 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4832 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4833 DMA_BIDIRECTIONAL); 4834 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4835 } 4836 4837 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4838 { 4839 int err; 4840 4841 /* Retry while portal is busy */ 4842 do { 4843 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4844 ppriv->store); 4845 } while (err == -EBUSY); 4846 4847 if (unlikely(err)) 4848 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4849 4850 return err; 4851 } 4852 4853 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4854 { 4855 struct dpaa2_dq *dq; 4856 int cleaned = 0, is_last; 4857 4858 do { 4859 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4860 if (unlikely(!dq)) { 4861 if (unlikely(!is_last)) { 4862 dev_dbg(ppriv->priv->dev, 4863 "FQ %d returned no valid frames\n", 4864 ppriv->rsp_fqid); 4865 /* 4866 * MUST retry until we get some sort of 4867 * valid response token (be it "empty dequeue" 4868 * or a valid frame). 4869 */ 4870 continue; 4871 } 4872 break; 4873 } 4874 4875 /* Process FD */ 4876 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4877 cleaned++; 4878 } while (!is_last); 4879 4880 return cleaned; 4881 } 4882 4883 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4884 { 4885 struct dpaa2_caam_priv_per_cpu *ppriv; 4886 struct dpaa2_caam_priv *priv; 4887 int err, cleaned = 0, store_cleaned; 4888 4889 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4890 priv = ppriv->priv; 4891 4892 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4893 return 0; 4894 4895 do { 4896 store_cleaned = dpaa2_caam_store_consume(ppriv); 4897 cleaned += store_cleaned; 4898 4899 if (store_cleaned == 0 || 4900 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4901 break; 4902 4903 /* Try to dequeue some more */ 4904 err = dpaa2_caam_pull_fq(ppriv); 4905 if (unlikely(err)) 4906 break; 4907 } while (1); 4908 4909 if (cleaned < budget) { 4910 napi_complete_done(napi, cleaned); 4911 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4912 if (unlikely(err)) 4913 dev_err(priv->dev, "Notification rearm failed: %d\n", 4914 err); 4915 } 4916 4917 return cleaned; 4918 } 4919 4920 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4921 u16 token) 4922 { 4923 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4924 struct device *dev = priv->dev; 4925 int err; 4926 4927 /* 4928 * Congestion group feature supported starting with DPSECI API v5.1 4929 * and only when object has been created with this capability. 4930 */ 4931 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4932 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4933 return 0; 4934 4935 priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, 4936 GFP_KERNEL | GFP_DMA); 4937 if (!priv->cscn_mem) 4938 return -ENOMEM; 4939 4940 priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); 4941 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, 4942 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4943 if (dma_mapping_error(dev, priv->cscn_dma)) { 4944 dev_err(dev, "Error mapping CSCN memory area\n"); 4945 err = -ENOMEM; 4946 goto err_dma_map; 4947 } 4948 4949 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4950 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4951 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4952 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4953 cong_notif_cfg.message_iova = priv->cscn_dma; 4954 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4955 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4956 DPSECI_CGN_MODE_COHERENT_WRITE; 4957 4958 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4959 &cong_notif_cfg); 4960 if (err) { 4961 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4962 goto err_set_cong; 4963 } 4964 4965 return 0; 4966 4967 err_set_cong: 4968 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4969 err_dma_map: 4970 kfree(priv->cscn_mem); 4971 4972 return err; 4973 } 4974 4975 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4976 { 4977 struct device *dev = &ls_dev->dev; 4978 struct dpaa2_caam_priv *priv; 4979 struct dpaa2_caam_priv_per_cpu *ppriv; 4980 int err, cpu; 4981 u8 i; 4982 4983 priv = dev_get_drvdata(dev); 4984 4985 priv->dev = dev; 4986 priv->dpsec_id = ls_dev->obj_desc.id; 4987 4988 /* Get a handle for the DPSECI this interface is associate with */ 4989 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 4990 if (err) { 4991 dev_err(dev, "dpseci_open() failed: %d\n", err); 4992 goto err_open; 4993 } 4994 4995 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 4996 &priv->minor_ver); 4997 if (err) { 4998 dev_err(dev, "dpseci_get_api_version() failed\n"); 4999 goto err_get_vers; 5000 } 5001 5002 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 5003 5004 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 5005 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 5006 if (err) { 5007 dev_err(dev, "dpseci_reset() failed\n"); 5008 goto err_get_vers; 5009 } 5010 } 5011 5012 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 5013 &priv->dpseci_attr); 5014 if (err) { 5015 dev_err(dev, "dpseci_get_attributes() failed\n"); 5016 goto err_get_vers; 5017 } 5018 5019 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 5020 &priv->sec_attr); 5021 if (err) { 5022 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 5023 goto err_get_vers; 5024 } 5025 5026 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 5027 if (err) { 5028 dev_err(dev, "setup_congestion() failed\n"); 5029 goto err_get_vers; 5030 } 5031 5032 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 5033 priv->dpseci_attr.num_tx_queues); 5034 if (priv->num_pairs > num_online_cpus()) { 5035 dev_warn(dev, "%d queues won't be used\n", 5036 priv->num_pairs - num_online_cpus()); 5037 priv->num_pairs = num_online_cpus(); 5038 } 5039 5040 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 5041 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5042 &priv->rx_queue_attr[i]); 5043 if (err) { 5044 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 5045 goto err_get_rx_queue; 5046 } 5047 } 5048 5049 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 5050 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5051 &priv->tx_queue_attr[i]); 5052 if (err) { 5053 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 5054 goto err_get_rx_queue; 5055 } 5056 } 5057 5058 i = 0; 5059 for_each_online_cpu(cpu) { 5060 u8 j; 5061 5062 j = i % priv->num_pairs; 5063 5064 ppriv = per_cpu_ptr(priv->ppriv, cpu); 5065 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 5066 5067 /* 5068 * Allow all cores to enqueue, while only some of them 5069 * will take part in dequeuing. 5070 */ 5071 if (++i > priv->num_pairs) 5072 continue; 5073 5074 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 5075 ppriv->prio = j; 5076 5077 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 5078 priv->rx_queue_attr[j].fqid, 5079 priv->tx_queue_attr[j].fqid); 5080 5081 ppriv->net_dev.dev = *dev; 5082 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 5083 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, 5084 DPAA2_CAAM_NAPI_WEIGHT); 5085 } 5086 5087 return 0; 5088 5089 err_get_rx_queue: 5090 dpaa2_dpseci_congestion_free(priv); 5091 err_get_vers: 5092 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5093 err_open: 5094 return err; 5095 } 5096 5097 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 5098 { 5099 struct device *dev = priv->dev; 5100 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5101 struct dpaa2_caam_priv_per_cpu *ppriv; 5102 int i; 5103 5104 for (i = 0; i < priv->num_pairs; i++) { 5105 ppriv = per_cpu_ptr(priv->ppriv, i); 5106 napi_enable(&ppriv->napi); 5107 } 5108 5109 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 5110 } 5111 5112 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 5113 { 5114 struct device *dev = priv->dev; 5115 struct dpaa2_caam_priv_per_cpu *ppriv; 5116 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5117 int i, err = 0, enabled; 5118 5119 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5120 if (err) { 5121 dev_err(dev, "dpseci_disable() failed\n"); 5122 return err; 5123 } 5124 5125 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5126 if (err) { 5127 dev_err(dev, "dpseci_is_enabled() failed\n"); 5128 return err; 5129 } 5130 5131 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 5132 5133 for (i = 0; i < priv->num_pairs; i++) { 5134 ppriv = per_cpu_ptr(priv->ppriv, i); 5135 napi_disable(&ppriv->napi); 5136 netif_napi_del(&ppriv->napi); 5137 } 5138 5139 return 0; 5140 } 5141 5142 static struct list_head hash_list; 5143 5144 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5145 { 5146 struct device *dev; 5147 struct dpaa2_caam_priv *priv; 5148 int i, err = 0; 5149 bool registered = false; 5150 5151 /* 5152 * There is no way to get CAAM endianness - there is no direct register 5153 * space access and MC f/w does not provide this attribute. 5154 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5155 * property. 5156 */ 5157 caam_little_end = true; 5158 5159 caam_imx = false; 5160 5161 dev = &dpseci_dev->dev; 5162 5163 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5164 if (!priv) 5165 return -ENOMEM; 5166 5167 dev_set_drvdata(dev, priv); 5168 5169 priv->domain = iommu_get_domain_for_dev(dev); 5170 5171 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5172 0, SLAB_CACHE_DMA, NULL); 5173 if (!qi_cache) { 5174 dev_err(dev, "Can't allocate SEC cache\n"); 5175 return -ENOMEM; 5176 } 5177 5178 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5179 if (err) { 5180 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5181 goto err_dma_mask; 5182 } 5183 5184 /* Obtain a MC portal */ 5185 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5186 if (err) { 5187 if (err == -ENXIO) 5188 err = -EPROBE_DEFER; 5189 else 5190 dev_err(dev, "MC portal allocation failed\n"); 5191 5192 goto err_dma_mask; 5193 } 5194 5195 priv->ppriv = alloc_percpu(*priv->ppriv); 5196 if (!priv->ppriv) { 5197 dev_err(dev, "alloc_percpu() failed\n"); 5198 err = -ENOMEM; 5199 goto err_alloc_ppriv; 5200 } 5201 5202 /* DPSECI initialization */ 5203 err = dpaa2_dpseci_setup(dpseci_dev); 5204 if (err) { 5205 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5206 goto err_dpseci_setup; 5207 } 5208 5209 /* DPIO */ 5210 err = dpaa2_dpseci_dpio_setup(priv); 5211 if (err) { 5212 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); 5213 goto err_dpio_setup; 5214 } 5215 5216 /* DPSECI binding to DPIO */ 5217 err = dpaa2_dpseci_bind(priv); 5218 if (err) { 5219 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5220 goto err_bind; 5221 } 5222 5223 /* DPSECI enable */ 5224 err = dpaa2_dpseci_enable(priv); 5225 if (err) { 5226 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5227 goto err_bind; 5228 } 5229 5230 dpaa2_dpseci_debugfs_init(priv); 5231 5232 /* register crypto algorithms the device supports */ 5233 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5234 struct caam_skcipher_alg *t_alg = driver_algs + i; 5235 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5236 5237 /* Skip DES algorithms if not supported by device */ 5238 if (!priv->sec_attr.des_acc_num && 5239 (alg_sel == OP_ALG_ALGSEL_3DES || 5240 alg_sel == OP_ALG_ALGSEL_DES)) 5241 continue; 5242 5243 /* Skip AES algorithms if not supported by device */ 5244 if (!priv->sec_attr.aes_acc_num && 5245 alg_sel == OP_ALG_ALGSEL_AES) 5246 continue; 5247 5248 /* Skip CHACHA20 algorithms if not supported by device */ 5249 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5250 !priv->sec_attr.ccha_acc_num) 5251 continue; 5252 5253 t_alg->caam.dev = dev; 5254 caam_skcipher_alg_init(t_alg); 5255 5256 err = crypto_register_skcipher(&t_alg->skcipher); 5257 if (err) { 5258 dev_warn(dev, "%s alg registration failed: %d\n", 5259 t_alg->skcipher.base.cra_driver_name, err); 5260 continue; 5261 } 5262 5263 t_alg->registered = true; 5264 registered = true; 5265 } 5266 5267 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5268 struct caam_aead_alg *t_alg = driver_aeads + i; 5269 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5270 OP_ALG_ALGSEL_MASK; 5271 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5272 OP_ALG_ALGSEL_MASK; 5273 5274 /* Skip DES algorithms if not supported by device */ 5275 if (!priv->sec_attr.des_acc_num && 5276 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5277 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5278 continue; 5279 5280 /* Skip AES algorithms if not supported by device */ 5281 if (!priv->sec_attr.aes_acc_num && 5282 c1_alg_sel == OP_ALG_ALGSEL_AES) 5283 continue; 5284 5285 /* Skip CHACHA20 algorithms if not supported by device */ 5286 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5287 !priv->sec_attr.ccha_acc_num) 5288 continue; 5289 5290 /* Skip POLY1305 algorithms if not supported by device */ 5291 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5292 !priv->sec_attr.ptha_acc_num) 5293 continue; 5294 5295 /* 5296 * Skip algorithms requiring message digests 5297 * if MD not supported by device. 5298 */ 5299 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5300 !priv->sec_attr.md_acc_num) 5301 continue; 5302 5303 t_alg->caam.dev = dev; 5304 caam_aead_alg_init(t_alg); 5305 5306 err = crypto_register_aead(&t_alg->aead); 5307 if (err) { 5308 dev_warn(dev, "%s alg registration failed: %d\n", 5309 t_alg->aead.base.cra_driver_name, err); 5310 continue; 5311 } 5312 5313 t_alg->registered = true; 5314 registered = true; 5315 } 5316 if (registered) 5317 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5318 5319 /* register hash algorithms the device supports */ 5320 INIT_LIST_HEAD(&hash_list); 5321 5322 /* 5323 * Skip registration of any hashing algorithms if MD block 5324 * is not present. 5325 */ 5326 if (!priv->sec_attr.md_acc_num) 5327 return 0; 5328 5329 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5330 struct caam_hash_alg *t_alg; 5331 struct caam_hash_template *alg = driver_hash + i; 5332 5333 /* register hmac version */ 5334 t_alg = caam_hash_alloc(dev, alg, true); 5335 if (IS_ERR(t_alg)) { 5336 err = PTR_ERR(t_alg); 5337 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5338 alg->hmac_driver_name, err); 5339 continue; 5340 } 5341 5342 err = crypto_register_ahash(&t_alg->ahash_alg); 5343 if (err) { 5344 dev_warn(dev, "%s alg registration failed: %d\n", 5345 t_alg->ahash_alg.halg.base.cra_driver_name, 5346 err); 5347 kfree(t_alg); 5348 } else { 5349 list_add_tail(&t_alg->entry, &hash_list); 5350 } 5351 5352 /* register unkeyed version */ 5353 t_alg = caam_hash_alloc(dev, alg, false); 5354 if (IS_ERR(t_alg)) { 5355 err = PTR_ERR(t_alg); 5356 dev_warn(dev, "%s alg allocation failed: %d\n", 5357 alg->driver_name, err); 5358 continue; 5359 } 5360 5361 err = crypto_register_ahash(&t_alg->ahash_alg); 5362 if (err) { 5363 dev_warn(dev, "%s alg registration failed: %d\n", 5364 t_alg->ahash_alg.halg.base.cra_driver_name, 5365 err); 5366 kfree(t_alg); 5367 } else { 5368 list_add_tail(&t_alg->entry, &hash_list); 5369 } 5370 } 5371 if (!list_empty(&hash_list)) 5372 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5373 5374 return err; 5375 5376 err_bind: 5377 dpaa2_dpseci_dpio_free(priv); 5378 err_dpio_setup: 5379 dpaa2_dpseci_free(priv); 5380 err_dpseci_setup: 5381 free_percpu(priv->ppriv); 5382 err_alloc_ppriv: 5383 fsl_mc_portal_free(priv->mc_io); 5384 err_dma_mask: 5385 kmem_cache_destroy(qi_cache); 5386 5387 return err; 5388 } 5389 5390 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5391 { 5392 struct device *dev; 5393 struct dpaa2_caam_priv *priv; 5394 int i; 5395 5396 dev = &ls_dev->dev; 5397 priv = dev_get_drvdata(dev); 5398 5399 dpaa2_dpseci_debugfs_exit(priv); 5400 5401 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5402 struct caam_aead_alg *t_alg = driver_aeads + i; 5403 5404 if (t_alg->registered) 5405 crypto_unregister_aead(&t_alg->aead); 5406 } 5407 5408 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5409 struct caam_skcipher_alg *t_alg = driver_algs + i; 5410 5411 if (t_alg->registered) 5412 crypto_unregister_skcipher(&t_alg->skcipher); 5413 } 5414 5415 if (hash_list.next) { 5416 struct caam_hash_alg *t_hash_alg, *p; 5417 5418 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5419 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5420 list_del(&t_hash_alg->entry); 5421 kfree(t_hash_alg); 5422 } 5423 } 5424 5425 dpaa2_dpseci_disable(priv); 5426 dpaa2_dpseci_dpio_free(priv); 5427 dpaa2_dpseci_free(priv); 5428 free_percpu(priv->ppriv); 5429 fsl_mc_portal_free(priv->mc_io); 5430 kmem_cache_destroy(qi_cache); 5431 5432 return 0; 5433 } 5434 5435 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5436 { 5437 struct dpaa2_fd fd; 5438 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5439 struct dpaa2_caam_priv_per_cpu *ppriv; 5440 int err = 0, i; 5441 5442 if (IS_ERR(req)) 5443 return PTR_ERR(req); 5444 5445 if (priv->cscn_mem) { 5446 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5447 DPAA2_CSCN_SIZE, 5448 DMA_FROM_DEVICE); 5449 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { 5450 dev_dbg_ratelimited(dev, "Dropping request\n"); 5451 return -EBUSY; 5452 } 5453 } 5454 5455 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5456 5457 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5458 DMA_BIDIRECTIONAL); 5459 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5460 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5461 goto err_out; 5462 } 5463 5464 memset(&fd, 0, sizeof(fd)); 5465 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5466 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5467 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5468 dpaa2_fd_set_flc(&fd, req->flc_dma); 5469 5470 ppriv = this_cpu_ptr(priv->ppriv); 5471 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5472 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5473 &fd); 5474 if (err != -EBUSY) 5475 break; 5476 5477 cpu_relax(); 5478 } 5479 5480 if (unlikely(err)) { 5481 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5482 goto err_out; 5483 } 5484 5485 return -EINPROGRESS; 5486 5487 err_out: 5488 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5489 DMA_BIDIRECTIONAL); 5490 return -EIO; 5491 } 5492 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5493 5494 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5495 { 5496 .vendor = FSL_MC_VENDOR_FREESCALE, 5497 .obj_type = "dpseci", 5498 }, 5499 { .vendor = 0x0 } 5500 }; 5501 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table); 5502 5503 static struct fsl_mc_driver dpaa2_caam_driver = { 5504 .driver = { 5505 .name = KBUILD_MODNAME, 5506 .owner = THIS_MODULE, 5507 }, 5508 .probe = dpaa2_caam_probe, 5509 .remove = dpaa2_caam_remove, 5510 .match_id_table = dpaa2_caam_match_id_table 5511 }; 5512 5513 MODULE_LICENSE("Dual BSD/GPL"); 5514 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5515 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5516 5517 module_fsl_mc_driver(dpaa2_caam_driver); 5518